diff --git a/docker-compose.jobs.yml b/docker-compose.jobs.yml index a5a6c553b..de928cf31 100644 --- a/docker-compose.jobs.yml +++ b/docker-compose.jobs.yml @@ -1,21 +1,19 @@ services: - # Constructive GraphQL API server - constructive-server: - container_name: constructive-server + # Constructive Admin GraphQL API server (internal, header-based routing) + constructive-admin-server: + container_name: constructive-admin-server image: constructive:dev build: context: . dockerfile: ./Dockerfile - # The image entrypoint already runs the Constructive CLI (`constructive`). - # We only need to provide the subcommand and flags here. - entrypoint: ["constructive", "server", "--port", "3000", "--origin", "*", "--strictAuth", "false"] + entrypoint: ["constructive", "server", "--host", "0.0.0.0", "--port", "3000", "--origin", "*"] environment: NODE_ENV: development # Server PORT: "3000" SERVER_HOST: "0.0.0.0" SERVER_TRUST_PROXY: "true" - SERVER_ORIGIN: "*" # allow all origins in dev + SERVER_ORIGIN: "*" SERVER_STRICT_AUTH: "false" # Postgres connection (matches postgres service) PGHOST: postgres @@ -23,42 +21,56 @@ services: PGUSER: postgres PGPASSWORD: password PGDATABASE: constructive - # API meta configuration (static mode for dev) - API_ENABLE_META: "true" - API_EXPOSED_SCHEMAS: "metaschema_public,services_public" + # Api configuration + API_ENABLE_SERVICES: "true" + API_EXPOSED_SCHEMAS: "metaschema_public,services_public,constructive_auth_public" + # API_IS_PUBLIC=false enables header-based routing (X-Api-Name, X-Database-Id, X-Meta-Schema) + API_IS_PUBLIC: "false" + # Meta schemas used for schema validation and X-Meta-Schema routing + API_META_SCHEMAS: "metaschema_public,services_public,metaschema_modules_public,constructive_auth_public" API_ANON_ROLE: "administrator" API_ROLE_NAME: "administrator" - API_DEFAULT_DATABASE_ID: "dbe" ports: - - "3000:3000" + - "3101:3000" networks: constructive-net: aliases: - # Let other containers call the admin API using the seeded domain route. - - admin.localhost + - constructive-admin-server - # Simple email function (Knative-style HTTP function) - simple-email: - container_name: simple-email + # Constructive Public GraphQL API server (external, domain-based routing) + constructive-server: + container_name: constructive-server image: constructive:dev - # Override the image entrypoint (Constructive CLI) and run the Node function directly. - entrypoint: ["node", "functions/simple-email/dist/index.js"] + entrypoint: ["constructive", "server", "--host", "0.0.0.0", "--port", "3000", "--origin", "*"] environment: NODE_ENV: development - LOG_LEVEL: info - SIMPLE_EMAIL_DRY_RUN: "${SIMPLE_EMAIL_DRY_RUN:-true}" - # Mailgun / email provider configuration for the Postmaster package - # Replace with real credentials for local testing. - MAILGUN_API_KEY: "${MAILGUN_API_KEY:-change-me-mailgun-api-key}" - MAILGUN_KEY: "${MAILGUN_KEY:-change-me-mailgun-api-key}" - MAILGUN_DOMAIN: "mg.constructive.io" - MAILGUN_FROM: "no-reply@mg.constructive.io" - MAILGUN_REPLY: "info@mg.constructive.io" + # Server + PORT: "3000" + SERVER_HOST: "0.0.0.0" + SERVER_TRUST_PROXY: "true" + SERVER_ORIGIN: "*" + SERVER_STRICT_AUTH: "false" + # Postgres connection (matches postgres service) + PGHOST: postgres + PGPORT: "5432" + PGUSER: postgres + PGPASSWORD: password + PGDATABASE: constructive + # Api configuration + API_ENABLE_SERVICES: "false" + API_EXPOSED_SCHEMAS: "metaschema_public,services_public,constructive_auth_public" + # Public-facing server + API_IS_PUBLIC: "true" + # Meta schemas used for schema validation + API_META_SCHEMAS: "metaschema_public,services_public,metaschema_modules_public,constructive_auth_public" + API_ANON_ROLE: "anonymous" + API_ROLE_NAME: "authenticated" ports: - # Expose function locally (optional) - - "8081:8080" + - "3102:3000" networks: - - constructive-net + constructive-net: + aliases: + - constructive-server # Send email link function (invite, password reset, verification) send-email-link: @@ -69,9 +81,11 @@ services: NODE_ENV: development LOG_LEVEL: info DEFAULT_DATABASE_ID: "dbe" - # Constructive selects the API by Host header; use a seeded domain route. - GRAPHQL_URL: "http://admin.localhost:3000/graphql" - META_GRAPHQL_URL: "http://admin.localhost:3000/graphql" + # Point to admin server (uses X-Api-Name header routing when API_IS_PUBLIC=false) + GRAPHQL_URL: "http://constructive-admin-server:3000/graphql" + META_GRAPHQL_URL: "http://constructive-admin-server:3000/graphql" + # API name for header-based routing (X-Api-Name header) - kept for future use + GRAPHQL_API_NAME: "private" # Optional: provide an existing API token (Bearer) if your server requires it. GRAPHQL_AUTH_TOKEN: "${GRAPHQL_AUTH_TOKEN:-}" # Mailgun / email provider configuration for the Postmaster package @@ -86,7 +100,6 @@ services: LOCAL_APP_PORT: "3000" SEND_EMAIL_LINK_DRY_RUN: "${SEND_EMAIL_LINK_DRY_RUN:-true}" ports: - # Expose function locally (optional) - "8082:8080" networks: - constructive-net @@ -95,10 +108,8 @@ services: knative-job-service: container_name: knative-job-service image: constructive:dev - # Override the image entrypoint and run the jobs runtime directly. entrypoint: ["node", "jobs/knative-job-service/dist/run.js"] depends_on: - - simple-email - send-email-link environment: NODE_ENV: development @@ -113,7 +124,7 @@ services: # Worker configuration JOBS_SUPPORT_ANY: "false" - JOBS_SUPPORTED: "simple-email,send-email-link" + JOBS_SUPPORTED: "send-email-link" HOSTNAME: "knative-job-service-1" # Callback HTTP server (job completion callbacks) @@ -123,12 +134,10 @@ services: JOBS_CALLBACK_HOST: "knative-job-service" # Function gateway base URL (used by worker when no dev map is present) - INTERNAL_GATEWAY_URL: "http://simple-email:8080" + INTERNAL_GATEWAY_URL: "http://send-email-link:8080" # Development-only map from task identifier -> function URL - # Used by @constructive-io/knative-job-worker when NODE_ENV !== 'production'. - # This lets the worker call the function containers directly in docker-compose. - INTERNAL_GATEWAY_DEVELOPMENT_MAP: '{"simple-email":"http://simple-email:8080","send-email-link":"http://send-email-link:8080"}' + INTERNAL_GATEWAY_DEVELOPMENT_MAP: '{"send-email-link":"http://send-email-link:8080"}' ports: - "8080:8080" diff --git a/docker-compose.yml b/docker-compose.yml index a92fe3f37..c1046ddeb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: postgres: container_name: postgres - image: pyramation/pgvector:13.3-alpine + image: ghcr.io/constructive-io/docker/postgres-plus:17 environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=password diff --git a/functions/send-email-link/src/index.ts b/functions/send-email-link/src/index.ts index d90cdabec..77ecd33f9 100644 --- a/functions/send-email-link/src/index.ts +++ b/functions/send-email-link/src/index.ts @@ -77,9 +77,20 @@ const getRequiredEnv = (name: string): string => { return value; }; +type GraphQLClientOptions = { + hostHeaderEnvVar?: string; + databaseId?: string; + useMetaSchema?: boolean; + apiName?: string; + schemata?: string; +}; + +// TODO: Consider moving this to @constructive-io/knative-job-fn as a shared +// utility so all job functions can create GraphQL clients with consistent +// header-based routing without duplicating this logic. const createGraphQLClient = ( url: string, - hostHeaderEnvVar?: string + options: GraphQLClientOptions = {} ): GraphQLClient => { const headers: Record = {}; @@ -87,12 +98,26 @@ const createGraphQLClient = ( headers.Authorization = `Bearer ${process.env.GRAPHQL_AUTH_TOKEN}`; } - const envName = hostHeaderEnvVar || 'GRAPHQL_HOST_HEADER'; + const envName = options.hostHeaderEnvVar || 'GRAPHQL_HOST_HEADER'; const hostHeader = process.env[envName]; if (hostHeader) { headers.host = hostHeader; } + // Header-based routing for internal cluster services (API_IS_PUBLIC=false) + if (options.databaseId) { + headers['X-Database-Id'] = options.databaseId; + } + if (options.useMetaSchema) { + headers['X-Meta-Schema'] = 'true'; + } + if (options.apiName) { + headers['X-Api-Name'] = options.apiName; + } + if (options.schemata) { + headers['X-Schemata'] = options.schemata; + } + return new GraphQLClient(url, { headers }); }; @@ -163,7 +188,19 @@ export const sendEmailLink = async ( const name = company.name; const primary = theme.primary; - const hostname = subdomain ? [subdomain, domain].join('.') : domain; + // Check if this is a localhost-style domain before building hostname + // TODO: Security consideration - this only affects localhost domains which + // should not exist in production. The isLocalHost check combined with isDryRun + // ensures special behavior (http, custom port) only applies in dev environments. + const isLocalDomain = + domain === 'localhost' || + domain.startsWith('localhost') || + domain === '0.0.0.0'; + + // For localhost, skip subdomain to generate cleaner URLs (http://localhost:3000) + const hostname = subdomain && !isLocalDomain + ? [subdomain, domain].join('.') + : domain; // Treat localhost-style hosts specially so we can generate // http://localhost[:port]/... links for local dev without @@ -313,8 +350,26 @@ app.post('/', async (req: any, res: any, next: any) => { const graphqlUrl = getRequiredEnv('GRAPHQL_URL'); const metaGraphqlUrl = process.env.META_GRAPHQL_URL || graphqlUrl; - const client = createGraphQLClient(graphqlUrl, 'GRAPHQL_HOST_HEADER'); - const meta = createGraphQLClient(metaGraphqlUrl, 'META_GRAPHQL_HOST_HEADER'); + // Get API name or schemata from env (for tenant queries like GetUser) + const apiName = process.env.GRAPHQL_API_NAME; + const schemata = process.env.GRAPHQL_SCHEMATA; + + // For GetUser query - needs tenant API access via X-Api-Name or X-Schemata + const client = createGraphQLClient(graphqlUrl, { + hostHeaderEnvVar: 'GRAPHQL_HOST_HEADER', + databaseId, + ...(apiName && { apiName }), + ...(schemata && { schemata }), + }); + + // For GetDatabaseInfo query - uses same API routing as client + // The private API exposes both user and database queries + const meta = createGraphQLClient(metaGraphqlUrl, { + hostHeaderEnvVar: 'META_GRAPHQL_HOST_HEADER', + databaseId, + ...(apiName && { apiName }), + ...(schemata && { schemata }), + }); const result = await sendEmailLink(params, { client, @@ -322,6 +377,11 @@ app.post('/', async (req: any, res: any, next: any) => { databaseId }); + // Validation failures return { missing: '...' } - treat as client error + if (result && typeof result === 'object' && 'missing' in result) { + return res.status(400).json({ error: `Missing required field: ${result.missing}` }); + } + res.status(200).json(result); } catch (err) { next(err); @@ -333,6 +393,22 @@ export default app; // When executed directly (e.g. via `node dist/index.js`), start an HTTP server. if (require.main === module) { const port = Number(process.env.PORT ?? 8080); + + // Log startup configuration (non-sensitive values only - no API keys or tokens) + logger.info('[send-email-link] Starting with config:', { + port, + graphqlUrl: process.env.GRAPHQL_URL || 'not set', + metaGraphqlUrl: process.env.META_GRAPHQL_URL || process.env.GRAPHQL_URL || 'not set', + apiName: process.env.GRAPHQL_API_NAME || 'not set', + defaultDatabaseId: process.env.DEFAULT_DATABASE_ID || 'not set', + dryRun: isDryRun, + useSmtp, + mailgunDomain: process.env.MAILGUN_DOMAIN || 'not set', + mailgunFrom: process.env.MAILGUN_FROM || 'not set', + localAppPort: process.env.LOCAL_APP_PORT || 'not set', + hasAuthToken: !!process.env.GRAPHQL_AUTH_TOKEN + }); + // @constructive-io/knative-job-fn exposes a .listen method that delegates to the Express app (app as any).listen(port, () => { logger.info(`listening on port ${port}`); diff --git a/graphql/env/src/env.ts b/graphql/env/src/env.ts index cb7b3fce3..2211526c4 100644 --- a/graphql/env/src/env.ts +++ b/graphql/env/src/env.ts @@ -31,10 +31,10 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial return { graphile: { - ...(GRAPHILE_SCHEMA && { - schema: GRAPHILE_SCHEMA.includes(',') + ...(GRAPHILE_SCHEMA && { + schema: GRAPHILE_SCHEMA.includes(',') ? GRAPHILE_SCHEMA.split(',').map(s => s.trim()) - : GRAPHILE_SCHEMA + : GRAPHILE_SCHEMA }), }, features: { diff --git a/graphql/server/src/middleware/api.ts b/graphql/server/src/middleware/api.ts index cf4a82fe6..dcc8e7100 100644 --- a/graphql/server/src/middleware/api.ts +++ b/graphql/server/src/middleware/api.ts @@ -80,11 +80,19 @@ export const getSubdomain = (reqDomains: string[]): string | null => { }; export const createApiMiddleware = (opts: any) => { + // Log middleware initialization once + const apiPublicSetting = opts.api?.isPublic; + log.info(`[api-middleware] Initialized: isPublic=${apiPublicSetting}, enableServicesApi=${opts.api?.enableServicesApi}`); + return async ( req: Request, res: Response, next: NextFunction ): Promise => { + // Log incoming request details at debug level to avoid excessive info logs in production + log.debug(`[api-middleware] Request: ${req.method} ${req.path}`); + log.debug(`[api-middleware] Headers: X-Api-Name=${req.get('X-Api-Name')}, X-Database-Id=${req.get('X-Database-Id')}, X-Meta-Schema=${req.get('X-Meta-Schema')}, Host=${req.get('Host')}`); + if (opts.api?.enableServicesApi === false) { const schemas = opts.api.exposedSchemas; const anonRole = opts.api.anonRole; @@ -169,13 +177,13 @@ const getHardCodedSchemata = ({ dbname: opts.pg.database, anonRole: 'administrator', roleName: 'administrator', - schemaNamesFromExt: { + apiExtensions: { nodes: schemata .split(',') .map((schema) => schema.trim()) .map((schemaName) => ({ schemaName })), }, - schemaNames: { nodes: [] as Array<{ schemaName: string }> }, + schemasByApiSchemaApiIdAndSchemaId: { nodes: [] as Array<{ schemaName: string }> }, apiModules: [] as Array, }, }, @@ -203,10 +211,10 @@ const getMetaSchema = ({ dbname: opts.pg.database, anonRole: 'administrator', roleName: 'administrator', - schemaNamesFromExt: { + apiExtensions: { nodes: schemata.map((schemaName: string) => ({ schemaName })), }, - schemaNames: { nodes: [] as Array<{ schemaName: string }> }, + schemasByApiSchemaApiIdAndSchemaId: { nodes: [] as Array<{ schemaName: string }> }, apiModules: [] as Array, }, }, @@ -280,8 +288,10 @@ const queryServiceByApiName = async ({ const data = result?.data; const apiPublic = (opts as any).api?.isPublic; - if (data?.api && data.api.isPublic === apiPublic) { - const svc = { data }; + const apiData = data?.apiByDatabaseIdAndName; + if (apiData && apiData.isPublic === apiPublic) { + // Restructure to match what transformServiceToApi expects (svc.data.api) + const svc = { data: { api: apiData } }; svcCache.set(key, svc); return svc; } @@ -366,8 +376,12 @@ export const getApiConfig = async ( const client = new GraphileQuery({ schema, pool: rootPgPool, settings }); const apiPublic = (opts as any).api?.isPublic; + log.debug(`[api-middleware] Routing: apiPublic=${apiPublic} (type: ${typeof apiPublic})`); + if (apiPublic === false) { + log.debug(`[api-middleware] Using header-based routing (apiPublic === false)`); if (req.get('X-Schemata')) { + log.debug(`[api-middleware] Route: X-Schemata`); svc = getHardCodedSchemata({ opts, key, @@ -375,6 +389,7 @@ export const getApiConfig = async ( databaseId: req.get('X-Database-Id'), }); } else if (req.get('X-Api-Name')) { + log.debug(`[api-middleware] Route: X-Api-Name=${req.get('X-Api-Name')}, X-Database-Id=${req.get('X-Database-Id')}`); svc = await queryServiceByApiName({ opts, key, @@ -382,13 +397,16 @@ export const getApiConfig = async ( name: req.get('X-Api-Name'), databaseId: req.get('X-Database-Id'), }); + log.debug(`[api-middleware] queryServiceByApiName result: ${svc ? 'found' : 'null'}`); } else if (req.get('X-Meta-Schema')) { + log.debug(`[api-middleware] Route: X-Meta-Schema`); svc = getMetaSchema({ opts, key, databaseId: req.get('X-Database-Id'), }); } else { + log.debug(`[api-middleware] Route: domain/subdomain fallback`); svc = await queryServiceByDomainAndSubdomain({ opts, key, @@ -398,6 +416,7 @@ export const getApiConfig = async ( }); } } else { + log.debug(`[api-middleware] Using domain-based routing (apiPublic !== false)`); svc = await queryServiceByDomainAndSubdomain({ opts, key, diff --git a/graphql/server/src/server.ts b/graphql/server/src/server.ts index 300a9110a..c084ed855 100644 --- a/graphql/server/src/server.ts +++ b/graphql/server/src/server.ts @@ -77,15 +77,21 @@ class Server { next(); }; - // Log startup config in dev mode - if (isDev()) { - log.debug( - `Database: ${effectiveOpts.pg?.database}@${effectiveOpts.pg?.host}:${effectiveOpts.pg?.port}` - ); - log.debug( - `Meta schemas: ${(effectiveOpts as any).api?.metaSchemas?.join(', ') || 'default'}` - ); - } + // Log startup configuration (non-sensitive values only) + const apiOpts = (effectiveOpts as any).api || {}; + log.info('[server] Starting with config:', { + database: effectiveOpts.pg?.database, + host: effectiveOpts.pg?.host, + port: effectiveOpts.pg?.port, + serverHost: effectiveOpts.server?.host, + serverPort: effectiveOpts.server?.port, + apiIsPublic: apiOpts.isPublic, + enableServicesApi: apiOpts.enableServicesApi, + metaSchemas: apiOpts.metaSchemas?.join(',') || 'default', + exposedSchemas: apiOpts.exposedSchemas?.join(',') || 'none', + anonRole: apiOpts.anonRole, + roleName: apiOpts.roleName + }); healthz(app); trustProxy(app, effectiveOpts.server.trustProxy); diff --git a/jobs/DEVELOPMENT_JOBS.md b/jobs/DEVELOPMENT_JOBS.md index 8604f3f55..1440ee831 100644 --- a/jobs/DEVELOPMENT_JOBS.md +++ b/jobs/DEVELOPMENT_JOBS.md @@ -3,8 +3,7 @@ This guide covers a local development workflow for the jobs stack: - Postgres + `pgpm-database-jobs` -- Constructive GraphQL API server -- `simple-email` function +- Constructive Admin GraphQL API server (header-based routing) - `send-email-link` function - `knative-job-service` @@ -102,16 +101,15 @@ docker compose -f docker-compose.jobs.yml up --build This starts: -- `constructive-server` – GraphQL API server -- `simple-email` – Knative-style HTTP function -- `send-email-link` – Knative-style HTTP function -- `knative-job-service` – jobs runtime (callback server + worker + scheduler) +- `constructive-admin-server` – GraphQL API server with `API_IS_PUBLIC=false` (port 3001) +- `send-email-link` – Knative-style HTTP function (port 8082) +- `knative-job-service` – jobs runtime (callback server + worker + scheduler) (port 8080) --- ### Switching dry run vs real Mailgun sending -By default, `docker-compose.jobs.yml` runs both email functions in dry-run mode (no real email is sent), and it uses placeholder Mailgun credentials. +By default, `docker-compose.jobs.yml` runs `send-email-link` in dry-run mode (no real email is sent), and it uses placeholder Mailgun credentials. Dry run (recommended for local development): @@ -121,74 +119,110 @@ docker compose -f docker-compose.jobs.yml up -d --build --force-recreate In dry-run mode: -- The `simple-email` and `send-email-link` containers log the payload they would send instead of hitting Mailgun. -- You should see log lines like `[simple-email] DRY RUN email (skipping send) ...` and `[send-email-link] DRY RUN email (skipping send) ...`. +- The `send-email-link` container logs the payload it would send instead of hitting Mailgun. +- You should see log lines like `[send-email-link] DRY RUN email (skipping send) ...`. --- ## 5. Ensure GraphQL host routing works for `send-email-link` -Constructive selects the API by the HTTP `Host` header using rows in `services_public.domains`. +The `send-email-link` function uses host-based routing via the `Host: private.localhost` header to access the private API. -For local development, `app-svc-local` seeds `admin.localhost` as the admin API domain. `docker-compose.jobs.yml` adds a Docker network alias so other containers can resolve `admin.localhost` to the `constructive-server` container, and `send-email-link` uses: +For local development, `docker-compose.jobs.yml` configures `send-email-link` with: -- `GRAPHQL_URL=http://admin.localhost:3000/graphql` +- `GRAPHQL_URL=http://constructive-admin-server:3000/graphql` +- `GRAPHQL_HOST_HEADER=private.localhost` -Quick check from your host (should return JSON, not HTML): +Quick check from your host (should return JSON with schema info): ```sh -curl -s -H 'Host: admin.localhost' \ +# Test private API access via host header routing +curl -s -H 'Host: private.localhost' \ -H 'Content-Type: application/json' \ - -X POST http://localhost:3000/graphql \ - --data '{"query":"query { __typename }"}' + -X POST http://localhost:3001/graphql \ + --data '{"query":"{ __schema { queryType { fields { name } } } }"}' + +# List databases +curl -s -H 'Host: private.localhost' \ + -H 'Content-Type: application/json' \ + -X POST http://localhost:3001/graphql \ + --data '{"query":"{ databases { nodes { id name } } }"}' + +# List users +curl -s -H 'Host: private.localhost' \ + -H 'Content-Type: application/json' \ + -X POST http://localhost:3001/graphql \ + --data '{"query":"{ users { nodes { id username displayName } } }"}' ``` +You can also access GraphiQL at: http://private.localhost:3001/graphiql + If your GraphQL server requires auth, set `GRAPHQL_AUTH_TOKEN` before starting the jobs stack (it is passed through to the `send-email-link` container). --- -## 6. Enqueue a test job (simple-email) +## 6. Enqueue a test job (`send-email-link`) + +`send-email-link` queries GraphQL for site/database metadata, so it requires: -With the jobs stack running, you can enqueue a test job from your host into the Postgres container: +- The app/meta packages deployed in step 3 (`app-svc-local`, `metaschema-schema`, `services`, `metaschema-modules`) +- A real `database_id` +- A GraphQL hostname that matches a seeded domain route (step 5) +- For localhost development, the site/domain metadata usually resolves to `localhost`. + In that case, the function will honor the `LOCAL_APP_PORT` env (default `3000` in + `docker-compose.jobs.yml`) and generate links like `http://localhost:3000/...` + when `SEND_EMAIL_LINK_DRY_RUN=true`. -First, grab a real `database_id` (required by `send-email-link`, optional for `simple-email`): +### Get required IDs ```sh -DBID="$(docker exec -i postgres psql -U postgres -d constructive -Atc 'SELECT id FROM metaschema_public.database ORDER BY created_at LIMIT 1;')" -echo "$DBID" +# Get Database ID +DBID="$(docker exec -i postgres psql -U postgres -d constructive -Atc \ + 'SELECT id FROM metaschema_public.database ORDER BY created_at LIMIT 1;')" +echo "Database ID: $DBID" + +# Get User ID (for sender_id in invite emails) +SENDER_ID="$(docker exec -i postgres psql -U postgres -d constructive -Atc \ + 'SELECT id FROM roles_public.users ORDER BY created_at LIMIT 1;')" +echo "Sender ID: $SENDER_ID" ``` +### Enqueue invite_email job + ```sh docker exec -it postgres \ psql -U postgres -d constructive -c " SELECT app_jobs.add_job( '$DBID'::uuid, - 'simple-email', + 'send-email-link', json_build_object( - 'to', 'user@example.com', - 'subject', 'Hello from Constructive jobs', - 'html', '

Hi from simple-email (dry run)

' + 'email_type', 'invite_email', + 'email', 'user@example.com', + 'invite_token', 'invite-token-123', + 'sender_id', '$SENDER_ID' )::json ); " ``` -You should then see the job picked up by `knative-job-service` and the email payload logged by the `simple-email` container in `docker compose -f docker-compose.jobs.yml logs -f`. - ---- - -## 7. Enqueue a test job (`send-email-link`) - -`send-email-link` queries GraphQL for site/database metadata, so it requires: +### Enqueue forgot_password job -- The app/meta packages deployed in step 3 (`app-svc-local`, `metaschema-schema`, `services`, `metaschema-modules`) -- A real `database_id` (use `$DBID` above) -- A GraphQL hostname that matches a seeded domain route (step 5) -- For localhost development, the site/domain metadata usually resolves to `localhost`. - In that case, the function will honor the `LOCAL_APP_PORT` env (default `3000` in - `docker-compose.jobs.yml`) and generate links like `http://localhost:3000/...` - when `SEND_EMAIL_LINK_DRY_RUN=true`. +```sh +docker exec -it postgres \ + psql -U postgres -d constructive -c " + SELECT app_jobs.add_job( + '$DBID'::uuid, + 'send-email-link', + json_build_object( + 'email_type', 'forgot_password', + 'email', 'user@example.com', + 'user_id', '$SENDER_ID', + 'reset_token', 'reset-token-123' + )::json + ); + " +``` -With `SEND_EMAIL_LINK_DRY_RUN=true` (default in `docker-compose.jobs.yml`), enqueue a job: +### Enqueue email_verification job ```sh docker exec -it postgres \ @@ -197,22 +231,32 @@ docker exec -it postgres \ '$DBID'::uuid, 'send-email-link', json_build_object( - 'email_type', 'invite_email', + 'email_type', 'email_verification', 'email', 'user@example.com', - 'invite_token', 'invite123', - 'sender_id', '00000000-0000-0000-0000-000000000000' + 'email_id', '$(uuidgen)', + 'verification_token', 'verify-token-123' )::json ); " ``` +### Watch the logs + +```sh +# Watch send-email-link function logs +docker logs -f send-email-link + +# Watch job service logs +docker logs -f knative-job-service +``` + You should see a log like: - `[send-email-link] DRY RUN email (skipping send) ...` --- -## 8. Inspect logs and iterate +## 7. Inspect logs and iterate To watch logs while you develop: @@ -222,8 +266,8 @@ docker compose -f docker-compose.jobs.yml logs -f Useful containers: -- `constructive-server` -- `simple-email` +- `constructive-admin-server` +- `send-email-link` - `knative-job-service` - `postgres` (from `docker-compose.yml`) @@ -236,7 +280,7 @@ docker compose -f docker-compose.jobs.yml up --build --- -## 9. Stopping services +## 8. Stopping services To stop only the jobs stack: @@ -252,14 +296,14 @@ docker compose down --- -## 10. Optional Mailgun secrets for real sending +## 9. Optional Mailgun secrets for real sending Real Mailgun credentials are **not required** to run the jobs stack locally; they are only needed if you want to send real email in development instead of using dry-run logging. To start the stack with real sending from the command line: ```sh -MAILGUN_API_KEY="your-mailgun-key" MAILGUN_KEY="your-mailgun-key" SIMPLE_EMAIL_DRY_RUN=false SEND_EMAIL_LINK_DRY_RUN=false docker compose -f docker-compose.jobs.yml up -d --build --force-recreate +MAILGUN_API_KEY="your-mailgun-key" MAILGUN_KEY="your-mailgun-key" SEND_EMAIL_LINK_DRY_RUN=false docker compose -f docker-compose.jobs.yml up -d --build --force-recreate ``` Alternatively, you can set the secrets in your shell or a local `.env` file (do not commit this file) in the `constructive/` directory: @@ -271,10 +315,9 @@ export MAILGUN_KEY="your-mailgun-key" If you're not using `mg.constructive.io`, also override `MAILGUN_DOMAIN`, `MAILGUN_FROM`, and `MAILGUN_REPLY` (for example in an override file) to match your Mailgun setup. -To have the containers send real email instead of dry-run, set: +To have the container send real email instead of dry-run, set: ```sh -export SIMPLE_EMAIL_DRY_RUN=false export SEND_EMAIL_LINK_DRY_RUN=false ``` @@ -288,10 +331,6 @@ If you prefer not to export env vars, create a local override file (don't commit ```yml services: - simple-email: - environment: - SIMPLE_EMAIL_DRY_RUN: "false" - send-email-link: environment: SEND_EMAIL_LINK_DRY_RUN: "false" @@ -303,7 +342,7 @@ Start the stack with both files: docker compose -f docker-compose.jobs.yml -f docker-compose.jobs.override.yml up -d --build --force-recreate ``` -To switch back to dry-run, set `SIMPLE_EMAIL_DRY_RUN=true` and `SEND_EMAIL_LINK_DRY_RUN=true` (or delete the override file) and recreate again. +To switch back to dry-run, set `SEND_EMAIL_LINK_DRY_RUN=true` (or delete the override file) and recreate again. ## NOTES: diff --git a/jobs/README.md b/jobs/README.md index 02d84e5fd..412d9cf8e 100644 --- a/jobs/README.md +++ b/jobs/README.md @@ -86,104 +86,176 @@ From `jobs/knative-job-service/src/env.ts`: --- -## 3. Example function: `simple-email` (dry-run) +## 3. Example function: `send-email-link` -The `functions/simple-email` package is a **Knative function** that: +The `functions/send-email-link` package is a **Knative function** that sends email links for: -- Uses `@constructive-io/knative-job-fn` as the HTTP wrapper -- Expects JSON payload: +- **invite_email** - User invitations +- **forgot_password** - Password reset emails +- **email_verification** - Email verification links -```json -{ - "to": "user@example.com", - "subject": "Hello from jobs", - "html": "

Hi from simple-email

" -} -``` +### How it works + +1. Receives job payload with email type and parameters +2. Queries GraphQL API (via `private.localhost` host routing) for: + - `GetDatabaseInfo` - Site configuration (domains, logo, theme, legal terms) + - `GetUser` - Sender info for invite emails +3. Generates HTML email using MJML templates +4. Sends via Mailgun (or logs in dry-run mode) -- Validates `to`, `subject`, and at least one of `html` or `text` -- Logs the email and payload, but does **not** send anything: +### Required env vars (send-email-link) -```ts -console.log('[simple-email] DRY RUN email', { ... }); -console.log('[simple-email] DRY RUN payload', payload); -res.status(200).json({ complete: true }); +```yaml +# GraphQL endpoints (admin server with host-based routing) +GRAPHQL_URL: "http://constructive-admin-server:3000/graphql" +META_GRAPHQL_URL: "http://constructive-admin-server:3000/graphql" +GRAPHQL_HOST_HEADER: "private.localhost" +META_GRAPHQL_HOST_HEADER: "private.localhost" + +# Mailgun configuration +MAILGUN_API_KEY: "your-api-key" +MAILGUN_DOMAIN: "mg.example.com" +MAILGUN_FROM: "no-reply@mg.example.com" +MAILGUN_REPLY: "support@example.com" + +# Dry run mode (no actual emails sent) +SEND_EMAIL_LINK_DRY_RUN: "true" ``` -It also starts an HTTP server when run directly (for Knative): +--- -```ts -if (require.main === module) { - const port = Number(process.env.PORT ?? 8080); - (app as any).listen(port, () => { - console.log(`[simple-email] listening on port ${port}`); - }); -} -``` +## 4. Local Development with Docker Compose -### Knative Service (simple-email) +### Start the jobs stack -Example Knative `Service` manifest: +```bash +# Start postgres and minio first +docker-compose up -d -```yaml -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: simple-email - namespace: interweb -spec: - template: - spec: - containers: - - name: simple-email - image: ghcr.io/constructive-io/constructive: - command: ["node"] - args: ["functions/simple-email/dist/index.js"] - ports: - - containerPort: 8080 - protocol: TCP - env: - - name: NODE_ENV - value: "production" +# Start the jobs services +docker-compose -f docker-compose.jobs.yml up --build +``` + +### Services started + +| Service | Port | Description | +|---------|------|-------------| +| `constructive-admin-server` | 3001 | GraphQL API with `API_IS_PUBLIC=false` | +| `send-email-link` | 8082 | Email link function | +| `knative-job-service` | 8080 | Job worker + callback server | + +### Test GraphQL access + +```bash +# Introspect the private API +curl -X POST http://localhost:3001/graphql \ + -H "Content-Type: application/json" \ + -H "Host: private.localhost" \ + -d '{"query": "{ __schema { queryType { fields { name } } } }"}' + +# List databases +curl -X POST http://localhost:3001/graphql \ + -H "Content-Type: application/json" \ + -H "Host: private.localhost" \ + -d '{"query": "{ databases { nodes { id name } } }"}' + +# List users +curl -X POST http://localhost:3001/graphql \ + -H "Content-Type: application/json" \ + -H "Host: private.localhost" \ + -d '{"query": "{ users { nodes { id username displayName } } }"}' ``` -With this in place, the in-cluster URL is: +--- + +## 5. Enqueue a job (send-email-link) + +### Get required IDs + +```bash +# Get Database ID +DBID="$(docker exec -i postgres psql -U postgres -d constructive -Atc \ + 'SELECT id FROM metaschema_public.database ORDER BY created_at LIMIT 1;')" +echo "Database ID: $DBID" -```text -http://simple-email.interweb.svc.cluster.local +# Get User ID (for sender_id in invite emails) +SENDER_ID="$(docker exec -i postgres psql -U postgres -d constructive -Atc \ + 'SELECT id FROM roles_public.users ORDER BY created_at LIMIT 1;')" +echo "Sender ID: $SENDER_ID" ``` -and the worker will call: +### Enqueue invite_email job + +```bash +docker exec -it postgres \ + psql -U postgres -d constructive -c " + SELECT app_jobs.add_job( + '$DBID'::uuid, + 'send-email-link', + json_build_object( + 'email_type', 'invite_email', + 'email', 'user@example.com', + 'invite_token', 'invite-token-123', + 'sender_id', '$SENDER_ID' + )::json + ); + " +``` -```text -POST http://simple-email.interweb.svc.cluster.local/ +### Enqueue forgot_password job + +```bash +docker exec -it postgres \ + psql -U postgres -d constructive -c " + SELECT app_jobs.add_job( + '$DBID'::uuid, + 'send-email-link', + json_build_object( + 'email_type', 'forgot_password', + 'email', 'user@example.com', + 'user_id', '$SENDER_ID', + 'reset_token', 'reset-token-123' + )::json + ); + " ``` ---- +### Enqueue email_verification job + +```bash +docker exec -it postgres \ + psql -U postgres -d constructive -c " + SELECT app_jobs.add_job( + '$DBID'::uuid, + 'send-email-link', + json_build_object( + 'email_type', 'email_verification', + 'email', 'user@example.com', + 'email_id', '$(uuidgen)', + 'verification_token', 'verify-token-123' + )::json + ); + " +``` -## 4. Enqueue a job (simple-email) +### Watch the logs -To enqueue a job directly via SQL: +```bash +# Watch send-email-link function logs +docker logs -f send-email-link -```sql -SELECT app_jobs.add_job( - '00000000-0000-0000-0000-000000000001'::uuid, -- database_id (any UUID; used for multi-tenant routing) - 'simple-email', -- task_identifier (must match function name) - json_build_object( - 'to', 'user@example.com', - 'subject', 'Hello from Constructive jobs', - 'html', '

Hi from simple-email (dry run)

' - )::json -- payload -); +# Watch job service logs +docker logs -f knative-job-service ``` -Flow: +### Job flow -1. `app_jobs.add_job` inserts into `app_jobs.jobs` and fires `NOTIFY "jobs:insert"`. -2. `@constructive-io/knative-job-worker` receives the notification, calls `getJob`, and picks up the row. -3. The worker `POST`s the payload to `KNATIVE_SERVICE_URL + '/simple-email'`. -4. `simple-email` logs the email and payload, then returns `{ complete: true }`. -5. The worker logs success. (In the current Knative flow we rely on immediate responses; callback-based completion can be added later if needed.) +1. `app_jobs.add_job` inserts into `app_jobs.jobs` and fires `NOTIFY "jobs:insert"` +2. `knative-job-worker` receives notification, picks up the job +3. Worker `POST`s payload to `http://send-email-link:8080/` +4. `send-email-link` queries GraphQL for site/user info +5. Generates email HTML and sends (or logs in dry-run mode) +6. Returns `{ complete: true }` and job is marked complete You can inspect the queue directly: diff --git a/jobs/knative-job-fn/src/index.ts b/jobs/knative-job-fn/src/index.ts index 641a149c3..69cc9be7c 100644 --- a/jobs/knative-job-fn/src/index.ts +++ b/jobs/knative-job-fn/src/index.ts @@ -192,13 +192,22 @@ const createJobApp = () => { // If an error handler already sent a callback, skip. if (res.locals.jobCallbackSent) return; res.locals.jobCallbackSent = true; + + // Treat 4xx/5xx status codes as errors + const isError = res.statusCode >= 400; + logger.info('Function completed', { workerId: ctx.workerId, jobId: ctx.jobId, databaseId: ctx.databaseId, statusCode: res.statusCode }); - void sendJobCallback(ctx, 'success'); + + if (isError) { + void sendJobCallback(ctx, 'error', `HTTP ${res.statusCode}`); + } else { + void sendJobCallback(ctx, 'success'); + } }); } diff --git a/jobs/knative-job-service/__tests__/jobs.e2e.test.ts b/jobs/knative-job-service/__tests__/jobs.e2e.test.ts index 7d8b421b1..987372009 100644 --- a/jobs/knative-job-service/__tests__/jobs.e2e.test.ts +++ b/jobs/knative-job-service/__tests__/jobs.e2e.test.ts @@ -553,6 +553,91 @@ describe('jobs e2e', () => { await waitForJobCompletion(graphqlClient, jobId); }); + it('creates and processes a send-email-link forgot_password job', async () => { + const jobInput = { + dbId: databaseId, + identifier: 'send-email-link', + payload: { + email_type: 'forgot_password', + email: 'user@example.com', + user_id: '00000000-0000-0000-0000-000000000000', + reset_token: 'reset-token-123' + } + }; + + const response = await sendGraphql(graphqlClient, addJobMutation, { + input: jobInput + }); + + expect(response.status).toBe(200); + expect(response.body?.errors).toBeUndefined(); + + const jobId = response.body?.data?.addJob?.job?.id; + + expect(jobId).toBeTruthy(); + + await waitForJobCompletion(graphqlClient, jobId); + }); + + it('creates and processes a send-email-link email_verification job', async () => { + const jobInput = { + dbId: databaseId, + identifier: 'send-email-link', + payload: { + email_type: 'email_verification', + email: 'user@example.com', + email_id: '55555555-5555-5555-5555-555555555555', + verification_token: 'verify-token-123' + } + }; + + const response = await sendGraphql(graphqlClient, addJobMutation, { + input: jobInput + }); + + expect(response.status).toBe(200); + expect(response.body?.errors).toBeUndefined(); + + const jobId = response.body?.data?.addJob?.job?.id; + + expect(jobId).toBeTruthy(); + + await waitForJobCompletion(graphqlClient, jobId); + }); + + it('fails send-email-link job when required fields are missing', async () => { + const jobInput = { + dbId: databaseId, + identifier: 'send-email-link', + maxAttempts: 1, + payload: { + email_type: 'forgot_password', + email: 'user@example.com' + // Missing: user_id, reset_token + } + }; + + const response = await sendGraphql(graphqlClient, addJobMutation, { + input: jobInput + }); + + expect(response.status).toBe(200); + expect(response.body?.errors).toBeUndefined(); + + const jobId = response.body?.data?.addJob?.job?.id; + + expect(jobId).toBeTruthy(); + + const job = await waitForJobFailure(graphqlClient, jobId, { + minAttempts: 1, + timeoutMs: 30000 + }); + + expect(job.attempts).toBe(1); + expect(job.maxAttempts).toBe(1); + expect(job.lastError).toBeTruthy(); + }); + it('records failed jobs when a function throws', async () => { const jobInput = { dbId: databaseId, diff --git a/jobs/knative-job-service/src/index.ts b/jobs/knative-job-service/src/index.ts index dec41c940..0c9c34203 100644 --- a/jobs/knative-job-service/src/index.ts +++ b/jobs/knative-job-service/src/index.ts @@ -433,6 +433,23 @@ export const bootJobs = async (): Promise => { ); const options = buildKnativeJobsSvcOptionsFromEnv(); + + // Log startup configuration (non-sensitive values only) + const pgConfig = getJobPgConfig(); + log.info('[knative-job-service] Starting with config:', { + database: pgConfig.database, + host: pgConfig.host, + port: pgConfig.port, + schema: getJobSchema(), + callbackPort: getJobsCallbackPort(), + workerHostname: getWorkerHostname(), + schedulerHostname: getSchedulerHostname(), + supportedTasks: getJobSupported(), + jobsEnabled: options.jobs?.enabled ?? true, + functionsEnabled: shouldEnableFunctions(options.functions), + functions: normalizeFunctionServices(options.functions).map(s => s.name) + }); + if (options.jobs?.enabled === false) { log.info('jobs disabled; skipping startup'); return; diff --git a/packages/cli/src/commands/server.ts b/packages/cli/src/commands/server.ts index 92472ad75..72983f1ec 100644 --- a/packages/cli/src/commands/server.ts +++ b/packages/cli/src/commands/server.ts @@ -211,6 +211,16 @@ export default async ( log.debug(`${key}: ${JSON.stringify(value)}`); } + // Debug: Log API routing configuration + const apiOpts = (options as any).api || {}; + log.debug(`📡 API Routing: isPublic=${apiOpts.isPublic}, enableServicesApi=${apiOpts.enableServicesApi}`); + if (apiOpts.isPublic === false) { + log.debug(` Header-based routing enabled (X-Api-Name, X-Database-Id, X-Meta-Schema)`); + } + if (apiOpts.metaSchemas?.length) { + log.debug(` Meta schemas: ${apiOpts.metaSchemas.join(', ')}`); + } + log.success('🚀 Launching Server...\n'); server(options); };