diff --git a/foundations/stream/Dockerfile b/foundations/stream/Dockerfile index 297954361b0..20f502f8ad6 100644 --- a/foundations/stream/Dockerfile +++ b/foundations/stream/Dockerfile @@ -21,8 +21,6 @@ ENV GOBIN=/bin ENV PATH=$PATH:$GOBIN ARG BUILDARCH=amd64 -COPY . ./ - FROM base AS linter RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 @@ -31,6 +29,8 @@ RUN golangci-lint run --verbose FROM base AS builder +COPY . ./ + RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/stream ./cmd/stream FROM alpine diff --git a/server/account-service/src/index.ts b/server/account-service/src/index.ts index 767496f16ba..577cb1d91f1 100644 --- a/server/account-service/src/index.ts +++ b/server/account-service/src/index.ts @@ -18,7 +18,7 @@ import accountRu from '@hcengineering/account/lang/ru.json' import { Analytics } from '@hcengineering/analytics' import { registerProviders } from '@hcengineering/auth-providers' import { metricsAggregate, type Branding, type BrandingMap, type MeasureContext } from '@hcengineering/core' -import platform, { Severity, Status, addStringsLoader, setMetadata } from '@hcengineering/platform' +import platform, { Severity, Status, addStringsLoader, setMetadata, unknownStatus } from '@hcengineering/platform' import serverToken, { decodeToken, decodeTokenVerbose, generateToken } from '@hcengineering/server-token' import cors from '@koa/cors' import type Cookies from 'cookies' @@ -424,15 +424,25 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap error: new Status(Severity.ERROR, platform.status.UnknownMethod, { method: request.method }) } - ctx.body = JSON.stringify(response) + ctx.res.writeHead(400, KEEP_ALIVE_HEADERS) + ctx.res.end(JSON.stringify(response)) return } - const result = await method(_ctx, db, branding, request, token, meta) + try { + const result = await method(_ctx, db, branding, request, token, meta) - const body = JSON.stringify(result) - ctx.res.writeHead(200, KEEP_ALIVE_HEADERS) - ctx.res.end(body) + const body = JSON.stringify(result) + ctx.res.writeHead(200, KEEP_ALIVE_HEADERS) + ctx.res.end(body) + } catch (err: any) { + const response = { + id: request.id, + error: unknownStatus(err.message) + } + ctx.res.writeHead(400, KEEP_ALIVE_HEADERS) + ctx.res.end(JSON.stringify(response)) + } }, { method: request.method } ) diff --git a/server/account/src/__tests__/postgres-real.test.ts b/server/account/src/__tests__/postgres-real.test.ts new file mode 100644 index 00000000000..2ecf61ba13b --- /dev/null +++ b/server/account/src/__tests__/postgres-real.test.ts @@ -0,0 +1,278 @@ +/** + * A set of tests against a real PostgreSQL database, for both CorockachDB and pure. + */ + +import { generateUuid, SocialIdType, type AccountUuid, type PersonId } from '@hcengineering/core' +import { getDBClient, shutdownPostgres, type PostgresClientReference } from '@hcengineering/postgres' +import { PostgresAccountDB } from '../collections/postgres/postgres' +import { type SocialId } from '../types' +import { createAccount, getDbFlavor, normalizeValue } from '../utils' + +jest.setTimeout(90000) + +describe('real-account', () => { + // It should create a DB and test on it for every execution, and drop it after it. + // + // Use environment variable or default to localhost CockroachDB + const cockroachDB: string = process.env.DB_URL ?? 'postgresql://root@localhost:26258/defaultdb?sslmode=disable' + + const postgresDB: string = process.env.POSTGRES_URL ?? 'postgresql://postgres:postgres@localhost:5433/postgres' + + let crDbUri = cockroachDB + let pgDbUri = postgresDB + + // Administrative client for creating/dropping test databases + // This connects to 'defaultdb' and is used ONLY for DB admin operations + let adminClientCRRef: PostgresClientReference + let adminClientPGRef: PostgresClientReference + + let dbUuid: string + + let crClient: PostgresClientReference + let pgClient: PostgresClientReference + + let crAccount: PostgresAccountDB + let pgAccount: PostgresAccountDB + + const users = [ + { + name: 'user1', + uuid: generateUuid() as AccountUuid, + email: 'user1@example.com', + firstName: 'Jon', + lastName: 'Doe' + }, + { + name: 'user2', + uuid: generateUuid() as AccountUuid, + email: 'user2@example.com', + firstName: 'Pavel', + lastName: 'Siaro' + } + ] + + async function addSocialId ( + account: PostgresAccountDB, + user: (typeof users)[0], + type: SocialIdType, + value: string + ): Promise { + const normalizedValue = normalizeValue(value) + const newSocialId = { + type, + value: normalizedValue, + personUuid: user.uuid + } + return await account.socialId.insertOne(newSocialId) + } + + async function prepareAccounts (account: PostgresAccountDB): Promise { + for (const user of users) { + const ex = await account.account.findOne({ uuid: user.uuid }) + if (ex == null) { + await account.person.insertOne({ uuid: user.uuid, firstName: user.firstName, lastName: user.lastName }) + await createAccount(account, user.uuid, true) + await addSocialId(account, user, SocialIdType.EMAIL, user.email) + } + } + } + + beforeAll(() => { + // Get admin client for database creation/deletion + // This client stays connected to 'defaultdb' for admin operations only + adminClientCRRef = getDBClient(cockroachDB) + adminClientPGRef = getDBClient(postgresDB) + }) + + afterAll(async () => { + adminClientCRRef.close() + adminClientPGRef.close() + await shutdownPostgres() + }) + + beforeEach(async () => { + // Create a unique database for each test to ensure isolation + dbUuid = 'accountdb' + Date.now().toString() + crDbUri = cockroachDB.replace('/defaultdb', '/' + dbUuid) + const c = postgresDB.split('/') + c[c.length - 1] = dbUuid + pgDbUri = c.join('/') + + try { + // Use admin client to create the test database + await Promise.all([initCockroachDB(adminClientCRRef, dbUuid), initPostgreSQL(adminClientPGRef, dbUuid)]) + } catch (err) { + console.error('Failed to create test database:', err) + throw err + } + + crClient = getDBClient(crDbUri) + const crPGClient = await crClient.getClient() + + pgClient = getDBClient(pgDbUri) + const pgPGClient = await pgClient.getClient() + + // Initial DB's + + crAccount = new PostgresAccountDB(crPGClient, dbUuid) + + pgAccount = new PostgresAccountDB(pgPGClient, dbUuid, await getDbFlavor(pgPGClient)) + + await Promise.all([migrateCockroachDB(crAccount, crDbUri), migratePostgreSQL(pgAccount, pgDbUri)]) + + await Promise.all([prepareAccounts(pgAccount), prepareAccounts(crAccount)]) + }) + + afterEach(async () => { + try { + pgClient.close() + crClient.close() + + // Use admin client to drop the test database + const adminClient = await adminClientCRRef.getClient() + await adminClient`DROP DATABASE IF EXISTS ${adminClient(dbUuid)} CASCADE` + + const adminClientPG = await adminClientPGRef.getClient() + await adminClientPG`DROP DATABASE IF EXISTS ${adminClient(dbUuid)}` + } catch (err) { + console.error('Cleanup error:', err) + } + }) + + it('Check accounts', async () => { + const user1 = await crAccount.account.findOne({ uuid: users[0].uuid }) + expect(user1).not.toBeNull() + expect(user1).toBeDefined() + + const user1PG = await pgAccount.account.findOne({ uuid: users[0].uuid }) + expect(user1).not.toBeNull() + expect(user1PG).toBeDefined() + }) + + it('Check social ids', async () => { + const user1 = await crAccount.account.findOne({ uuid: users[0].uuid }) + expect(user1).not.toBeNull() + expect(user1).toBeDefined() + + const socialIds = await crAccount.socialId.find({ personUuid: user1?.uuid }) + expect(socialIds).not.toBeNull() + expect(socialIds).toBeDefined() + expect(socialIds.length).toEqual(2) + + const user1PG = await pgAccount.account.findOne({ uuid: users[0].uuid }) + expect(user1).not.toBeNull() + expect(user1PG).toBeDefined() + + const socialIdsPG = await pgAccount.socialId.find({ personUuid: user1PG?.uuid }) + expect(socialIdsPG).not.toBeNull() + expect(socialIdsPG).toBeDefined() + expect(socialIdsPG.length).toEqual(2) + + const em = socialIdsPG.find((it) => it.type === SocialIdType.EMAIL) as SocialId + expect(em).toBeDefined() + expect(em.key).toEqual('email:user1@example.com') + }) + it('List accounts', async () => { + const users = await crAccount.listAccounts() + expect(users.length).toBe(2) + + const usersPG = await pgAccount.listAccounts() + expect(usersPG.length).toBe(2) + }) + + it('check invites', async () => { + const wsUuid = await crAccount.createWorkspace( + { + url: 'test-ws', + name: 'test-ws', + allowGuestSignUp: true, + allowReadOnlyGuest: true + }, + { + isDisabled: false, + mode: 'active', + versionMajor: 0, + versionMinor: 7, + versionPatch: 0 + } + ) + const inviteLink = await crAccount.invite.insertOne({ + workspaceUuid: wsUuid, + expiresOn: new Date(Date.now() + 1000 * 60 * 60 * 24 * 30).getTime() + }) + expect(inviteLink).toBeDefined() + + const wsUuidPG = await pgAccount.createWorkspace( + { + url: 'test-ws', + name: 'test-ws', + allowGuestSignUp: true, + allowReadOnlyGuest: true + }, + { + isDisabled: false, + mode: 'active', + versionMajor: 0, + versionMinor: 7, + versionPatch: 0 + } + ) + const inviteLinkPG = await pgAccount.invite.insertOne({ + workspaceUuid: wsUuidPG, + expiresOn: new Date(Date.now() + 1000 * 60 * 60 * 24 * 30).getTime() + }) + expect(inviteLinkPG).toBeDefined() + }) +}) + +async function migratePostgreSQL (pgAccount: PostgresAccountDB, pgDbUri: string): Promise { + let error = false + do { + try { + await pgAccount.init() + error = false + } catch (e) { + console.error('Error while initializing postgres account db', e, pgDbUri) + error = true + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + } while (error) +} + +async function migrateCockroachDB (crAccount: PostgresAccountDB, crDbUri: string): Promise { + let error: boolean = false + do { + try { + await crAccount.init() + error = false + } catch (e) { + console.error('Error while initializing postgres account db', e, crDbUri) + error = true + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + } while (error) +} + +async function initPostgreSQL (adminClientPGRef: PostgresClientReference, dbUuid: string): Promise { + const adminClientPg = await adminClientPGRef.getClient() + // Clean up any leftover test databases with prefix 'accountdb' for Postgres + const existingPgs = await adminClientPg`SELECT datname FROM pg_database WHERE datname LIKE 'accountdb%'` + for (const row of existingPgs) { + try { + await adminClientPg`DROP DATABASE IF EXISTS ${adminClientPg(row.datname)}` + } catch (err: any) { + // Ignore, Postgress says database is being used by other users + } + } + await adminClientPg`CREATE DATABASE ${adminClientPg(dbUuid)}` +} + +async function initCockroachDB (adminClientCRRef: PostgresClientReference, dbUuid: string): Promise { + const adminClient = await adminClientCRRef.getClient() + // Clean up any leftover test databases with prefix 'accountdb' + const existingCrs = await adminClient`SELECT datname FROM pg_database WHERE datname LIKE 'accountdb%'` + for (const row of existingCrs) { + await adminClient`DROP DATABASE IF EXISTS ${adminClient(row.datname)} CASCADE` + } + await adminClient`CREATE DATABASE ${adminClient(dbUuid)}` +} diff --git a/server/account/src/__tests__/postgres.test.ts b/server/account/src/__tests__/postgres.test.ts index 5a99a88b9f7..5b9e434c119 100644 --- a/server/account/src/__tests__/postgres.test.ts +++ b/server/account/src/__tests__/postgres.test.ts @@ -330,7 +330,7 @@ describe('AccountPostgresDbCollection', () => { expect(mockClient.unsafe).toHaveBeenCalledWith( `SELECT * FROM ( - SELECT + SELECT a.uuid, a.timezone, a.locale, @@ -678,7 +678,7 @@ describe('PostgresAccountDB', () => { expect( mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')') ).toEqual( - `SELECT + `SELECT w.uuid, w.name, w.url, @@ -688,7 +688,7 @@ describe('PostgresAccountDB', () => { w.region, w.created_by, w.created_on, - w.billing_account, + w.billing_account, json_build_object( 'mode', s.mode, 'processing_progress', s.processing_progress, @@ -747,7 +747,7 @@ describe('PostgresAccountDB', () => { expect( mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')') ).toEqual( - `SELECT + `SELECT w.uuid, w.name, w.url, @@ -757,7 +757,7 @@ describe('PostgresAccountDB', () => { w.region, w.created_by, w.created_on, - w.billing_account, + w.billing_account, json_build_object( 'mode', s.mode, 'processing_progress', s.processing_progress, @@ -820,7 +820,7 @@ describe('PostgresAccountDB', () => { expect( mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')') ).toEqual( - `SELECT + `SELECT w.uuid, w.name, w.url, @@ -830,7 +830,7 @@ describe('PostgresAccountDB', () => { w.region, w.created_by, w.created_on, - w.billing_account, + w.billing_account, json_build_object( 'mode', s.mode, 'processing_progress', s.processing_progress, @@ -912,7 +912,7 @@ describe('PostgresAccountDB', () => { expect( mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')') ).toEqual( - `SELECT + `SELECT w.uuid, w.name, w.url, @@ -922,7 +922,7 @@ describe('PostgresAccountDB', () => { w.region, w.created_by, w.created_on, - w.billing_account, + w.billing_account, json_build_object( 'mode', s.mode, 'processing_progress', s.processing_progress, @@ -963,8 +963,8 @@ describe('PostgresAccountDB', () => { expect( mockClient.unsafe.mock.calls[1][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')') ).toEqual( - `UPDATE global_account.workspace_status - SET processing_attempts = processing_attempts + 1, "last_processing_time" = $1 + `UPDATE global_account.workspace_status + SET processing_attempts = processing_attempts + 1, "last_processing_time" = $1 WHERE workspace_uuid = $2` .replace(/\s+/g, ' ') .replace(/\(\s/g, '(') @@ -997,7 +997,13 @@ describe('PostgresAccountDB', () => { expect(mockClient).toHaveBeenCalledWith('global_account.account_passwords') expect(mockClient).toHaveBeenCalledWith( - ['UPSERT INTO ', ' (account_uuid, hash, salt) VALUES (', ', ', '::bytea, ', '::bytea)'], + [ + 'INSERT INTO ', + ' (account_uuid, hash, salt) VALUES (', + ', ', + '::bytea, ', + '::bytea) ON CONFLICT (account_uuid) DO UPDATE SET hash = EXCLUDED.hash, salt = EXCLUDED.salt;' + ], expect.anything(), accountId, hash.buffer, diff --git a/server/account/src/collections/postgres/migrations.ts b/server/account/src/collections/postgres/migrations.ts index 829318c6c51..c9bce8442c1 100644 --- a/server/account/src/collections/postgres/migrations.ts +++ b/server/account/src/collections/postgres/migrations.ts @@ -13,86 +13,158 @@ // limitations under the License. // -export function getMigrations (ns: string): [string, string][] { +import { type DBFlavor } from '../../types' + +// Type definitions for different database flavors. +// The keys match the DBFlavor type ('postgres' and 'cockroach'). +// The SQL syntax used (e.g., BIGSERIAL, JSONB, generated columns) is compatible with +// modern PostgreSQL versions (v13+) and is expected to be forward-compatible with future versions like 18.1. +const dbTypes = { + ['cockroach' as DBFlavor]: { + string: 'STRING', + bytes: 'BYTES', + int2: 'INT2', + int4: 'INT4', + int8: 'INT8', + bool: 'BOOL', + // unique_rowid() generates a unique INT8 + autoIncrementInt8: (ns: string) => 'INT8 NOT NULL DEFAULT unique_rowid()' + }, + ['postgres' as DBFlavor]: { + string: 'TEXT', + bytes: 'BYTEA', + int2: 'SMALLINT', + int4: 'INTEGER', + int8: 'BIGINT', + bool: 'BOOLEAN', + // Use special function to generate a cryptographic random bigint + autoIncrementInt8: (ns: string) => `BIGINT NOT NULL DEFAULT ${ns}.gen_random_bigint()` + } +} + +export function getMigrations (ns: string, flavor: DBFlavor): [string, string][] { + if (flavor === 'unknown') { + throw new Error('Cannot generate migrations for an unknown database flavor.') + } + + const types = dbTypes[flavor] + if (types === undefined) { + // This should not happen if DBFlavor is typed correctly, but it's a good safeguard. + throw new Error(`Unsupported database flavor: ${flavor}`) + } + return [ - getV1Migration(ns), - getV2Migration1(ns), - getV2Migration2(ns), - getV2Migration3(ns), - getV3Migration(ns), - getV4Migration(ns), - getV4Migration1(ns), - getV5Migration(ns), - getV6Migration(ns), - getV7Migration(ns), - getV8Migration(ns), - getV9Migration(ns), - getV10Migration1(ns), - getV10Migration2(ns), - getV11Migration(ns), - getV12Migration(ns), - getV13Migration(ns), - getV14Migration(ns), - getV15Migration(ns), - getV16Migration(ns), - getV17Migration(ns), - getV18Migration(ns), - getV19Migration(ns), - getV20Migration(ns), - getV21Migration(ns), - getV22Migration(ns), - getV23Migration(ns), - getV24Migration(ns) + getV1Migration(ns, flavor), + getV2Migration1(ns, flavor), + getV2Migration2(ns, flavor), + getV2Migration3(ns, flavor), + getV3Migration(ns, flavor), + getV4Migration(ns, flavor), + getV4Migration1(ns, flavor), + getV5Migration(ns, flavor), + getV6Migration(ns, flavor), + getV7Migration(ns, flavor), + getV8Migration(ns, flavor), + getV9Migration(ns, flavor), + getV10Migration1(ns, flavor), + getV10Migration2(ns, flavor), + getV11Migration(ns, flavor), + getV12Migration(ns, flavor), + getV13Migration(ns, flavor), + getV14Migration(ns, flavor), + getV15Migration(ns, flavor), + getV16Migration(ns, flavor), + getV17Migration(ns, flavor), + getV18Migration(ns, flavor), + getV19Migration(ns, flavor), + getV20Migration(ns, flavor), + getV21Migration(ns, flavor), + getV22Migration(ns, flavor), + getV23Migration(ns, flavor), + getV24Migration(ns, flavor) ] } // NOTE: NEVER MODIFY EXISTING MIGRATIONS. IF YOU NEED TO ADJUST THE SCHEMA, ADD A NEW MIGRATION. -function getV1Migration (ns: string): [string, string] { +function getV1Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + + // Define the generated 'key' column with flavor-specific syntax. + // For PostgreSQL, we use a custom immutable function to ensure the expression is valid. + const keyColumnDefinition = + flavor === 'postgres' + ? `key ${types.string} GENERATED ALWAYS AS (${ns}.social_id_type_to_text(type) || ':' || value) STORED` + : `key ${types.string} AS (CONCAT(type::TEXT, ':', value)) STORED` + return [ 'account_db_v1_global_init', ` /* ======= FUNCTIONS ======= */ - CREATE OR REPLACE FUNCTION current_epoch_ms() - RETURNS BIGINT AS $$ - SELECT (extract(epoch from current_timestamp) * 1000)::bigint; + CREATE OR REPLACE FUNCTION current_epoch_ms() + RETURNS BIGINT AS $$ SELECT (extract(epoch from current_timestamp) * 1000)::bigint; $$ LANGUAGE SQL; + /* ======= E X T E N S I O N S ======= */ + -- Enable the pgcrypto extension for cryptographic functions, e.g., gen_random_bytes(). + -- This is required for secure, non-sequential ID generation in PostgreSQL. + -- This is a no-op for CockroachDB as it doesn't support extensions this way. + ${flavor === 'postgres' ? 'CREATE EXTENSION IF NOT EXISTS pgcrypto;' : '-- pgcrypto not needed for CockroachDB'} + /* ======= T Y P E S ======= */ - CREATE TYPE IF NOT EXISTS ${ns}.social_id_type AS ENUM ('email', 'github', 'google', 'phone', 'oidc', 'huly', 'telegram'); - CREATE TYPE IF NOT EXISTS ${ns}.location AS ENUM ('kv', 'weur', 'eeur', 'wnam', 'enam', 'apac'); - CREATE TYPE IF NOT EXISTS ${ns}.workspace_role AS ENUM ('OWNER', 'MAINTAINER', 'USER', 'GUEST', 'DOCGUEST'); + CREATE TYPE ${ns}.social_id_type AS ENUM ('email', 'github', 'google', 'phone', 'oidc', 'huly', 'telegram'); + CREATE TYPE ${ns}.location AS ENUM ('kv', 'weur', 'eeur', 'wnam', 'enam', 'apac'); + CREATE TYPE ${ns}.workspace_role AS ENUM ('OWNER', 'MAINTAINER', 'USER', 'GUEST', 'DOCGUEST'); + + /* ======= HELPER FUNCTIONS FOR GENERATED COLUMNS ======= */ + -- Create an immutable function to cast the social_id_type enum to text. + -- This is required for PostgreSQL to consider the generated expression immutable. + CREATE OR REPLACE FUNCTION ${ns}.social_id_type_to_text(val ${ns}.social_id_type) + RETURNS TEXT AS $$ SELECT val::TEXT; + $$ LANGUAGE SQL IMMUTABLE; + + ${ + flavor === 'postgres' + ? ` + -- Create a function to generate a random, non-sequential, non-negative BIGINT for secure IDs. + -- This prevents enumeration attacks. We use a standard method of encoding to hex and casting. + CREATE OR REPLACE FUNCTION ${ns}.gen_random_bigint() + RETURNS BIGINT AS $$ SELECT ('x' || encode(gen_random_bytes(8), 'hex'))::bit(64)::bigint & 9223372036854775807::bigint; + $$ LANGUAGE SQL VOLATILE; + ` + : '' + } /* ======= P E R S O N ======= */ CREATE TABLE IF NOT EXISTS ${ns}.person ( uuid UUID NOT NULL DEFAULT gen_random_uuid(), - first_name STRING NOT NULL, - last_name STRING NOT NULL, - country STRING, - city STRING, + first_name ${types.string} NOT NULL, + last_name ${types.string} NOT NULL, + country ${types.string}, + city ${types.string}, CONSTRAINT person_pk PRIMARY KEY (uuid) ); /* ======= A C C O U N T ======= */ CREATE TABLE IF NOT EXISTS ${ns}.account ( uuid UUID NOT NULL, - timezone STRING, - locale STRING, + timezone ${types.string}, + locale ${types.string}, CONSTRAINT account_pk PRIMARY KEY (uuid), CONSTRAINT account_person_fk FOREIGN KEY (uuid) REFERENCES ${ns}.person(uuid) ); CREATE TABLE IF NOT EXISTS ${ns}.account_passwords ( account_uuid UUID NOT NULL, - hash BYTES NOT NULL, - salt BYTES NOT NULL, + hash ${types.bytes} NOT NULL, + salt ${types.bytes} NOT NULL, CONSTRAINT account_auth_pk PRIMARY KEY (account_uuid), CONSTRAINT account_passwords_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid) ); CREATE TABLE IF NOT EXISTS ${ns}.account_events ( account_uuid UUID NOT NULL, - event_type STRING NOT NULL, + event_type ${types.string} NOT NULL, time BIGINT NOT NULL DEFAULT current_epoch_ms(), data JSONB, CONSTRAINT account_events_pk PRIMARY KEY (account_uuid, event_type, time), @@ -102,26 +174,25 @@ function getV1Migration (ns: string): [string, string] { /* ======= S O C I A L I D S ======= */ CREATE TABLE IF NOT EXISTS ${ns}.social_id ( type ${ns}.social_id_type NOT NULL, - value STRING NOT NULL, - key STRING AS (CONCAT(type::STRING, ':', value)) STORED, + value ${types.string} NOT NULL, + ${keyColumnDefinition}, person_uuid UUID NOT NULL, created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), verified_on BIGINT, CONSTRAINT social_id_pk PRIMARY KEY (type, value), CONSTRAINT social_id_key_unique UNIQUE (key), - INDEX social_id_account_idx (person_uuid), CONSTRAINT social_id_person_fk FOREIGN KEY (person_uuid) REFERENCES ${ns}.person(uuid) ); /* ======= W O R K S P A C E ======= */ CREATE TABLE IF NOT EXISTS ${ns}.workspace ( uuid UUID NOT NULL DEFAULT gen_random_uuid(), - name STRING NOT NULL, - url STRING NOT NULL, - data_id STRING, - branding STRING, + name ${types.string} NOT NULL, + url ${types.string} NOT NULL, + data_id ${types.string}, + branding ${types.string}, location ${ns}.location, - region STRING, + region ${types.string}, created_by UUID, -- account uuid created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), billing_account UUID, @@ -133,16 +204,16 @@ function getV1Migration (ns: string): [string, string] { CREATE TABLE IF NOT EXISTS ${ns}.workspace_status ( workspace_uuid UUID NOT NULL, - mode STRING, - processing_progress INT2 DEFAULT 0, - version_major INT2 NOT NULL DEFAULT 0, - version_minor INT2 NOT NULL DEFAULT 0, - version_patch INT4 NOT NULL DEFAULT 0, + mode ${types.string}, + processing_progress ${types.int2} DEFAULT 0, + version_major ${types.int2} NOT NULL DEFAULT 0, + version_minor ${types.int2} NOT NULL DEFAULT 0, + version_patch ${types.int4} NOT NULL DEFAULT 0, last_processing_time BIGINT DEFAULT 0, last_visit BIGINT, - is_disabled BOOL DEFAULT FALSE, - processing_attempts INT2 DEFAULT 0, - processing_message STRING, + is_disabled ${types.bool} DEFAULT FALSE, + processing_attempts ${types.int2} DEFAULT 0, + processing_message ${types.string}, backup_info JSONB, CONSTRAINT workspace_status_pk PRIMARY KEY (workspace_uuid), CONSTRAINT workspace_status_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid) @@ -163,8 +234,8 @@ function getV1Migration (ns: string): [string, string] { /* ======= O T P ======= */ CREATE TABLE IF NOT EXISTS ${ns}.otp ( - social_id STRING NOT NULL, - code STRING NOT NULL, + social_id ${types.string} NOT NULL, + code ${types.string} NOT NULL, expires_on BIGINT NOT NULL, created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), CONSTRAINT otp_pk PRIMARY KEY (social_id, code), @@ -173,67 +244,93 @@ function getV1Migration (ns: string): [string, string] { /* ======= I N V I T E ======= */ CREATE TABLE IF NOT EXISTS ${ns}.invite ( - id INT8 NOT NULL DEFAULT unique_rowid(), + id ${types.autoIncrementInt8(ns)} PRIMARY KEY, workspace_uuid UUID NOT NULL, expires_on BIGINT NOT NULL, - email_pattern STRING, - remaining_uses INT2, + email_pattern ${types.string}, + remaining_uses ${types.int2}, role ${ns}.workspace_role NOT NULL DEFAULT 'USER', - migrated_from STRING, - CONSTRAINT invite_pk PRIMARY KEY (id), - INDEX workspace_invite_idx (workspace_uuid), - INDEX migrated_from_idx (migrated_from), + migrated_from ${types.string}, CONSTRAINT invite_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid) ); + + /* ======= I N D E X E S ======= */ + CREATE INDEX IF NOT EXISTS social_id_account_idx ON ${ns}.social_id (person_uuid); + CREATE INDEX IF NOT EXISTS workspace_invite_idx ON ${ns}.invite (workspace_uuid); + CREATE INDEX IF NOT EXISTS migrated_from_idx ON ${ns}.invite (migrated_from); ` ] } -function getV2Migration1 (ns: string): [string, string] { +function getV2Migration1 (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] return [ 'account_db_v2_social_id_id_add', ` - -- Add _id column to social_id table + -- Add the _id column to the social_id table with a random default value. ALTER TABLE ${ns}.social_id - ADD COLUMN IF NOT EXISTS _id INT8 NOT NULL DEFAULT unique_rowid(); + ADD COLUMN IF NOT EXISTS _id ${types.autoIncrementInt8(ns)}; ` ] } -function getV2Migration2 (ns: string): [string, string] { +function getV2Migration2 (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v2_social_id_pk_change', ` - -- Drop existing otp foreign key constraint + -- Drop the existing otp foreign key constraint ALTER TABLE ${ns}.otp DROP CONSTRAINT IF EXISTS otp_social_id_fk; - -- Drop existing primary key on social_id + -- Drop the existing primary key on social_id ALTER TABLE ${ns}.social_id DROP CONSTRAINT IF EXISTS social_id_pk; - -- Add new primary key on _id + -- Add the new primary key on _id ALTER TABLE ${ns}.social_id ADD CONSTRAINT social_id_pk PRIMARY KEY (_id); ` ] } -function getV2Migration3 (ns: string): [string, string] { +function getV2Migration3 (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + + // Generate flavor-specific SQL for adding the constraint. + const addConstraintSql = + flavor === 'postgres' + ? ` + -- Add unique constraint on type, value (PostgreSQL compatible with DO block) + DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint + WHERE conname = 'social_id_tv_key_unique' + AND connamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}') + AND conrelid = (SELECT oid FROM pg_class WHERE relname = 'social_id' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}')) + ) THEN + ALTER TABLE ${ns}.social_id + ADD CONSTRAINT social_id_tv_key_unique UNIQUE (type, value); + END IF; + END $$; + ` + : ` + -- Add unique constraint on type, value (CockroachDB compatible) + ALTER TABLE ${ns}.social_id + ADD CONSTRAINT IF NOT EXISTS social_id_tv_key_unique UNIQUE (type, value); + ` + return [ 'account_db_v2_social_id_constraints', ` - -- Add unique constraint on type, value - ALTER TABLE ${ns}.social_id - ADD CONSTRAINT social_id_tv_key_unique UNIQUE (type, value); + ${addConstraintSql} - -- Drop old table - DROP TABLE ${ns}.otp; + -- Drop the old table + DROP TABLE IF EXISTS ${ns}.otp; - -- Create new OTP table with correct column type + -- Create the new OTP table with the correct column type CREATE TABLE ${ns}.otp ( - social_id INT8 NOT NULL, - code STRING NOT NULL, + social_id ${types.int8} NOT NULL, + code ${types.string} NOT NULL, expires_on BIGINT NOT NULL, created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), CONSTRAINT otp_new_pk PRIMARY KEY (social_id, code), @@ -243,42 +340,46 @@ function getV2Migration3 (ns: string): [string, string] { ] } -function getV3Migration (ns: string): [string, string] { +function getV3Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v3_add_invite_auto_join_final', ` ALTER TABLE ${ns}.invite - ADD COLUMN IF NOT EXISTS email STRING, - ADD COLUMN IF NOT EXISTS auto_join BOOL DEFAULT FALSE; + ADD COLUMN IF NOT EXISTS email ${types.string}, + ADD COLUMN IF NOT EXISTS auto_join ${types.bool} DEFAULT FALSE; ALTER TABLE ${ns}.account - ADD COLUMN IF NOT EXISTS automatic BOOL; + ADD COLUMN IF NOT EXISTS automatic ${types.bool}; ` ] } -function getV4Migration (ns: string): [string, string] { +function getV4Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v4_mailbox', ` CREATE TABLE IF NOT EXISTS ${ns}.mailbox ( account_uuid UUID NOT NULL, - mailbox STRING NOT NULL, + mailbox ${types.string} NOT NULL, CONSTRAINT mailbox_pk PRIMARY KEY (mailbox), CONSTRAINT mailbox_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid) ); CREATE TABLE IF NOT EXISTS ${ns}.mailbox_secrets ( - mailbox STRING NOT NULL, - app STRING, - secret STRING NOT NULL, + mailbox ${types.string} NOT NULL, + app ${types.string}, + secret ${types.string} NOT NULL, CONSTRAINT mailbox_secret_mailbox_fk FOREIGN KEY (mailbox) REFERENCES ${ns}.mailbox(mailbox) ); ` ] } -function getV4Migration1 (ns: string): [string, string] { +function getV4Migration1 (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v4_remove_mailbox_account_fk', ` @@ -288,46 +389,54 @@ function getV4Migration1 (ns: string): [string, string] { ] } -function getV5Migration (ns: string): [string, string] { +function getV5Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v5_social_id_is_deleted', ` ALTER TABLE ${ns}.social_id - ADD COLUMN IF NOT EXISTS is_deleted BOOL NOT NULL DEFAULT FALSE; + ADD COLUMN IF NOT EXISTS is_deleted ${types.bool} NOT NULL DEFAULT FALSE; ` ] } -function getV6Migration (ns: string): [string, string] { +function getV6Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + + // Generated column syntax: PostgreSQL uses GENERATED ALWAYS AS (...) STORED + // CockroachDB supports both syntaxes, so we use PostgreSQL-compatible one return [ 'account_db_v6_add_social_id_integrations', ` CREATE TABLE IF NOT EXISTS ${ns}.integrations ( - social_id INT8 NOT NULL, - kind STRING NOT NULL, + social_id ${types.int8} NOT NULL, + kind ${types.string} NOT NULL, workspace_uuid UUID, - _def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000')) STORED NOT VISIBLE, + _def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000'::UUID)) STORED, data JSONB, CONSTRAINT integrations_pk PRIMARY KEY (social_id, kind, _def_ws_uuid), - INDEX integrations_kind_idx (kind), CONSTRAINT integrations_social_id_fk FOREIGN KEY (social_id) REFERENCES ${ns}.social_id(_id), CONSTRAINT integrations_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid) ); CREATE TABLE IF NOT EXISTS ${ns}.integration_secrets ( - social_id INT8 NOT NULL, - kind STRING NOT NULL, + social_id ${types.int8} NOT NULL, + kind ${types.string} NOT NULL, workspace_uuid UUID, - _def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000')) STORED NOT VISIBLE, - key STRING, - secret STRING NOT NULL, + _def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000'::UUID)) STORED, + key ${types.string}, + secret ${types.string} NOT NULL, CONSTRAINT integration_secrets_pk PRIMARY KEY (social_id, kind, _def_ws_uuid, key) ); + + /* ======= I N D E X E S ======= */ + CREATE INDEX IF NOT EXISTS integrations_kind_idx ON ${ns}.integrations (kind); ` ] } -function getV7Migration (ns: string): [string, string] { +function getV7Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v7_add_display_value', ` @@ -337,7 +446,7 @@ function getV7Migration (ns: string): [string, string] { ] } -function getV8Migration (ns: string): [string, string] { +function getV8Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v8_add_account_max_workspaces', ` @@ -347,7 +456,7 @@ function getV8Migration (ns: string): [string, string] { ] } -function getV9Migration (ns: string): [string, string] { +function getV9Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v9_add_migrated_to_person', ` @@ -358,130 +467,172 @@ function getV9Migration (ns: string): [string, string] { ] } -function getV10Migration1 (ns: string): [string, string] { - return [ - 'account_db_v10_add_readonly_role', - ` - ALTER TYPE ${ns}.workspace_role ADD VALUE 'READONLYGUEST' AFTER 'DOCGUEST'; - ` - ] +function getV10Migration1 (ns: string, flavor: DBFlavor): [string, string] { + // For PostgreSQL, we need to check if the value exists before adding it + const addValueSql = + flavor === 'postgres' + ? ` + -- Add READONLYGUEST value to workspace_role enum (PostgreSQL) + DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumlabel = 'READONLYGUEST' + AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'workspace_role' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}')) + ) THEN + ALTER TYPE ${ns}.workspace_role ADD VALUE 'READONLYGUEST'; + END IF; + END $$; + ` + : ` + -- Add READONLYGUEST value to workspace_role enum (CockroachDB) + ALTER TYPE ${ns}.workspace_role ADD VALUE IF NOT EXISTS 'READONLYGUEST'; + ` + + return ['account_db_v10_add_readonly_role', addValueSql] } -function getV10Migration2 (ns: string): [string, string] { +function getV10Migration2 (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v10_add_allow_guests_flag_to_workspace', ` ALTER TABLE ${ns}.workspace - ADD COLUMN IF NOT EXISTS allow_read_only_guest BOOL NOT NULL DEFAULT FALSE; + ADD COLUMN IF NOT EXISTS allow_read_only_guest ${types.bool} NOT NULL DEFAULT FALSE; ` ] } -function getV11Migration (ns: string): [string, string] { +function getV11Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ - 'account_db_v10_add_migrated_to_person', + 'account_db_v11_add_pending_workspace_lock', ` CREATE TABLE IF NOT EXISTS ${ns}._pending_workspace_lock ( - id INT8 DEFAULT 1 PRIMARY KEY, + id ${types.int8} DEFAULT 1 PRIMARY KEY, CONSTRAINT single_row CHECK (id = 1) ); - + INSERT INTO ${ns}._pending_workspace_lock (id) VALUES (1) ON CONFLICT (id) DO NOTHING; ` ] } -function getV12Migration (ns: string): [string, string] { +function getV12Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v12_update_account_events_fk', ` - -- Drop existing foreign key constraint + -- Drop the existing foreign key constraint ALTER TABLE ${ns}.account_events DROP CONSTRAINT IF EXISTS account_events_account_fk; - -- Add new foreign key constraint referencing person table + -- Add the new foreign key constraint referencing the person table ALTER TABLE ${ns}.account_events ADD CONSTRAINT account_events_person_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.person(uuid); ` ] } -function getV13Migration (ns: string): [string, string] { +function getV13Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v13_update_workspace_fk_to_person', ` - -- Drop existing foreign key constraints + -- Drop the existing foreign key constraints ALTER TABLE ${ns}.workspace DROP CONSTRAINT IF EXISTS workspace_created_by_fk, DROP CONSTRAINT IF EXISTS workspace_billing_account_fk; - -- Add new foreign key constraints referencing person table + -- Add the new foreign key constraints referencing the person table ALTER TABLE ${ns}.workspace - ADD CONSTRAINT workspace_created_by_person_fk + ADD CONSTRAINT workspace_created_by_person_fk FOREIGN KEY (created_by) REFERENCES ${ns}.person(uuid), - ADD CONSTRAINT workspace_billing_account_person_fk + ADD CONSTRAINT workspace_billing_account_person_fk FOREIGN KEY (billing_account) REFERENCES ${ns}.person(uuid); ` ] } -function getV14Migration (ns: string): [string, string] { +function getV14Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v14_add_allow_guest_signup_flag_to_workspace', ` ALTER TABLE ${ns}.workspace - ADD COLUMN IF NOT EXISTS allow_guest_sign_up BOOL NOT NULL DEFAULT FALSE; + ADD COLUMN IF NOT EXISTS allow_guest_sign_up ${types.bool} NOT NULL DEFAULT FALSE; ` ] } -function getV15Migration (ns: string): [string, string] { +function getV15Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v15_add_target_region_to_workspace_status', ` ALTER TABLE ${ns}.workspace_status - ADD COLUMN IF NOT EXISTS target_region STRING; + ADD COLUMN IF NOT EXISTS target_region ${types.string}; ` ] } -function getV16Migration (ns: string): [string, string] { - return [ - 'account_db_v16_add_huly_assistant_social_id_type', - ` - ALTER TYPE ${ns}.social_id_type ADD VALUE 'huly-assistant' AFTER 'telegram'; - ` - ] +function getV16Migration (ns: string, flavor: DBFlavor): [string, string] { + // For PostgreSQL, we need to check if the value exists before adding it + const addValueSql = + flavor === 'postgres' + ? ` + -- Add huly-assistant value to social_id_type enum (PostgreSQL) + DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumlabel = 'huly-assistant' + AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'social_id_type' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}')) + ) THEN + ALTER TYPE ${ns}.social_id_type ADD VALUE 'huly-assistant'; + END IF; + END $$; + ` + : ` + -- Add huly-assistant value to social_id_type enum (CockroachDB) + ALTER TYPE ${ns}.social_id_type ADD VALUE IF NOT EXISTS 'huly-assistant'; + ` + + return ['account_db_v16_add_huly_assistant_social_id_type', addValueSql] } -function getV17Migration (ns: string): [string, string] { +function getV17Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v17_create_user_profile_table', ` /* ======= U S E R P R O F I L E ======= */ CREATE TABLE IF NOT EXISTS ${ns}.user_profile ( person_uuid UUID NOT NULL, - bio STRING, - city STRING, - country STRING, - website STRING, + bio ${types.string}, + city ${types.string}, + country ${types.string}, + website ${types.string}, social_links JSONB, - is_public BOOL NOT NULL DEFAULT FALSE, + is_public ${types.bool} NOT NULL DEFAULT FALSE, CONSTRAINT user_profile_pk PRIMARY KEY (person_uuid), - CONSTRAINT user_profile_person_fk FOREIGN KEY (person_uuid) REFERENCES ${ns}.person(uuid) ON DELETE CASCADE, - INDEX user_profile_is_public_idx (is_public) WHERE is_public = TRUE + CONSTRAINT user_profile_person_fk FOREIGN KEY (person_uuid) REFERENCES ${ns}.person(uuid) ON DELETE CASCADE ); - /* Remove city and country from person table */ + /* Remove city and country from the person table */ ALTER TABLE ${ns}.person DROP COLUMN IF EXISTS city, DROP COLUMN IF EXISTS country; + + /* ======= I N D E X E S ======= */ + CREATE INDEX IF NOT EXISTS user_profile_is_public_idx ON ${ns}.user_profile (is_public) WHERE is_public = TRUE; ` ] } -function getV18Migration (ns: string): [string, string] { +function getV18Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v18_populate_user_profiles', ` @@ -497,17 +648,19 @@ function getV18Migration (ns: string): [string, string] { ] } -function getV19Migration (ns: string): [string, string] { +function getV19Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] + return [ 'account_db_v19_subscription_table', ` /* ======= S U B S C R I P T I O N ======= */ /* Provider-agnostic subscription information for workspaces */ - /* Managed by billing service via payment provider webhooks (e.g. Polar.sh, Stripe) */ + /* Managed by a billing service via payment provider webhooks (e.g. Polar.sh, Stripe) */ /* Multiple active subscriptions allowed per workspace (tier + addons + support) */ /* Historical subscriptions preserved with status: canceled/expired */ - - CREATE TYPE IF NOT EXISTS ${ns}.subscription_status AS ENUM ( + + CREATE TYPE ${ns}.subscription_status AS ENUM ( 'active', 'trialing', 'past_due', @@ -515,55 +668,57 @@ function getV19Migration (ns: string): [string, string] { 'paused', 'expired' ); - + CREATE TABLE IF NOT EXISTS ${ns}.subscription ( - id STRING NOT NULL DEFAULT gen_random_uuid()::STRING, + id ${types.string} NOT NULL DEFAULT gen_random_uuid()::TEXT, workspace_uuid UUID NOT NULL, account_uuid UUID NOT NULL, -- Account that paid for the subscription - + -- Provider details - provider STRING NOT NULL, -- Payment provider identifier (e.g. 'polar', 'stripe', 'manual') - provider_subscription_id STRING NOT NULL, -- External subscription ID from the provider - provider_checkout_id STRING, -- External checkout/session ID that created this subscription - + provider ${types.string} NOT NULL, -- Payment provider identifier (e.g. 'polar', 'stripe', 'manual') + provider_subscription_id ${types.string} NOT NULL, -- External subscription ID from the provider + provider_checkout_id ${types.string}, -- External checkout/session ID that created this subscription + -- Subscription classification - type STRING NOT NULL DEFAULT 'tier', -- tier, support, etc. + type ${types.string} NOT NULL DEFAULT 'tier', -- tier, support, etc. status ${ns}.subscription_status NOT NULL DEFAULT 'active', - plan STRING NOT NULL, -- Plan identifier (e.g. 'rare', 'epic', 'legendary', 'custom') - + plan ${types.string} NOT NULL, -- Plan identifier (e.g. 'rare', 'epic', 'legendary', 'custom') + -- Amount paid (in cents, e.g. 9999 = $99.99) -- Used primarily for pay-what-you-want/donation subscriptions to track actual payment - amount INT8, - + amount ${types.int8}, + -- Billing period (optional) period_start BIGINT, period_end BIGINT, - + -- Trial information (optional) trial_end BIGINT, - + -- Cancellation info (optional) canceled_at BIGINT, will_cancel_at BIGINT, -- Scheduled cancellation date (cancel at period end) - + -- Provider-specific data (stored as JSONB for flexibility) -- e.g. customerExternalId, metadata, etc. provider_data JSONB, - + created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), updated_on BIGINT NOT NULL DEFAULT current_epoch_ms(), - + CONSTRAINT subscription_pk PRIMARY KEY (id), CONSTRAINT subscription_provider_subscription_id_unique UNIQUE (provider, provider_subscription_id), CONSTRAINT subscription_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid), - CONSTRAINT subscription_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid), - INDEX subscription_workspace_status_idx (workspace_uuid, status) + CONSTRAINT subscription_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid) ); + + /* ======= I N D E X E S ======= */ + CREATE INDEX IF NOT EXISTS subscription_workspace_status_idx ON ${ns}.subscription (workspace_uuid, status); ` ] } -function getV20Migration (ns: string): [string, string] { +function getV20Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v20_usage_info', ` @@ -573,7 +728,7 @@ function getV20Migration (ns: string): [string, string] { ] } -function getV21Migration (ns: string): [string, string] { +function getV21Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v21_add_failed_login_attempts', ` @@ -583,7 +738,7 @@ function getV21Migration (ns: string): [string, string] { ] } -function getV22Migration (ns: string): [string, string] { +function getV22Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v22_add_password_change_event_index', ` @@ -593,7 +748,7 @@ function getV22Migration (ns: string): [string, string] { ] } -function getV23Migration (ns: string): [string, string] { +function getV23Migration (ns: string, flavor: DBFlavor): [string, string] { return [ 'account_db_v23_add_password_aging_rule_to_workspace', ` @@ -603,7 +758,8 @@ function getV23Migration (ns: string): [string, string] { ] } -function getV24Migration (ns: string): [string, string] { +function getV24Migration (ns: string, flavor: DBFlavor): [string, string] { + const types = dbTypes[flavor] return [ 'account_db_v24_add_workspace_permissions_table', ` @@ -611,14 +767,18 @@ function getV24Migration (ns: string): [string, string] { CREATE TABLE IF NOT EXISTS ${ns}.workspace_permissions ( workspace_uuid UUID NOT NULL, account_uuid UUID NOT NULL, - permission STRING NOT NULL, - created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), + permission ${types.string} NOT NULL, + created_on ${types.int8} NOT NULL DEFAULT current_epoch_ms(), CONSTRAINT workspace_permissions_pk PRIMARY KEY (workspace_uuid, account_uuid, permission), CONSTRAINT workspace_permissions_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid), - CONSTRAINT workspace_permissions_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid), - INDEX workspace_permissions_account_idx (account_uuid), - INDEX workspace_permissions_permission_idx (permission) + CONSTRAINT workspace_permissions_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid) ); + + CREATE INDEX IF NOT EXISTS workspace_permissions_account_idx + ON ${ns}.workspace_permissions (account_uuid); + + CREATE INDEX IF NOT EXISTS workspace_permissions_permission_idx + ON ${ns}.workspace_permissions (permission); ` ] } diff --git a/server/account/src/collections/postgres/postgres.ts b/server/account/src/collections/postgres/postgres.ts index f748d7f31e3..4bb1b8e0193 100644 --- a/server/account/src/collections/postgres/postgres.ts +++ b/server/account/src/collections/postgres/postgres.ts @@ -49,7 +49,8 @@ import type { AccountAggregatedInfo, UserProfile, Subscription, - WorkspacePermission + WorkspacePermission, + DBFlavor } from '../../types' function toSnakeCase (str: string): string { @@ -117,7 +118,8 @@ implements DbCollection { constructor ( readonly name: string, readonly client: Sql, - readonly options: PostgresDbCollectionOptions = {} + readonly options: PostgresDbCollectionOptions = {}, + readonly filterFields: string[] = [] ) {} get ns (): string { @@ -249,6 +251,14 @@ implements DbCollection { for (const field of this.timestampFields) { res[field] = convertTimestamp(res[field]) } + if (this.filterFields.length > 0) { + for (const key of Object.keys(res)) { + if (this.filterFields.includes(key.toLowerCase())) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete res[key] + } + } + } return res as T } @@ -334,7 +344,7 @@ implements DbCollection { .join(', ') const sql = ` - INSERT INTO ${this.getTableName()} + INSERT INTO ${this.getTableName()} (${columnsList.map((k) => `"${k}"`).join(', ')}) VALUES ${placeholders} RETURNING * @@ -432,7 +442,7 @@ export class AccountPostgresDbCollection protected buildSelectClause (): string { return `SELECT * FROM ( - SELECT + SELECT a.uuid, a.timezone, a.locale, @@ -528,7 +538,8 @@ export class PostgresAccountDB implements AccountDB { constructor ( readonly client: Sql, - readonly ns: string = 'global_account' + readonly ns: string = 'global_account', + readonly dbFlavor: DBFlavor = 'cockroach' ) { const withRetryClient = this.withRetry this.person = new PostgresDbCollection('person', client, { ns, idKey: 'uuid', withRetryClient }) @@ -564,11 +575,19 @@ export class PostgresAccountDB implements AccountDB { }) this.mailbox = new PostgresDbCollection('mailbox', client, { ns, withRetryClient }) this.mailboxSecret = new PostgresDbCollection('mailbox_secrets', client, { ns, withRetryClient }) - this.integration = new PostgresDbCollection('integrations', client, { ns, withRetryClient }) - this.integrationSecret = new PostgresDbCollection('integration_secrets', client, { - ns, - withRetryClient - }) + this.integration = new PostgresDbCollection('integrations', client, { ns, withRetryClient }, [ + '_def_ws_uuid', + '_defwsuuid' + ]) + this.integrationSecret = new PostgresDbCollection( + 'integration_secrets', + client, + { + ns, + withRetryClient + }, + ['_def_ws_uuid', '_defwsuuid'] + ) this.userProfile = new PostgresDbCollection('user_profile', client, { ns, idKey: 'personUuid', @@ -614,8 +633,8 @@ export class PostgresAccountDB implements AccountDB { const executeMigration = async (client: Sql): Promise => { updateInterval = setInterval(() => { this.client` - UPDATE ${this.client(this.ns)}._account_applied_migrations - SET last_processed_at = NOW() + UPDATE ${this.client(this.ns)}._account_applied_migrations + SET last_processed_at = NOW() WHERE identifier = ${name} AND applied_at IS NULL `.catch((err) => { console.error(`Failed to update last_processed_at for migration ${name}:`, err) @@ -634,7 +653,7 @@ export class PostgresAccountDB implements AccountDB { // Only locks if row exists and is not already locked const existing = await client` SELECT identifier, applied_at, last_processed_at - FROM ${this.client(this.ns)}._account_applied_migrations + FROM ${this.client(this.ns)}._account_applied_migrations WHERE identifier = ${name} FOR UPDATE NOWAIT ` @@ -649,7 +668,7 @@ export class PostgresAccountDB implements AccountDB { ) { // Take over the stale migration await client` - UPDATE ${this.client(this.ns)}._account_applied_migrations + UPDATE ${this.client(this.ns)}._account_applied_migrations SET last_processed_at = NOW() WHERE identifier = ${name} ` @@ -658,8 +677,8 @@ export class PostgresAccountDB implements AccountDB { } } else { const res = await client` - INSERT INTO ${this.client(this.ns)}._account_applied_migrations - (identifier, ddl, last_processed_at) + INSERT INTO ${this.client(this.ns)}._account_applied_migrations + (identifier, ddl, last_processed_at) VALUES (${name}, ${ddl}, NOW()) ON CONFLICT (identifier) DO NOTHING ` @@ -674,8 +693,8 @@ export class PostgresAccountDB implements AccountDB { if (executed) { await this.client` - UPDATE ${this.client(this.ns)}._account_applied_migrations - SET applied_at = NOW() + UPDATE ${this.client(this.ns)}._account_applied_migrations + SET applied_at = NOW() WHERE identifier = ${name} ` migrationComplete = true @@ -715,14 +734,14 @@ export class PostgresAccountDB implements AccountDB { , last_processed_at TIMESTAMP WITH TIME ZONE ); - ALTER TABLE ${this.ns}._account_applied_migrations + ALTER TABLE ${this.ns}._account_applied_migrations ADD COLUMN IF NOT EXISTS last_processed_at TIMESTAMP WITH TIME ZONE; ` ) const constraintsExist = await this.client` - SELECT 1 - FROM information_schema.columns + SELECT 1 + FROM information_schema.columns WHERE table_schema = ${this.ns} AND table_name = '_account_applied_migrations' AND column_name = 'applied_at' @@ -733,10 +752,10 @@ export class PostgresAccountDB implements AccountDB { try { await this.client.unsafe( ` - ALTER TABLE ${this.ns}._account_applied_migrations + ALTER TABLE ${this.ns}._account_applied_migrations ALTER COLUMN applied_at DROP DEFAULT; - ALTER TABLE ${this.ns}._account_applied_migrations + ALTER TABLE ${this.ns}._account_applied_migrations ALTER COLUMN applied_at DROP NOT NULL; ` ) @@ -815,7 +834,7 @@ export class PostgresAccountDB implements AccountDB { const values = data.flat() const sql = ` - INSERT INTO ${this.getWsMembersTableName()} + INSERT INTO ${this.getWsMembersTableName()} (account_uuid, workspace_uuid, role) VALUES ${placeholders} ` @@ -868,7 +887,7 @@ export class PostgresAccountDB implements AccountDB { } async getAccountWorkspaces (accountUuid: AccountUuid): Promise { - const sql = `SELECT + const sql = `SELECT w.uuid, w.name, w.url, @@ -892,8 +911,8 @@ export class PostgresAccountDB implements AccountDB { 'processing_message', s.processing_message, 'backup_info', s.backup_info, 'usage_info', s.usage_info - ) status - FROM ${this.getWsMembersTableName()} as m + ) status + FROM ${this.getWsMembersTableName()} as m INNER JOIN ${this.workspace.getTableName()} as w ON m.workspace_uuid = w.uuid INNER JOIN ${this.workspaceStatus.getTableName()} as s ON s.workspace_uuid = w.uuid WHERE m.account_uuid = $1 @@ -922,7 +941,7 @@ export class PostgresAccountDB implements AccountDB { wsLivenessMs?: number ): Promise { const sqlChunks: string[] = [ - `SELECT + `SELECT w.uuid, w.name, w.url, @@ -932,7 +951,7 @@ export class PostgresAccountDB implements AccountDB { w.region, w.created_by, w.created_on, - w.billing_account, + w.billing_account, json_build_object( 'mode', s.mode, 'processing_progress', s.processing_progress, @@ -1028,7 +1047,7 @@ export class PostgresAccountDB implements AccountDB { async setPassword (accountUuid: AccountUuid, hash: Buffer, salt: Buffer): Promise { await this.withRetry( async (rTx) => - await rTx`UPSERT INTO ${this.client(this.account.getPasswordsTableName())} (account_uuid, hash, salt) VALUES (${accountUuid}, ${hash.buffer as any}::bytea, ${salt.buffer as any}::bytea)` + await rTx`INSERT INTO ${this.client(this.account.getPasswordsTableName())} (account_uuid, hash, salt) VALUES (${accountUuid}, ${hash.buffer as any}::bytea, ${salt.buffer as any}::bytea) ON CONFLICT (account_uuid) DO UPDATE SET hash = EXCLUDED.hash, salt = EXCLUDED.salt;` ) } @@ -1070,7 +1089,7 @@ export class PostgresAccountDB implements AccountDB { const sqlChunks: string[] = [ ` WITH account_data AS ( - SELECT + SELECT a.uuid, a.timezone, a.locale, @@ -1078,16 +1097,16 @@ export class PostgresAccountDB implements AccountDB { a.max_workspaces, p.first_name, p.last_name, - p.country, - p.city, + up.country, + up.city, p.migrated_to, ( SELECT jsonb_agg(jsonb_build_object( 'socialId', i.social_id, 'kind', i.kind, 'workspaceUuid', i.workspace_uuid - )) - FROM ${this.integration.getTableName()} i + )) + FROM ${this.integration.getTableName()} i WHERE i.social_id IN (SELECT _id FROM ${this.socialId.getTableName()} s WHERE s.person_uuid = a.uuid) ) as integrations, ( @@ -1121,6 +1140,7 @@ export class PostgresAccountDB implements AccountDB { ) as workspaces FROM ${this.account.getTableName()} a INNER JOIN ${this.ns}.person p ON p.uuid = a.uuid + LEFT JOIN ${this.userProfile.getTableName()} up ON up.person_uuid = p.uuid ` ] @@ -1129,11 +1149,11 @@ export class PostgresAccountDB implements AccountDB { if (search !== undefined && search !== '') { sqlChunks.push(` - WHERE - p.first_name ILIKE $${paramIndex} OR + WHERE + p.first_name ILIKE $${paramIndex} OR p.last_name ILIKE $${paramIndex} OR EXISTS ( - SELECT 1 FROM ${this.socialId.getTableName()} s + SELECT 1 FROM ${this.socialId.getTableName()} s WHERE s.person_uuid = a.uuid AND s.value ILIKE $${paramIndex} ) `) @@ -1193,7 +1213,7 @@ export class PostgresAccountDB implements AccountDB { } protected getMigrations (): [string, string][] { - return getMigrations(this.ns) + return getMigrations(this.ns, this.dbFlavor) } async batchAssignWorkspacePermission ( diff --git a/server/account/src/types.ts b/server/account/src/types.ts index 1ce2627dc8d..90912d89b39 100644 --- a/server/account/src/types.ts +++ b/server/account/src/types.ts @@ -304,6 +304,8 @@ export type WorkspaceStatusData = Omit export type WorkspaceInviteData = Omit +export type DBFlavor = 'postgres' | 'cockroach' | 'unknown' + /* ========= D A T A B A S E C O L L E C T I O N S ========= */ export interface AccountDB { person: DbCollection diff --git a/server/account/src/utils.ts b/server/account/src/utils.ts index 07e9218e423..3796f9e3b11 100644 --- a/server/account/src/utils.ts +++ b/server/account/src/utils.ts @@ -64,12 +64,31 @@ import { type WorkspaceInvite, type WorkspaceJoinInfo, type WorkspaceLoginInfo, - type WorkspaceStatus + type WorkspaceStatus, + type DBFlavor } from './types' import { isAdminEmail } from './admin' +import { type Sql } from 'postgres' export const GUEST_ACCOUNT = 'b6996120-416f-49cd-841e-e4a5d2e49c9b' as PersonUuid +export async function getDbFlavor (pgClient: Sql): Promise { + // Run the version query + const [{ version }] = await pgClient`SELECT version()` + + // CockroachDB’s string contains “Cockroach” (case‑insensitive) + if (/cockroach/i.test(version)) { + return 'cockroach' + } + + // Anything else that looks like a PostgreSQL version string + if (/postgresql/i.test(version)) { + return 'postgres' + } + + // Fallback – could be a custom build or something unexpected + return 'unknown' +} export async function getAccountDB ( uri: string, dbNs?: string, @@ -98,10 +117,25 @@ export async function getAccountDB ( }) const client = getDBClient(uri) const pgClient = await client.getClient() - const pgAccount = new PostgresAccountDB(pgClient, dbNs ?? 'global_account') + + let flavor: DBFlavor = 'unknown' let error = false + do { + try { + flavor = await getDbFlavor(pgClient) + error = false + } catch (err: any) { + error = true + console.error('Error while initializing postgres account db', err.message) + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + } while (error) + error = false + + const pgAccount = new PostgresAccountDB(pgClient, dbNs ?? 'global_account', flavor) + do { try { await pgAccount.init() diff --git a/services/datalake/pod-datalake/src/datalake/db.ts b/services/datalake/pod-datalake/src/datalake/db.ts index 83388710d41..683d6de101c 100644 --- a/services/datalake/pod-datalake/src/datalake/db.ts +++ b/services/datalake/pod-datalake/src/datalake/db.ts @@ -90,12 +90,15 @@ export async function createDb (ctx: MeasureContext, connectionString: string): fetch_types: false, prepare: false, types: { - // https://jdbc.postgresql.org/documentation/publicapi/constant-values.html int8: { - to: 0, + // Corrected OID for BIGINT is 20. + // We parse BIGINT as Number to maintain compatibility with parseInt() used in the codebase. + // WARNING: This can lead to precision loss for values larger than Number.MAX_SAFE_INTEGER. + // If you expect to handle sizes larger than ~9 PB, consider migrating the codebase to use BigInt. + to: 20, from: [20], - serialize: (value: string) => value.toString(), - parse: (value: number) => Number(value) + serialize: (value: number | string) => value.toString(), + parse: (value: string) => Number(value) } } }) @@ -143,7 +146,7 @@ export class PostgresDB implements BlobDB { )`) const appliedMigrations = (await this.execute('SELECT name FROM blob.migrations')).map((row) => row.name) - ctx.info('applied migrations', { migrations: appliedMigrations }) + ctx.info('Applied migrations', { migrations: appliedMigrations }) for (const [name, sql] of getMigrations()) { if (appliedMigrations.includes(name)) { @@ -151,11 +154,11 @@ export class PostgresDB implements BlobDB { } try { - ctx.warn('applying migration', { migration: name }) + ctx.warn('Applying migration', { migration: name }) await this.execute(sql) await this.execute('INSERT INTO blob.migrations (name) VALUES ($1)', [name]) } catch (err: any) { - ctx.error('failed to apply migration', { migration: name, error: err }) + ctx.error('Failed to apply migration', { migration: name, error: err }) throw err } } @@ -237,8 +240,13 @@ export class PostgresDB implements BlobDB { await this.execute( ` - UPSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at) + INSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at) VALUES ($1, $2, $3, $4, $5, NULL) + ON CONFLICT (workspace, name) DO UPDATE SET + hash = EXCLUDED.hash, + location = EXCLUDED.location, + parent = EXCLUDED.parent, + deleted_at = EXCLUDED.deleted_at `, [workspace, name, hash, location, parent] ) @@ -249,8 +257,12 @@ export class PostgresDB implements BlobDB { await this.execute( ` - UPSERT INTO blob.data (hash, location, filename, size, type) + INSERT INTO blob.data (hash, location, filename, size, type) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (hash, location) DO UPDATE SET + filename = EXCLUDED.filename, + size = EXCLUDED.size, + type = EXCLUDED.type `, [hash, location, filename, size, type] ) @@ -259,18 +271,29 @@ export class PostgresDB implements BlobDB { async createBlobData (ctx: MeasureContext, data: BlobWithDataRecord): Promise { const { workspace, name, hash, location, parent, filename, size, type } = data + // First upsert into data table await this.execute( ` - UPSERT INTO blob.data (hash, location, filename, size, type) + INSERT INTO blob.data (hash, location, filename, size, type) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (hash, location) DO UPDATE SET + filename = EXCLUDED.filename, + size = EXCLUDED.size, + type = EXCLUDED.type `, [hash, location, filename, size, type] ) + // Then upsert into blob table await this.execute( ` - UPSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at) + INSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at) VALUES ($1, $2, $3, $4, $5, NULL) + ON CONFLICT (workspace, name) DO UPDATE SET + hash = EXCLUDED.hash, + location = EXCLUDED.location, + parent = EXCLUDED.parent, + deleted_at = EXCLUDED.deleted_at `, [workspace, name, hash, location, parent] ) @@ -336,10 +359,12 @@ export class PostgresDB implements BlobDB { await this.execute( ` - UPSERT INTO blob.meta (workspace, name, meta) - VALUES ($1, $2, $3) + INSERT INTO blob.meta (workspace, name, meta) + VALUES ($1, $2, $3::jsonb) + ON CONFLICT (workspace, name) DO UPDATE SET + meta = EXCLUDED.meta `, - [workspace, name, meta] + [workspace, name, JSON.stringify(meta)] ) } @@ -564,7 +589,7 @@ export function escape (value: any): string { if (value instanceof Date) { return `'${value.toISOString()}'` } else { - return `'${JSON.stringify(value)}'` + return `'${JSON.stringify(value).replace(/'/g, "''")}'` } default: throw new Error(`Unsupported value type: ${typeof value}`) @@ -577,33 +602,33 @@ function getMigrations (): [string, string][] { function migrationV1 (): [string, string] { const sql = ` - CREATE TYPE IF NOT EXISTS blob.location AS ENUM ('eu', 'weur', 'eeur', 'wnam', 'enam', 'apac'); + CREATE TYPE blob.location AS ENUM ('eu', 'weur', 'eeur', 'wnam', 'enam', 'apac'); CREATE TABLE IF NOT EXISTS blob.data ( hash UUID NOT NULL, location blob.location NOT NULL, - size INT8 NOT NULL, - filename STRING(255) NOT NULL, - type STRING(255) NOT NULL, + size BIGINT NOT NULL, + filename VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, CONSTRAINT pk_data PRIMARY KEY (hash, location) ); CREATE TABLE IF NOT EXISTS blob.blob ( - workspace STRING(255) NOT NULL, - name STRING(255) NOT NULL, + workspace VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, hash UUID NOT NULL, location blob.location NOT NULL, - parent STRING(255) DEFAULT NULL, + parent VARCHAR(255) DEFAULT NULL, created_at TIMESTAMP NOT NULL DEFAULT NOW(), deleted_at TIMESTAMP DEFAULT NULL, CONSTRAINT pk_blob PRIMARY KEY (workspace, name), CONSTRAINT fk_data FOREIGN KEY (hash, location) REFERENCES blob.data (hash, location), - CONSTRAINT fk_parent FOREIGN KEY (workspace, parent) REFERENCES blob.blob (workspace, name) ON DELETE CASCADE + CONSTRAINT fk_parent FOREIGN KEY (workspace, parent) REFERENCES blob.blob (workspace, name) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS blob.meta ( - workspace STRING(255) NOT NULL, - name STRING(255) NOT NULL, + workspace VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, meta JSONB NOT NULL, CONSTRAINT pk_meta PRIMARY KEY (workspace, name), CONSTRAINT fk_blob FOREIGN KEY (workspace, name) REFERENCES blob.blob (workspace, name) @@ -623,7 +648,7 @@ function migrationV2 (): [string, string] { AND NOT EXISTS ( SELECT 1 FROM blob.blob existing - WHERE existing.workspace = w.uuid::string + WHERE existing.workspace = w.uuid::text AND existing.name = blob.blob.name ); @@ -634,7 +659,7 @@ function migrationV2 (): [string, string] { AND NOT EXISTS ( SELECT 1 FROM blob.meta existing - WHERE existing.workspace = w.uuid::string + WHERE existing.workspace = w.uuid::text AND existing.name = blob.meta.name ); diff --git a/tests/.env b/tests/.env index 7c1f4743864..44004e3a971 100644 --- a/tests/.env +++ b/tests/.env @@ -6,4 +6,5 @@ BACKUP_BUCKET_NAME=dev-backups DB_URL=mongodb://mongodb:27018 MONGO_URL=mongodb://mongodb:27018 DB_PG_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable +DB_PURE_PG_URL=postgresql://postgres:postgres@postgres:5433/postgres QUEUE_CONFIG='redpanda:9093;-staging' diff --git a/tests/docker-compose.purepg.yaml b/tests/docker-compose.purepg.yaml new file mode 100644 index 00000000000..3adbaf65b33 --- /dev/null +++ b/tests/docker-compose.purepg.yaml @@ -0,0 +1,52 @@ +services: + account: + environment: + - DB_URL=${DB_PURE_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + - PROCEED_V7_MONGO=false + transactor: + environment: + - DB_URL=${DB_PURE_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + workspace: + environment: + - DB_URL=${DB_PURE_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + - ACCOUNTS_DB_URL=${DB_PURE_PG_URL} + fulltext: + environment: + - DB_URL=${DB_PURE_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + front: + environment: + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + - FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename + - UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace + - DATALAKE_URL=http://localhost:4031 + - PREVIEW_URL=http://localhost:4031 + collaborator: + environment: + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + datalake: + image: hardcoreeng/datalake + depends_on: + minio: + condition: service_healthy + cockroach: + condition: service_started + stats: + condition: service_started + account: + condition: service_started + ports: + - 4031:4031 + environment: + - PORT=4031 + - SECRET=secret + - ACCOUNTS_URL=http://account:3003 + - STATS_URL=http://stats:4901 + - STREAM_URL=http://localhost:1081/recording + - DB_URL=${DB_PURE_PG_URL} + - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin + - QUEUE_CONFIG=${QUEUE_CONFIG} + restart: unless-stopped diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 2c47cf92a34..a2e50d5afc1 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -34,6 +34,21 @@ services: - '18089:8080' command: start-single-node --insecure restart: unless-stopped + postgres: + image: postgres:18.1 + command: ["postgres", "-p", "5433"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -p 5433"] + interval: 5s + timeout: 5s + retries: 10 + ports: + - 5433:5433 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + restart: unless-stopped redpanda: image: docker.redpanda.com/redpandadata/redpanda:v24.3.6 command: @@ -110,10 +125,14 @@ services: account: image: hardcoreeng/account pull_policy: never + depends_on: + postgres: + condition: service_healthy links: - mongodb - minio - cockroach + - postgres ports: - 3003:3003 volumes: @@ -136,6 +155,7 @@ services: links: - mongodb - cockroach + - postgres - minio - redpanda depends_on: @@ -198,6 +218,7 @@ services: - minio - rekoni - cockroach + - postgres - account - stats - redpanda diff --git a/tests/prepare-cockroach.sh b/tests/prepare-cockroach.sh new file mode 100755 index 00000000000..069c99ea348 --- /dev/null +++ b/tests/prepare-cockroach.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +docker compose -p sanity kill +docker compose -p sanity down --volumes +docker compose -p sanity up -d --force-recreate --renew-anon-volumes +docker_exit=$? +if [ ${docker_exit} -eq 0 ]; then + echo "Container started successfully" +else + echo "Container started with errors" + exit ${docker_exit} +fi + +if [ "x$DO_CLEAN" == 'xtrue' ]; then + echo 'Do docker Clean' + docker system prune -a -f +fi + +./wait-elastic.sh 9201 + +# Create user record in accounts +./tool-cockroach.sh create-account user1 -f John -l Appleseed -p 1234 +./tool-cockroach.sh create-account user2 -f Kainin -l Dirak -p 1234 +./tool-cockroach.sh create-account admin -f Super -l User -p 1234 + +# Create workspace record in accounts +./tool-cockroach.sh create-workspace sanity-ws email:user1 + +./restore-cockroach.sh +rm -rf ./sanity/.auth diff --git a/tests/prepare-pg.sh b/tests/prepare-pg.sh index 319ffc35c21..1ee197c956d 100755 --- a/tests/prepare-pg.sh +++ b/tests/prepare-pg.sh @@ -2,7 +2,7 @@ docker compose -p sanity kill docker compose -p sanity down --volumes -docker compose -p sanity up -d --force-recreate --renew-anon-volumes +docker compose -f docker-compose.yaml -f docker-compose.purepg.yaml -p sanity up -d --force-recreate --renew-anon-volumes docker_exit=$? if [ ${docker_exit} -eq 0 ]; then echo "Container started successfully" diff --git a/tests/prepare-tests.sh b/tests/prepare-tests.sh index b439ad24322..cd6434a0605 100755 --- a/tests/prepare-tests.sh +++ b/tests/prepare-tests.sh @@ -2,7 +2,7 @@ docker compose -p sanity kill docker compose -p sanity down --volumes -docker compose -f docker-compose.yaml -p sanity up elastic mongodb cockroach redpanda redpanda_console -d --force-recreate --renew-anon-volumes +docker compose -f docker-compose.yaml -p sanity up elastic mongodb cockroach postgres redpanda -d --force-recreate --renew-anon-volumes docker_exit=$? if [ ${docker_exit} -eq 0 ]; then echo "Container started successfully" @@ -11,4 +11,4 @@ else exit ${docker_exit} fi -./wait-elastic.sh 9201 \ No newline at end of file +./wait-elastic.sh 9201 diff --git a/tests/restore-cockroach.sh b/tests/restore-cockroach.sh new file mode 100755 index 00000000000..c81cd00434d --- /dev/null +++ b/tests/restore-cockroach.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Restore workspace contents in mongo/elastic +./tool-cockroach.sh backup-restore ./sanity-ws sanity-ws --upgrade + +# ./tool-cockroach.sh upgrade-workspace sanity-ws + +# Re-assign user to workspace. +./tool-cockroach.sh assign-workspace user1 sanity-ws +./tool-cockroach.sh assign-workspace user2 sanity-ws +./tool-cockroach.sh set-user-role user1 sanity-ws OWNER +./tool-cockroach.sh set-user-role user2 sanity-ws OWNER + +./tool-cockroach.sh configure sanity-ws --enable=* +./tool-cockroach.sh configure sanity-ws --list + +# setup issue createdOn for yesterday +./tool-cockroach.sh change-field sanity-ws --objectId 65e47f1f1b875b51e3b4b983 --objectClass tracker:class:Issue --attribute createdOn --value $(($(date +%s)*1000 - 86400000)) --type number diff --git a/tests/tool-cockroach.sh b/tests/tool-cockroach.sh new file mode 100755 index 00000000000..bb968735fcc --- /dev/null +++ b/tests/tool-cockroach.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MODEL_VERSION=$(node ../common/scripts/show_version.js) +export STORAGE_CONFIG="datalake|http://localhost:4031" +export ACCOUNTS_URL=http://localhost:3003 +export TRANSACTOR_URL=ws://localhost:3334 +export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export MONGO_URL=mongodb://localhost:27018 +export ELASTIC_URL=http://localhost:9201 +export SERVER_SECRET=secret +export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export QUEUE_CONFIG='localhost:19093;-staging' + +node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@ diff --git a/tests/tool-pg.sh b/tests/tool-pg.sh index 580a10c031f..10f772476a0 100755 --- a/tests/tool-pg.sh +++ b/tests/tool-pg.sh @@ -4,11 +4,11 @@ export MODEL_VERSION=$(node ../common/scripts/show_version.js) export STORAGE_CONFIG="datalake|http://localhost:4031" export ACCOUNTS_URL=http://localhost:3003 export TRANSACTOR_URL=ws://localhost:3334 -export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export ACCOUNT_DB_URL=postgresql://postgres:postgres@localhost:5433/postgres export MONGO_URL=mongodb://localhost:27018 export ELASTIC_URL=http://localhost:9201 export SERVER_SECRET=secret -export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export DB_URL=postgresql://postgres:postgres@localhost:5433/postgres export QUEUE_CONFIG='localhost:19093;-staging' -node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@ \ No newline at end of file +node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@ diff --git a/ws-tests/docker-compose.yaml b/ws-tests/docker-compose.yaml index 00fd51e2138..e1cf6fc9073 100644 --- a/ws-tests/docker-compose.yaml +++ b/ws-tests/docker-compose.yaml @@ -147,6 +147,9 @@ services: pull_policy: never extra_hosts: - 'huly.local:host-gateway' + depends_on: + cockroach: + condition: service_started links: - mongodb - minio