From 054a4e554422185e6f74c945dade686a676ba9ff Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Thu, 5 Dec 2024 06:49:13 -0600
Subject: [PATCH 01/28] add ec2 instance with pgbouncer

---
 cdk/PgBouncer.ts               | 203 +++++++++++++++++++++++++++++++++
 cdk/PgStacInfra.ts             | 137 +++++++++++++++-------
 cdk/handlers/raster_handler.py |   6 +-
 3 files changed, 299 insertions(+), 47 deletions(-)
 create mode 100644 cdk/PgBouncer.ts

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
new file mode 100644
index 0000000..2e02716
--- /dev/null
+++ b/cdk/PgBouncer.ts
@@ -0,0 +1,203 @@
+import {
+  aws_ec2 as ec2,
+  aws_iam as iam,
+  aws_secretsmanager as secretsmanager,
+  Stack,
+} from "aws-cdk-lib";
+import { Construct } from "constructs";
+
+export interface PgBouncerProps {
+  /**
+   * VPC to deploy PgBouncer into
+   */
+  vpc: ec2.IVpc;
+
+  /**
+   * The RDS instance to connect to
+   */
+  database: {
+    connections: ec2.Connections;
+    secret: secretsmanager.ISecret;
+  };
+
+  /**
+   * Security groups that need access to PgBouncer
+   */
+  clientSecurityGroups: ec2.ISecurityGroup[];
+
+  /**
+   * Whether to deploy in public subnet
+   * @default false
+   */
+  usePublicSubnet?: boolean;
+
+  /**
+   * Instance type for PgBouncer
+   * @default t3.small
+   */
+  instanceType?: ec2.InstanceType;
+
+  /**
+   * PgBouncer configuration options
+   */
+  pgBouncerConfig?: {
+    poolMode?: "transaction" | "session" | "statement";
+    maxClientConn?: number;
+    defaultPoolSize?: number;
+    minPoolSize?: number;
+    reservePoolSize?: number;
+    reservePoolTimeout?: number;
+    maxDbConnections?: number;
+    maxUserConnections?: number;
+  };
+}
+
+export class PgBouncer extends Construct {
+  public readonly securityGroup: ec2.ISecurityGroup;
+  public readonly instance: ec2.Instance;
+  public readonly endpoint: string;
+
+  constructor(scope: Construct, id: string, props: PgBouncerProps) {
+    super(scope, id);
+
+    const {
+      vpc,
+      database,
+      clientSecurityGroups,
+      usePublicSubnet = false,
+      instanceType = ec2.InstanceType.of(
+        ec2.InstanceClass.T3,
+        ec2.InstanceSize.SMALL,
+      ),
+      pgBouncerConfig = {
+        poolMode: "transaction",
+        maxClientConn: 1000,
+        defaultPoolSize: 20,
+        minPoolSize: 10,
+        reservePoolSize: 5,
+        reservePoolTimeout: 5,
+        maxDbConnections: 50,
+        maxUserConnections: 50,
+      },
+    } = props;
+
+    // Create security group for PgBouncer
+    this.securityGroup = new ec2.SecurityGroup(this, "SecurityGroup", {
+      vpc,
+      description: "Security group for PgBouncer instance",
+      allowAllOutbound: true,
+    });
+
+    // Allow incoming PostgreSQL traffic from client security groups
+    clientSecurityGroups.forEach((clientSg, index) => {
+      this.securityGroup.addIngressRule(
+        clientSg,
+        ec2.Port.tcp(5432),
+        `Allow PostgreSQL access from client security group ${index + 1}`,
+      );
+    });
+
+    // Allow PgBouncer to connect to RDS
+    database.connections.allowFrom(
+      this.securityGroup,
+      ec2.Port.tcp(5432),
+      "Allow PgBouncer to connect to RDS",
+    );
+
+    // Create role for PgBouncer instance
+    const role = new iam.Role(this, "InstanceRole", {
+      assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"),
+      managedPolicies: [
+        iam.ManagedPolicy.fromAwsManagedPolicyName(
+          "AmazonSSMManagedInstanceCore",
+        ),
+      ],
+    });
+
+    // Add policy to allow reading RDS credentials from Secrets Manager
+    role.addToPolicy(
+      new iam.PolicyStatement({
+        actions: ["secretsmanager:GetSecretValue"],
+        resources: [database.secret.secretArn],
+      }),
+    );
+
+    // Create PgBouncer instance
+    this.instance = new ec2.Instance(this, "Instance", {
+      vpc,
+      vpcSubnets: {
+        subnetType: usePublicSubnet
+          ? ec2.SubnetType.PUBLIC
+          : ec2.SubnetType.PRIVATE_WITH_EGRESS,
+      },
+      instanceType,
+      machineImage: new ec2.AmazonLinuxImage({
+        generation: ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
+      }),
+      securityGroup: this.securityGroup,
+      role,
+    });
+
+    // Create user data script
+    const userDataScript = ec2.UserData.forLinux();
+    userDataScript.addCommands(
+      "yum update -y",
+      "yum install -y pgbouncer jq aws-cli",
+      // Create configuration update script
+      `cat <<EOF > /usr/local/bin/update-pgbouncer-config.sh
+#!/bin/bash
+SECRET_ARN="${database.secret.secretArn}"
+REGION="${Stack.of(this).region}"
+
+# Fetch secret
+SECRET=\$(aws secretsmanager get-secret-value --secret-id \$SECRET_ARN --region \$REGION --query SecretString --output text)
+
+# Parse values
+DB_HOST=\$(echo \$SECRET | jq -r '.host')
+DB_PORT=\$(echo \$SECRET | jq -r '.port')
+DB_NAME=\$(echo \$SECRET | jq -r '.dbname')
+DB_USER=\$(echo \$SECRET | jq -r '.username')
+DB_PASSWORD=\$(echo \$SECRET | jq -r '.password')
+
+# Create pgbouncer config
+cat <<EOC > /etc/pgbouncer/pgbouncer.ini
+[databases]
+* = host=\$DB_HOST port=\$DB_PORT dbname=\$DB_NAME
+
+[pgbouncer]
+listen_addr = *
+listen_port = 5432
+auth_type = md5
+auth_file = /etc/pgbouncer/userlist.txt
+pool_mode = ${pgBouncerConfig.poolMode}
+max_client_conn = ${pgBouncerConfig.maxClientConn}
+default_pool_size = ${pgBouncerConfig.defaultPoolSize}
+min_pool_size = ${pgBouncerConfig.minPoolSize}
+reserve_pool_size = ${pgBouncerConfig.reservePoolSize}
+reserve_pool_timeout = ${pgBouncerConfig.reservePoolTimeout}
+max_db_connections = ${pgBouncerConfig.maxDbConnections}
+max_user_connections = ${pgBouncerConfig.maxUserConnections}
+EOC
+
+# Create auth file
+echo "\\"$DB_USER\\" \\"$DB_PASSWORD\\"" > /etc/pgbouncer/userlist.txt
+
+# Set permissions
+chown pgbouncer:pgbouncer /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
+chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
+
+# Restart pgbouncer
+systemctl restart pgbouncer
+EOF`,
+      "chmod +x /usr/local/bin/update-pgbouncer-config.sh",
+      "/usr/local/bin/update-pgbouncer-config.sh",
+      "systemctl enable pgbouncer",
+      "systemctl start pgbouncer",
+    );
+
+    this.instance.addUserData(userDataScript.render());
+
+    // Set the endpoint
+    this.endpoint = this.instance.instancePrivateIp;
+  }
+}
diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index 0bf0843..0fe811e 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -8,7 +8,14 @@ import {
   aws_cloudfront as cloudfront,
   aws_cloudfront_origins as origins,
 } from "aws-cdk-lib";
-import { Aws, Duration, RemovalPolicy, Stack, StackProps, Tags } from "aws-cdk-lib";
+import {
+  Aws,
+  Duration,
+  RemovalPolicy,
+  Stack,
+  StackProps,
+  Tags,
+} from "aws-cdk-lib";
 import { Construct } from "constructs";
 import {
   BastionHost,
@@ -22,6 +29,7 @@ import {
 import { DomainName } from "@aws-cdk/aws-apigatewayv2-alpha";
 import { readFileSync } from "fs";
 import { load } from "js-yaml";
+import { PgBouncer } from "./PgBouncer";
 
 export class PgStacInfra extends Stack {
   constructor(scope: Construct, id: string, props: Props) {
@@ -40,7 +48,7 @@ export class PgStacInfra extends Stack {
       titilerBucketsPath,
     } = props;
 
-    const maapLoggingBucket = new s3.Bucket(this, 'maapLoggingBucket', {
+    const maapLoggingBucket = new s3.Bucket(this, "maapLoggingBucket", {
       accessControl: s3.BucketAccessControl.LOG_DELIVERY_WRITE,
       removalPolicy: RemovalPolicy.DESTROY,
       blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
@@ -175,6 +183,38 @@ export class PgStacInfra extends Stack {
       titilerPgstacApi.titilerPgstacLambdaFunction.addToRolePolicy(permission);
     });
 
+    const titilerLambdaSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(
+      this,
+      "TitilerLambdaSG",
+      titilerPgstacApi.titilerPgstacLambdaFunction.connections.securityGroups[0]
+        .securityGroupId,
+    );
+
+    const pgBouncer = new PgBouncer(this, "PgBouncer", {
+      vpc: props.vpc,
+      database: {
+        connections: db.connections,
+        secret: pgstacSecret,
+      },
+      clientSecurityGroups: [titilerLambdaSecurityGroup],
+      usePublicSubnet: props.dbSubnetPublic,
+      pgBouncerConfig: {
+        poolMode: "transaction",
+        maxClientConn: 1000,
+        defaultPoolSize: 20,
+        minPoolSize: 10,
+        reservePoolSize: 5,
+        reservePoolTimeout: 5,
+        maxDbConnections: 50,
+        maxUserConnections: 50,
+      },
+    });
+
+    titilerPgstacApi.titilerPgstacLambdaFunction.addEnvironment(
+      "PGBOUNCER_HOST",
+      pgBouncer.endpoint,
+    );
+
     new BastionHost(this, "bastion-host", {
       vpc,
       db,
@@ -219,7 +259,7 @@ export class PgStacInfra extends Stack {
     });
 
     // STAC Browser Infrastructure
-    const stacBrowserBucket = new s3.Bucket(this, 'stacBrowserBucket', {
+    const stacBrowserBucket = new s3.Bucket(this, "stacBrowserBucket", {
       accessControl: s3.BucketAccessControl.PRIVATE,
       removalPolicy: RemovalPolicy.DESTROY,
       blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
@@ -227,24 +267,29 @@ export class PgStacInfra extends Stack {
       enforceSSL: true,
     });
 
-    const stacBrowserOrigin = new cloudfront.Distribution(this, 'stacBrowserDistro', {
-      defaultBehavior: { origin: new origins.S3Origin(stacBrowserBucket) },
-      defaultRootObject: 'index.html',
-      domainNames: [props.stacBrowserCustomDomainName],
-      certificate: acm.Certificate.fromCertificateArn(
-        this,
-        "stacBrowserCustomDomainNameCertificate",
-        props.stacBrowserCertificateArn,
-      ),
-      enableLogging: true,
-      logBucket: maapLoggingBucket,
-      logFilePrefix: 'stac-browser',
-    });
-    
+    const stacBrowserOrigin = new cloudfront.Distribution(
+      this,
+      "stacBrowserDistro",
+      {
+        defaultBehavior: { origin: new origins.S3Origin(stacBrowserBucket) },
+        defaultRootObject: "index.html",
+        domainNames: [props.stacBrowserCustomDomainName],
+        certificate: acm.Certificate.fromCertificateArn(
+          this,
+          "stacBrowserCustomDomainNameCertificate",
+          props.stacBrowserCertificateArn,
+        ),
+        enableLogging: true,
+        logBucket: maapLoggingBucket,
+        logFilePrefix: "stac-browser",
+      },
+    );
+
     new StacBrowser(this, "stac-browser", {
       bucketArn: stacBrowserBucket.bucketArn,
-      stacCatalogUrl: props.stacApiCustomDomainName.startsWith('https://') ? 
-        props.stacApiCustomDomainName : `https://${props.stacApiCustomDomainName}/`,
+      stacCatalogUrl: props.stacApiCustomDomainName.startsWith("https://")
+        ? props.stacApiCustomDomainName
+        : `https://${props.stacApiCustomDomainName}/`,
       githubRepoTag: props.stacBrowserRepoTag,
       websiteIndexDocument: "index.html",
     });
@@ -252,31 +297,35 @@ export class PgStacInfra extends Stack {
     const accountId = Aws.ACCOUNT_ID;
     const distributionArn = `arn:aws:cloudfront::${accountId}:distribution/${stacBrowserOrigin.distributionId}`;
 
-    stacBrowserBucket.addToResourcePolicy(new iam.PolicyStatement({
-      sid: 'AllowCloudFrontServicePrincipal',
-      effect: iam.Effect.ALLOW, 
-      actions: ['s3:GetObject'],
-      principals: [new iam.ServicePrincipal('cloudfront.amazonaws.com')],
-      resources: [stacBrowserBucket.arnForObjects('*')],
-      conditions: {
-        'StringEquals': {
-          'aws:SourceArn': distributionArn,
-        }
-      }
-    }));
-
-    maapLoggingBucket.addToResourcePolicy(new iam.PolicyStatement({
-      sid: 'AllowCloudFrontServicePrincipal',
-      effect: iam.Effect.ALLOW,
-      actions: ['s3:PutObject'],
-      resources: [maapLoggingBucket.arnForObjects('AWSLogs/*')],
-      principals: [new iam.ServicePrincipal('cloudfront.amazonaws.com')],
-      conditions: {
-        'StringEquals': {
-          'aws:SourceArn': distributionArn, 
-        }, 
-      }, 
-    }));
+    stacBrowserBucket.addToResourcePolicy(
+      new iam.PolicyStatement({
+        sid: "AllowCloudFrontServicePrincipal",
+        effect: iam.Effect.ALLOW,
+        actions: ["s3:GetObject"],
+        principals: [new iam.ServicePrincipal("cloudfront.amazonaws.com")],
+        resources: [stacBrowserBucket.arnForObjects("*")],
+        conditions: {
+          StringEquals: {
+            "aws:SourceArn": distributionArn,
+          },
+        },
+      }),
+    );
+
+    maapLoggingBucket.addToResourcePolicy(
+      new iam.PolicyStatement({
+        sid: "AllowCloudFrontServicePrincipal",
+        effect: iam.Effect.ALLOW,
+        actions: ["s3:PutObject"],
+        resources: [maapLoggingBucket.arnForObjects("AWSLogs/*")],
+        principals: [new iam.ServicePrincipal("cloudfront.amazonaws.com")],
+        conditions: {
+          StringEquals: {
+            "aws:SourceArn": distributionArn,
+          },
+        },
+      }),
+    );
   }
 }
 
diff --git a/cdk/handlers/raster_handler.py b/cdk/handlers/raster_handler.py
index c03737a..aab126f 100644
--- a/cdk/handlers/raster_handler.py
+++ b/cdk/handlers/raster_handler.py
@@ -8,11 +8,12 @@
 
 # Update postgres env variables before importing titiler.pgstac.main
 pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"]
+pgbouncer_host = os.getenv("PGBOUNCER_HOST")
 
 secret = get_secret_dict(pgstac_secret_arn)
 os.environ.update(
     {
-        "postgres_host": secret["host"],
+        "postgres_host": pgbouncer_host or secret["host"],
         "postgres_dbname": secret["dbname"],
         "postgres_user": secret["username"],
         "postgres_pass": secret["password"],
@@ -20,11 +21,10 @@
     }
 )
 
+from eoapi.raster.main import app
 from mangum import Mangum
 from titiler.pgstac.db import connect_to_db
 
-from eoapi.raster.main import app
-
 logging.getLogger("mangum.lifespan").setLevel(logging.ERROR)
 logging.getLogger("mangum.http").setLevel(logging.ERROR)
 

From 6d561ceb84bc78249e3bae5f44f54cbf84c5c8ec Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Thu, 5 Dec 2024 09:42:52 -0600
Subject: [PATCH 02/28] switch to a ubuntu image

---
 cdk/PgBouncer.ts | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 2e02716..41715f3 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -5,6 +5,7 @@ import {
   Stack,
 } from "aws-cdk-lib";
 import { Construct } from "constructs";
+import { GenericLinuxImage } from "aws-cdk-lib/aws-ec2";
 
 export interface PgBouncerProps {
   /**
@@ -67,7 +68,7 @@ export class PgBouncer extends Construct {
       usePublicSubnet = false,
       instanceType = ec2.InstanceType.of(
         ec2.InstanceClass.T3,
-        ec2.InstanceSize.SMALL,
+        ec2.InstanceSize.NANO,
       ),
       pgBouncerConfig = {
         poolMode: "transaction",
@@ -131,9 +132,10 @@ export class PgBouncer extends Construct {
           : ec2.SubnetType.PRIVATE_WITH_EGRESS,
       },
       instanceType,
-      machineImage: new ec2.AmazonLinuxImage({
-        generation: ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
-      }),
+      machineImage: ec2.MachineImage.fromSsmParameter(
+        "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
+        { os: ec2.OperatingSystemType.LINUX },
+      ),
       securityGroup: this.securityGroup,
       role,
     });
@@ -141,8 +143,8 @@ export class PgBouncer extends Construct {
     // Create user data script
     const userDataScript = ec2.UserData.forLinux();
     userDataScript.addCommands(
-      "yum update -y",
-      "yum install -y pgbouncer jq aws-cli",
+      "apt-get update",
+      "apt-get install -y pgbouncer jq awscli",
       // Create configuration update script
       `cat <<EOF > /usr/local/bin/update-pgbouncer-config.sh
 #!/bin/bash

From 108e6f62d4078a9fca4dfbb56ebb8fadd1257fd6 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Thu, 5 Dec 2024 11:45:35 -0600
Subject: [PATCH 03/28] try new approach for startup script

---
 cdk/PgBouncer.ts | 107 ++++++++++++++++++++++++++---------------------
 1 file changed, 60 insertions(+), 47 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 41715f3..eb73758 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -132,6 +132,7 @@ export class PgBouncer extends Construct {
           : ec2.SubnetType.PRIVATE_WITH_EGRESS,
       },
       instanceType,
+      instanceName: `pgbouncer-${Date.now()}`,
       machineImage: ec2.MachineImage.fromSsmParameter(
         "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
         { os: ec2.OperatingSystemType.LINUX },
@@ -143,56 +144,68 @@ export class PgBouncer extends Construct {
     // Create user data script
     const userDataScript = ec2.UserData.forLinux();
     userDataScript.addCommands(
+      "set -euxo pipefail", // Add error handling and debugging
+
+      // Install required packages
       "apt-get update",
-      "apt-get install -y pgbouncer jq awscli",
-      // Create configuration update script
-      `cat <<EOF > /usr/local/bin/update-pgbouncer-config.sh
-#!/bin/bash
-SECRET_ARN="${database.secret.secretArn}"
-REGION="${Stack.of(this).region}"
-
-# Fetch secret
-SECRET=\$(aws secretsmanager get-secret-value --secret-id \$SECRET_ARN --region \$REGION --query SecretString --output text)
-
-# Parse values
-DB_HOST=\$(echo \$SECRET | jq -r '.host')
-DB_PORT=\$(echo \$SECRET | jq -r '.port')
-DB_NAME=\$(echo \$SECRET | jq -r '.dbname')
-DB_USER=\$(echo \$SECRET | jq -r '.username')
-DB_PASSWORD=\$(echo \$SECRET | jq -r '.password')
-
-# Create pgbouncer config
-cat <<EOC > /etc/pgbouncer/pgbouncer.ini
-[databases]
-* = host=\$DB_HOST port=\$DB_PORT dbname=\$DB_NAME
-
-[pgbouncer]
-listen_addr = *
-listen_port = 5432
-auth_type = md5
-auth_file = /etc/pgbouncer/userlist.txt
-pool_mode = ${pgBouncerConfig.poolMode}
-max_client_conn = ${pgBouncerConfig.maxClientConn}
-default_pool_size = ${pgBouncerConfig.defaultPoolSize}
-min_pool_size = ${pgBouncerConfig.minPoolSize}
-reserve_pool_size = ${pgBouncerConfig.reservePoolSize}
-reserve_pool_timeout = ${pgBouncerConfig.reservePoolTimeout}
-max_db_connections = ${pgBouncerConfig.maxDbConnections}
-max_user_connections = ${pgBouncerConfig.maxUserConnections}
-EOC
-
-# Create auth file
-echo "\\"$DB_USER\\" \\"$DB_PASSWORD\\"" > /etc/pgbouncer/userlist.txt
-
-# Set permissions
-chown pgbouncer:pgbouncer /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
-chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
-
-# Restart pgbouncer
-systemctl restart pgbouncer
-EOF`,
+      "DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli",
+
+      // Create update script
+      "cat <<'EOF' > /usr/local/bin/update-pgbouncer-config.sh",
+      "#!/bin/bash",
+      "set -euxo pipefail",
+
+      `SECRET_ARN=${database.secret.secretArn}`,
+      `REGION=${Stack.of(this).region}`,
+
+      "echo 'Fetching secret from ARN: ' ${SECRET_ARN}",
+      "SECRET=$(aws secretsmanager get-secret-value --secret-id $SECRET_ARN --region $REGION --query SecretString --output text)",
+
+      "# Parse database credentials",
+      "DB_HOST=$(echo $SECRET | jq -r '.host')",
+      "DB_PORT=$(echo $SECRET | jq -r '.port')",
+      "DB_NAME=$(echo $SECRET | jq -r '.dbname')",
+      "DB_USER=$(echo $SECRET | jq -r '.username')",
+      "DB_PASSWORD=$(echo $SECRET | jq -r '.password')",
+
+      "echo 'Creating PgBouncer configuration...'",
+
+      "# Create pgbouncer.ini",
+      "cat <<EOC > /etc/pgbouncer/pgbouncer.ini",
+      "[databases]",
+      "* = host=$DB_HOST port=$DB_PORT dbname=$DB_NAME",
+      "",
+      "[pgbouncer]",
+      "listen_addr = '*'",
+      "listen_port = 5432",
+      "auth_type = md5",
+      "auth_file = /etc/pgbouncer/userlist.txt",
+      `pool_mode = ${pgBouncerConfig.poolMode}`,
+      `max_client_conn = ${pgBouncerConfig.maxClientConn}`,
+      `default_pool_size = ${pgBouncerConfig.defaultPoolSize}`,
+      `min_pool_size = ${pgBouncerConfig.minPoolSize}`,
+      `reserve_pool_size = ${pgBouncerConfig.reservePoolSize}`,
+      `reserve_pool_timeout = ${pgBouncerConfig.reservePoolTimeout}`,
+      `max_db_connections = ${pgBouncerConfig.maxDbConnections}`,
+      `max_user_connections = ${pgBouncerConfig.maxUserConnections}`,
+      "EOC",
+
+      "# Create userlist.txt",
+      'echo "\\"$DB_USER\\" \\"$DB_PASSWORD\\"" > /etc/pgbouncer/userlist.txt',
+
+      "# Set correct permissions",
+      "chown pgbouncer:pgbouncer /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
+      "chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
+
+      "# Restart pgbouncer",
+      "systemctl restart pgbouncer",
+      "EOF",
+
+      // Make script executable and run it
       "chmod +x /usr/local/bin/update-pgbouncer-config.sh",
       "/usr/local/bin/update-pgbouncer-config.sh",
+
+      // Enable and start pgbouncer service
       "systemctl enable pgbouncer",
       "systemctl start pgbouncer",
     );

From 7282e610fbc9f0da6a1cb24ad3b45a75ac6fe611 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Thu, 5 Dec 2024 12:37:55 -0600
Subject: [PATCH 04/28] do not write separate script, just execute it

---
 cdk/PgBouncer.ts | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index eb73758..c4c3485 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -151,7 +151,6 @@ export class PgBouncer extends Construct {
       "DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli",
 
       // Create update script
-      "cat <<'EOF' > /usr/local/bin/update-pgbouncer-config.sh",
       "#!/bin/bash",
       "set -euxo pipefail",
 
@@ -199,11 +198,6 @@ export class PgBouncer extends Construct {
 
       "# Restart pgbouncer",
       "systemctl restart pgbouncer",
-      "EOF",
-
-      // Make script executable and run it
-      "chmod +x /usr/local/bin/update-pgbouncer-config.sh",
-      "/usr/local/bin/update-pgbouncer-config.sh",
 
       // Enable and start pgbouncer service
       "systemctl enable pgbouncer",

From a6d4293357fa60b09fd7573e4d40cdda849b9aca Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Thu, 5 Dec 2024 14:47:30 -0600
Subject: [PATCH 05/28] fix pgbouncer.ini

---
 cdk/PgBouncer.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index c4c3485..74e912a 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -175,7 +175,7 @@ export class PgBouncer extends Construct {
       "* = host=$DB_HOST port=$DB_PORT dbname=$DB_NAME",
       "",
       "[pgbouncer]",
-      "listen_addr = '*'",
+      "listen_addr = 0.0.0.0",
       "listen_port = 5432",
       "auth_type = md5",
       "auth_file = /etc/pgbouncer/userlist.txt",
@@ -187,6 +187,7 @@ export class PgBouncer extends Construct {
       `reserve_pool_timeout = ${pgBouncerConfig.reservePoolTimeout}`,
       `max_db_connections = ${pgBouncerConfig.maxDbConnections}`,
       `max_user_connections = ${pgBouncerConfig.maxUserConnections}`,
+      "ignore_startup_parameters = application_name,search_path",
       "EOC",
 
       "# Create userlist.txt",

From 1cb1e6737c946fe8514a1c171e6442d5bfbf5e04 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Thu, 5 Dec 2024 15:13:24 -0600
Subject: [PATCH 06/28] fix permissions

---
 cdk/PgBouncer.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 74e912a..be8ee18 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -194,7 +194,7 @@ export class PgBouncer extends Construct {
       'echo "\\"$DB_USER\\" \\"$DB_PASSWORD\\"" > /etc/pgbouncer/userlist.txt',
 
       "# Set correct permissions",
-      "chown pgbouncer:pgbouncer /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
+      "chown postgres:postgres /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
       "chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
 
       "# Restart pgbouncer",

From a7def34f3995fc4632deefbdac00aba1ac263336 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Thu, 5 Dec 2024 20:14:11 -0600
Subject: [PATCH 07/28] set up cloudwatch and healthcheck for pgbouncer

---
 cdk/PgBouncer.ts | 100 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 96 insertions(+), 4 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index be8ee18..338ad59 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -112,6 +112,9 @@ export class PgBouncer extends Construct {
         iam.ManagedPolicy.fromAwsManagedPolicyName(
           "AmazonSSMManagedInstanceCore",
         ),
+        iam.ManagedPolicy.fromAwsManagedPolicyName(
+          "CloudWatchAgentServerPolicy",
+        ),
       ],
     });
 
@@ -139,6 +142,17 @@ export class PgBouncer extends Construct {
       ),
       securityGroup: this.securityGroup,
       role,
+      detailedMonitoring: true, // Enable detailed CloudWatch monitoring
+      blockDevices: [
+        {
+          deviceName: "/dev/xvda",
+          volume: ec2.BlockDeviceVolume.ebs(20, {
+            volumeType: ec2.EbsDeviceVolumeType.GP3,
+            encrypted: true,
+            deleteOnTermination: true,
+          }),
+        },
+      ],
     });
 
     // Create user data script
@@ -146,14 +160,14 @@ export class PgBouncer extends Construct {
     userDataScript.addCommands(
       "set -euxo pipefail", // Add error handling and debugging
 
+      // add the postgres repository
+      "curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -",
+      "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'",
+
       // Install required packages
       "apt-get update",
       "DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli",
 
-      // Create update script
-      "#!/bin/bash",
-      "set -euxo pipefail",
-
       `SECRET_ARN=${database.secret.secretArn}`,
       `REGION=${Stack.of(this).region}`,
 
@@ -203,6 +217,84 @@ export class PgBouncer extends Construct {
       // Enable and start pgbouncer service
       "systemctl enable pgbouncer",
       "systemctl start pgbouncer",
+
+      // Health check
+      "# Create health check script",
+      "cat <<EOC > /usr/local/bin/check-pgbouncer.sh",
+      "#!/bin/bash",
+      "if ! pgrep pgbouncer > /dev/null; then",
+      "    systemctl start pgbouncer",
+      "    echo 'PgBouncer was down, restarted'",
+      "fi",
+      "EOC",
+      "chmod +x /usr/local/bin/check-pgbouncer.sh",
+
+      "# Add to crontab",
+      "(crontab -l 2>/dev/null; echo '* * * * * /usr/local/bin/check-pgbouncer.sh') | crontab -",
+
+      // CloudWatch
+      "# Install CloudWatch agent",
+      "wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb",
+      "dpkg -i amazon-cloudwatch-agent.deb",
+
+      "# Create CloudWatch agent configuration",
+      "cat <<EOC > /opt/aws/amazon-cloudwatch-agent/bin/config.json",
+      "{",
+      '  "agent": {',
+      '    "metrics_collection_interval": 60,',
+      '    "run_as_user": "root"',
+      "  },",
+      '  "logs": {',
+      '    "logs_collected": {',
+      '      "files": {',
+      '        "collect_list": [',
+      "          {",
+      '            "file_path": "/var/log/pgbouncer/pgbouncer.log",',
+      '            "log_group_name": "/pgbouncer/logs",',
+      '            "log_stream_name": "{instance_id}",',
+      '            "timestamp_format": "%Y-%m-%d %H:%M:%S"',
+      "          }",
+      "        ]",
+      "      }",
+      "    }",
+      "  },",
+      '  "metrics": {',
+      '    "metrics_collected": {',
+      '      "procstat": [',
+      "        {",
+      '          "pattern": "pgbouncer",',
+      '          "measurement": [',
+      '            "cpu_usage",',
+      '            "memory_rss",',
+      '            "read_bytes",',
+      '            "write_bytes"',
+      "          ]",
+      "        }",
+      "      ]",
+      "    }",
+      "  }",
+      "}",
+      "EOC",
+
+      "# Start CloudWatch agent",
+      "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json",
+      "systemctl enable amazon-cloudwatch-agent",
+      "systemctl start amazon-cloudwatch-agent",
+
+      // PgBouncer metrics
+      "# Create PgBouncer metrics script",
+      "cat <<EOC > /usr/local/bin/pgbouncer-metrics.sh",
+      "#!/bin/bash",
+      "PGPASSWORD=$DB_PASSWORD psql -h localhost -p 5432 -U $DB_USER pgbouncer -c 'SHOW POOLS;' | \\",
+      'awk \'NR>2 {print "pgbouncer_pool,database=" $1 " cl_active=" $3 ",cl_waiting=" $4 ",sv_active=" $5 ",sv_idle=" $6 ",sv_used=" $7 ",sv_tested=" $8 ",sv_login=" $9 ",maxwait=" $10}\' | \\',
+      "while IFS= read -r line; do",
+      '    aws cloudwatch put-metric-data --namespace PgBouncer --metric-name "$line" --region $REGION',
+      "done",
+      "EOC",
+      "chmod +x /usr/local/bin/pgbouncer-metrics.sh",
+
+      "# Add to crontab",
+      "(crontab -l 2>/dev/null; echo '* * * * * /usr/local/bin/pgbouncer-metrics.sh') | crontab -",
     );
 
     this.instance.addUserData(userDataScript.render());

From 7f2f4f62d7b1fd72d15bbfd24113c6b02cabe31c Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Fri, 6 Dec 2024 06:28:01 -0600
Subject: [PATCH 08/28] switch to session pooling mode

---
 cdk/PgStacInfra.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index 0fe811e..1b78567 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -199,7 +199,7 @@ export class PgStacInfra extends Stack {
       clientSecurityGroups: [titilerLambdaSecurityGroup],
       usePublicSubnet: props.dbSubnetPublic,
       pgBouncerConfig: {
-        poolMode: "transaction",
+        poolMode: "session",
         maxClientConn: 1000,
         defaultPoolSize: 20,
         minPoolSize: 10,

From b2a5919a0d697e589534dc95bd8110f6f65369b1 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Fri, 6 Dec 2024 07:00:42 -0600
Subject: [PATCH 09/28] use hash to force instance replacement

---
 cdk/PgBouncer.ts   | 93 ++++++++++++++++++++++++++++++----------------
 cdk/PgStacInfra.ts |  3 +-
 2 files changed, 64 insertions(+), 32 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 338ad59..5fb58eb 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -2,12 +2,18 @@ import {
   aws_ec2 as ec2,
   aws_iam as iam,
   aws_secretsmanager as secretsmanager,
+  CfnResource,
   Stack,
 } from "aws-cdk-lib";
+import * as crypto from "crypto";
 import { Construct } from "constructs";
-import { GenericLinuxImage } from "aws-cdk-lib/aws-ec2";
 
 export interface PgBouncerProps {
+  /**
+   * Name for the pgbouncer instance
+   */
+  instanceName: string;
+
   /**
    * VPC to deploy PgBouncer into
    */
@@ -58,6 +64,18 @@ export class PgBouncer extends Construct {
   public readonly instance: ec2.Instance;
   public readonly endpoint: string;
 
+  private calculateUserDataHash(userDataScript: ec2.UserData): string {
+    // Get the rendered user data script
+    const userData = userDataScript.render();
+
+    // Calculate hash of the user data
+    return crypto
+      .createHash("sha256")
+      .update(userData)
+      .digest("hex")
+      .substring(0, 8); // Use first 8 characters of hash
+  }
+
   constructor(scope: Construct, id: string, props: PgBouncerProps) {
     super(scope, id);
 
@@ -126,35 +144,6 @@ export class PgBouncer extends Construct {
       }),
     );
 
-    // Create PgBouncer instance
-    this.instance = new ec2.Instance(this, "Instance", {
-      vpc,
-      vpcSubnets: {
-        subnetType: usePublicSubnet
-          ? ec2.SubnetType.PUBLIC
-          : ec2.SubnetType.PRIVATE_WITH_EGRESS,
-      },
-      instanceType,
-      instanceName: `pgbouncer-${Date.now()}`,
-      machineImage: ec2.MachineImage.fromSsmParameter(
-        "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
-        { os: ec2.OperatingSystemType.LINUX },
-      ),
-      securityGroup: this.securityGroup,
-      role,
-      detailedMonitoring: true, // Enable detailed CloudWatch monitoring
-      blockDevices: [
-        {
-          deviceName: "/dev/xvda",
-          volume: ec2.BlockDeviceVolume.ebs(20, {
-            volumeType: ec2.EbsDeviceVolumeType.GP3,
-            encrypted: true,
-            deleteOnTermination: true,
-          }),
-        },
-      ],
-    });
-
     // Create user data script
     const userDataScript = ec2.UserData.forLinux();
     userDataScript.addCommands(
@@ -297,7 +286,49 @@ export class PgBouncer extends Construct {
       "(crontab -l 2>/dev/null; echo '* * * * * /usr/local/bin/pgbouncer-metrics.sh') | crontab -",
     );
 
-    this.instance.addUserData(userDataScript.render());
+    const userDataHash = this.calculateUserDataHash(userDataScript);
+
+    // Create PgBouncer instance
+    this.instance = new ec2.Instance(this, "Instance", {
+      vpc,
+      vpcSubnets: {
+        subnetType: usePublicSubnet
+          ? ec2.SubnetType.PUBLIC
+          : ec2.SubnetType.PRIVATE_WITH_EGRESS,
+      },
+      instanceType,
+      instanceName: `pgbouncer-${Date.now()}`,
+      machineImage: ec2.MachineImage.fromSsmParameter(
+        "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
+        { os: ec2.OperatingSystemType.LINUX },
+      ),
+      securityGroup: this.securityGroup,
+      role,
+      detailedMonitoring: true, // Enable detailed CloudWatch monitoring
+      blockDevices: [
+        {
+          deviceName: "/dev/xvda",
+          volume: ec2.BlockDeviceVolume.ebs(20, {
+            volumeType: ec2.EbsDeviceVolumeType.GP3,
+            encrypted: true,
+            deleteOnTermination: true,
+          }),
+        },
+      ],
+      userData: userDataScript,
+    });
+
+    const replacementTrigger = new CfnResource(
+      this,
+      "InstanceReplacementTrigger",
+      {
+        type: "Custom::InstanceReplacementTrigger",
+        properties: {
+          UserDataHash: userDataHash,
+        },
+      },
+    );
+    this.instance.node.addDependency(replacementTrigger);
 
     // Set the endpoint
     this.endpoint = this.instance.instancePrivateIp;
diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index 1b78567..b4548a5 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -190,7 +190,8 @@ export class PgStacInfra extends Stack {
         .securityGroupId,
     );
 
-    const pgBouncer = new PgBouncer(this, "PgBouncer", {
+    const pgBouncer = new PgBouncer(this, "pgbouncer", {
+      instanceName: `pgbouncer-${stage}`,
       vpc: props.vpc,
       database: {
         connections: db.connections,

From 2f1dff3dc53e046f2790002009783b422157c180 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Fri, 6 Dec 2024 09:30:14 -0600
Subject: [PATCH 10/28] run init script on each boot

---
 cdk/PgBouncer.ts | 47 ++++++++++++++++++++---------------------------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 5fb58eb..c816da5 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -2,10 +2,8 @@ import {
   aws_ec2 as ec2,
   aws_iam as iam,
   aws_secretsmanager as secretsmanager,
-  CfnResource,
   Stack,
 } from "aws-cdk-lib";
-import * as crypto from "crypto";
 import { Construct } from "constructs";
 
 export interface PgBouncerProps {
@@ -64,18 +62,6 @@ export class PgBouncer extends Construct {
   public readonly instance: ec2.Instance;
   public readonly endpoint: string;
 
-  private calculateUserDataHash(userDataScript: ec2.UserData): string {
-    // Get the rendered user data script
-    const userData = userDataScript.render();
-
-    // Calculate hash of the user data
-    return crypto
-      .createHash("sha256")
-      .update(userData)
-      .digest("hex")
-      .substring(0, 8); // Use first 8 characters of hash
-  }
-
   constructor(scope: Construct, id: string, props: PgBouncerProps) {
     super(scope, id);
 
@@ -155,6 +141,7 @@ export class PgBouncer extends Construct {
 
       // Install required packages
       "apt-get update",
+      "sleep 5",
       "DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli",
 
       `SECRET_ARN=${database.secret.secretArn}`,
@@ -286,7 +273,24 @@ export class PgBouncer extends Construct {
       "(crontab -l 2>/dev/null; echo '* * * * * /usr/local/bin/pgbouncer-metrics.sh') | crontab -",
     );
 
-    const userDataHash = this.calculateUserDataHash(userDataScript);
+    // ensure the init script gets run on every boot
+    userDataScript.addCommands(
+      // Create a per-boot script
+      "mkdir -p /var/lib/cloud/scripts/per-boot",
+
+      "cat <<'EOF' > /var/lib/cloud/scripts/per-boot/00-run-config.sh",
+      "#!/bin/bash",
+      "# Stop existing services",
+      "systemctl stop pgbouncer",
+
+      "# Re-run configuration",
+      "curl -o /tmp/user-data http://169.254.169.254/latest/user-data",
+      "chmod +x /tmp/user-data",
+      "/tmp/user-data",
+      "EOF",
+
+      "chmod +x /var/lib/cloud/scripts/per-boot/00-run-config.sh",
+    );
 
     // Create PgBouncer instance
     this.instance = new ec2.Instance(this, "Instance", {
@@ -316,20 +320,9 @@ export class PgBouncer extends Construct {
         },
       ],
       userData: userDataScript,
+      userDataCausesReplacement: true,
     });
 
-    const replacementTrigger = new CfnResource(
-      this,
-      "InstanceReplacementTrigger",
-      {
-        type: "Custom::InstanceReplacementTrigger",
-        properties: {
-          UserDataHash: userDataHash,
-        },
-      },
-    );
-    this.instance.node.addDependency(replacementTrigger);
-
     // Set the endpoint
     this.endpoint = this.instance.instancePrivateIp;
   }

From 84e4c76537dd032453fc75efa2bebc3879a3cc3c Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Fri, 6 Dec 2024 10:48:40 -0600
Subject: [PATCH 11/28] switch it back to transaction mode

---
 cdk/PgBouncer.ts   | 5 +----
 cdk/PgStacInfra.ts | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index c816da5..d0a8901 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -187,12 +187,9 @@ export class PgBouncer extends Construct {
       "chown postgres:postgres /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
       "chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
 
-      "# Restart pgbouncer",
-      "systemctl restart pgbouncer",
-
       // Enable and start pgbouncer service
       "systemctl enable pgbouncer",
-      "systemctl start pgbouncer",
+      "systemctl restart pgbouncer",
 
       // Health check
       "# Create health check script",
diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index b4548a5..03ac5ed 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -200,7 +200,7 @@ export class PgStacInfra extends Stack {
       clientSecurityGroups: [titilerLambdaSecurityGroup],
       usePublicSubnet: props.dbSubnetPublic,
       pgBouncerConfig: {
-        poolMode: "session",
+        poolMode: "transaction",
         maxClientConn: 1000,
         defaultPoolSize: 20,
         minPoolSize: 10,

From 8017a3831a02041328442bc7116313578837c294 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Mon, 9 Dec 2024 09:35:27 -0600
Subject: [PATCH 12/28] move startup script to external file

---
 cdk/PgBouncer.ts               | 219 +++++++---------------------
 cdk/scripts/pgbouncer-setup.sh | 252 +++++++++++++++++++++++++++++++++
 2 files changed, 300 insertions(+), 171 deletions(-)
 create mode 100644 cdk/scripts/pgbouncer-setup.sh

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index d0a8901..059e3a5 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -6,6 +6,9 @@ import {
 } from "aws-cdk-lib";
 import { Construct } from "constructs";
 
+import * as fs from "fs";
+import * as path from "path";
+
 export interface PgBouncerProps {
   /**
    * Name for the pgbouncer instance
@@ -45,15 +48,15 @@ export interface PgBouncerProps {
   /**
    * PgBouncer configuration options
    */
-  pgBouncerConfig?: {
-    poolMode?: "transaction" | "session" | "statement";
-    maxClientConn?: number;
-    defaultPoolSize?: number;
-    minPoolSize?: number;
-    reservePoolSize?: number;
-    reservePoolTimeout?: number;
-    maxDbConnections?: number;
-    maxUserConnections?: number;
+  pgBouncerConfig: {
+    poolMode: "transaction" | "session" | "statement";
+    maxClientConn: number;
+    defaultPoolSize: number;
+    minPoolSize: number;
+    reservePoolSize: number;
+    reservePoolTimeout: number;
+    maxDbConnections: number;
+    maxUserConnections: number;
   };
 }
 
@@ -130,165 +133,6 @@ export class PgBouncer extends Construct {
       }),
     );
 
-    // Create user data script
-    const userDataScript = ec2.UserData.forLinux();
-    userDataScript.addCommands(
-      "set -euxo pipefail", // Add error handling and debugging
-
-      // add the postgres repository
-      "curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -",
-      "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'",
-
-      // Install required packages
-      "apt-get update",
-      "sleep 5",
-      "DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli",
-
-      `SECRET_ARN=${database.secret.secretArn}`,
-      `REGION=${Stack.of(this).region}`,
-
-      "echo 'Fetching secret from ARN: ' ${SECRET_ARN}",
-      "SECRET=$(aws secretsmanager get-secret-value --secret-id $SECRET_ARN --region $REGION --query SecretString --output text)",
-
-      "# Parse database credentials",
-      "DB_HOST=$(echo $SECRET | jq -r '.host')",
-      "DB_PORT=$(echo $SECRET | jq -r '.port')",
-      "DB_NAME=$(echo $SECRET | jq -r '.dbname')",
-      "DB_USER=$(echo $SECRET | jq -r '.username')",
-      "DB_PASSWORD=$(echo $SECRET | jq -r '.password')",
-
-      "echo 'Creating PgBouncer configuration...'",
-
-      "# Create pgbouncer.ini",
-      "cat <<EOC > /etc/pgbouncer/pgbouncer.ini",
-      "[databases]",
-      "* = host=$DB_HOST port=$DB_PORT dbname=$DB_NAME",
-      "",
-      "[pgbouncer]",
-      "listen_addr = 0.0.0.0",
-      "listen_port = 5432",
-      "auth_type = md5",
-      "auth_file = /etc/pgbouncer/userlist.txt",
-      `pool_mode = ${pgBouncerConfig.poolMode}`,
-      `max_client_conn = ${pgBouncerConfig.maxClientConn}`,
-      `default_pool_size = ${pgBouncerConfig.defaultPoolSize}`,
-      `min_pool_size = ${pgBouncerConfig.minPoolSize}`,
-      `reserve_pool_size = ${pgBouncerConfig.reservePoolSize}`,
-      `reserve_pool_timeout = ${pgBouncerConfig.reservePoolTimeout}`,
-      `max_db_connections = ${pgBouncerConfig.maxDbConnections}`,
-      `max_user_connections = ${pgBouncerConfig.maxUserConnections}`,
-      "ignore_startup_parameters = application_name,search_path",
-      "EOC",
-
-      "# Create userlist.txt",
-      'echo "\\"$DB_USER\\" \\"$DB_PASSWORD\\"" > /etc/pgbouncer/userlist.txt',
-
-      "# Set correct permissions",
-      "chown postgres:postgres /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
-      "chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt",
-
-      // Enable and start pgbouncer service
-      "systemctl enable pgbouncer",
-      "systemctl restart pgbouncer",
-
-      // Health check
-      "# Create health check script",
-      "cat <<EOC > /usr/local/bin/check-pgbouncer.sh",
-      "#!/bin/bash",
-      "if ! pgrep pgbouncer > /dev/null; then",
-      "    systemctl start pgbouncer",
-      "    echo 'PgBouncer was down, restarted'",
-      "fi",
-      "EOC",
-      "chmod +x /usr/local/bin/check-pgbouncer.sh",
-
-      "# Add to crontab",
-      "(crontab -l 2>/dev/null; echo '* * * * * /usr/local/bin/check-pgbouncer.sh') | crontab -",
-
-      // CloudWatch
-      "# Install CloudWatch agent",
-      "wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb",
-      "dpkg -i amazon-cloudwatch-agent.deb",
-
-      "# Create CloudWatch agent configuration",
-      "cat <<EOC > /opt/aws/amazon-cloudwatch-agent/bin/config.json",
-      "{",
-      '  "agent": {',
-      '    "metrics_collection_interval": 60,',
-      '    "run_as_user": "root"',
-      "  },",
-      '  "logs": {',
-      '    "logs_collected": {',
-      '      "files": {',
-      '        "collect_list": [',
-      "          {",
-      '            "file_path": "/var/log/pgbouncer/pgbouncer.log",',
-      '            "log_group_name": "/pgbouncer/logs",',
-      '            "log_stream_name": "{instance_id}",',
-      '            "timestamp_format": "%Y-%m-%d %H:%M:%S"',
-      "          }",
-      "        ]",
-      "      }",
-      "    }",
-      "  },",
-      '  "metrics": {',
-      '    "metrics_collected": {',
-      '      "procstat": [',
-      "        {",
-      '          "pattern": "pgbouncer",',
-      '          "measurement": [',
-      '            "cpu_usage",',
-      '            "memory_rss",',
-      '            "read_bytes",',
-      '            "write_bytes"',
-      "          ]",
-      "        }",
-      "      ]",
-      "    }",
-      "  }",
-      "}",
-      "EOC",
-
-      "# Start CloudWatch agent",
-      "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json",
-      "systemctl enable amazon-cloudwatch-agent",
-      "systemctl start amazon-cloudwatch-agent",
-
-      // PgBouncer metrics
-      "# Create PgBouncer metrics script",
-      "cat <<EOC > /usr/local/bin/pgbouncer-metrics.sh",
-      "#!/bin/bash",
-      "PGPASSWORD=$DB_PASSWORD psql -h localhost -p 5432 -U $DB_USER pgbouncer -c 'SHOW POOLS;' | \\",
-      'awk \'NR>2 {print "pgbouncer_pool,database=" $1 " cl_active=" $3 ",cl_waiting=" $4 ",sv_active=" $5 ",sv_idle=" $6 ",sv_used=" $7 ",sv_tested=" $8 ",sv_login=" $9 ",maxwait=" $10}\' | \\',
-      "while IFS= read -r line; do",
-      '    aws cloudwatch put-metric-data --namespace PgBouncer --metric-name "$line" --region $REGION',
-      "done",
-      "EOC",
-      "chmod +x /usr/local/bin/pgbouncer-metrics.sh",
-
-      "# Add to crontab",
-      "(crontab -l 2>/dev/null; echo '* * * * * /usr/local/bin/pgbouncer-metrics.sh') | crontab -",
-    );
-
-    // ensure the init script gets run on every boot
-    userDataScript.addCommands(
-      // Create a per-boot script
-      "mkdir -p /var/lib/cloud/scripts/per-boot",
-
-      "cat <<'EOF' > /var/lib/cloud/scripts/per-boot/00-run-config.sh",
-      "#!/bin/bash",
-      "# Stop existing services",
-      "systemctl stop pgbouncer",
-
-      "# Re-run configuration",
-      "curl -o /tmp/user-data http://169.254.169.254/latest/user-data",
-      "chmod +x /tmp/user-data",
-      "/tmp/user-data",
-      "EOF",
-
-      "chmod +x /var/lib/cloud/scripts/per-boot/00-run-config.sh",
-    );
-
     // Create PgBouncer instance
     this.instance = new ec2.Instance(this, "Instance", {
       vpc,
@@ -298,14 +142,14 @@ export class PgBouncer extends Construct {
           : ec2.SubnetType.PRIVATE_WITH_EGRESS,
       },
       instanceType,
-      instanceName: `pgbouncer-${Date.now()}`,
+      instanceName: props.instanceName,
       machineImage: ec2.MachineImage.fromSsmParameter(
         "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
         { os: ec2.OperatingSystemType.LINUX },
       ),
       securityGroup: this.securityGroup,
       role,
-      detailedMonitoring: true, // Enable detailed CloudWatch monitoring
+      detailedMonitoring: true,
       blockDevices: [
         {
           deviceName: "/dev/xvda",
@@ -316,11 +160,44 @@ export class PgBouncer extends Construct {
           }),
         },
       ],
-      userData: userDataScript,
+      userData: this.loadUserDataScript(pgBouncerConfig, database),
       userDataCausesReplacement: true,
     });
 
     // Set the endpoint
     this.endpoint = this.instance.instancePrivateIp;
   }
+
+  private loadUserDataScript(
+    pgBouncerConfig: Required<NonNullable<PgBouncerProps["pgBouncerConfig"]>>,
+    database: { secret: secretsmanager.ISecret },
+  ): ec2.UserData {
+    const userDataScript = ec2.UserData.forLinux();
+
+    // Set environment variables with configuration parameters
+    userDataScript.addCommands(
+      'export SECRET_ARN="' + database.secret.secretArn + '"',
+      'export REGION="' + Stack.of(this).region + '"',
+      'export POOL_MODE="' + pgBouncerConfig.poolMode + '"',
+      'export MAX_CLIENT_CONN="' + pgBouncerConfig.maxClientConn + '"',
+      'export DEFAULT_POOL_SIZE="' + pgBouncerConfig.defaultPoolSize + '"',
+      'export MIN_POOL_SIZE="' + pgBouncerConfig.minPoolSize + '"',
+      'export RESERVE_POOL_SIZE="' + pgBouncerConfig.reservePoolSize + '"',
+      'export RESERVE_POOL_TIMEOUT="' +
+        pgBouncerConfig.reservePoolTimeout +
+        '"',
+      'export MAX_DB_CONNECTIONS="' + pgBouncerConfig.maxDbConnections + '"',
+      'export MAX_USER_CONNECTIONS="' +
+        pgBouncerConfig.maxUserConnections +
+        '"',
+    );
+
+    // Load the startup script
+    const scriptPath = path.join(__dirname, "./scripts/pgbouncer-setup.sh");
+    let script = fs.readFileSync(scriptPath, "utf8");
+
+    userDataScript.addCommands(script);
+
+    return userDataScript;
+  }
 }
diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
new file mode 100644
index 0000000..781375a
--- /dev/null
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -0,0 +1,252 @@
+#!/bin/bash
+set -euxo pipefail
+
+# These variables will be replaced by the TypeScript code
+SECRET_ARN=${SECRET_ARN}
+REGION=${REGION}
+POOL_MODE=${POOL_MODE}
+MAX_CLIENT_CONN=${MAX_CLIENT_CONN}
+DEFAULT_POOL_SIZE=${DEFAULT_POOL_SIZE}
+MIN_POOL_SIZE=${MIN_POOL_SIZE}
+RESERVE_POOL_SIZE=${RESERVE_POOL_SIZE}
+RESERVE_POOL_TIMEOUT=${RESERVE_POOL_TIMEOUT}
+MAX_DB_CONNECTIONS=${MAX_DB_CONNECTIONS}
+MAX_USER_CONNECTIONS=${MAX_USER_CONNECTIONS}
+
+# Add the postgres repository
+curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+
+# Install required packages
+apt-get update
+sleep 5
+DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli
+
+echo "Fetching secret from ARN: ${SECRET_ARN}"
+SECRET=$(aws secretsmanager get-secret-value --secret-id ${SECRET_ARN} --region ${REGION} --query SecretString --output text)
+
+# Before handling secrets, turn off command tracing
+set +x
+
+# Create a read-only file descriptor for sensitive operations
+exec 3<<< "$SECRET"
+
+# Parse database credentials without echoing
+DB_HOST=$(jq -r '.host' <&3)
+DB_PORT=$(jq -r '.port' <&3)
+DB_NAME=$(jq -r '.dbname' <&3)
+DB_USER=$(jq -r '.username' <&3)
+DB_PASSWORD=$(jq -r '.password' <&3)
+
+# Close the file descriptor
+exec 3<&-
+
+echo 'Creating PgBouncer configuration...'
+
+# Create pgbouncer.ini
+cat <<EOC > /etc/pgbouncer/pgbouncer.ini
+[databases]
+* = host=$DB_HOST port=$DB_PORT dbname=$DB_NAME
+
+[pgbouncer]
+listen_addr = 0.0.0.0
+listen_port = 5432
+auth_type = md5
+auth_file = /etc/pgbouncer/userlist.txt
+pool_mode = ${POOL_MODE}
+max_client_conn = ${MAX_CLIENT_CONN}
+default_pool_size = ${DEFAULT_POOL_SIZE}
+min_pool_size = ${MIN_POOL_SIZE}
+reserve_pool_size = ${RESERVE_POOL_SIZE}
+reserve_pool_timeout = ${RESERVE_POOL_TIMEOUT}
+max_db_connections = ${MAX_DB_CONNECTIONS}
+max_user_connections = ${MAX_USER_CONNECTIONS}
+ignore_startup_parameters = application_name,search_path
+logfile = /var/log/pgbouncer/pgbouncer.log
+pid_file = /var/run/pgbouncer/pgbouncer.pid
+admin_users = $DB_USER
+stats_users = $DB_USER
+log_connections = 1
+log_disconnections = 1
+log_pooler_errors = 1
+log_stats = 1
+stats_period = 60
+EOC
+
+# Create userlist.txt without echoing sensitive info
+{
+    echo "\"$DB_USER\" \"$DB_PASSWORD\""
+} > /etc/pgbouncer/userlist.txt
+
+# Turn command tracing back on
+set -x
+
+# Set correct permissions
+chown postgres:postgres /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
+chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
+
+# Enable and start pgbouncer service
+systemctl enable pgbouncer
+systemctl restart pgbouncer
+
+# Configure logging
+mkdir -p /var/log/pgbouncer /var/run/pgbouncer
+chown postgres:postgres /var/log/pgbouncer /var/run/pgbouncer
+chmod 755 /var/log/pgbouncer /var/run/pgbouncer
+
+cat <<EOC > /etc/logrotate.d/pgbouncer
+/var/log/pgbouncer/pgbouncer.log {
+    daily
+    rotate 7
+    compress
+    delaycompress
+    missingok
+    copytruncate
+    create 640 postgres postgres
+}
+EOC
+
+# Create monitoring scripts directory
+mkdir -p /opt/pgbouncer/scripts
+
+# Create the health check script
+cat <<'EOC' > /opt/pgbouncer/scripts/check-pgbouncer.sh
+#!/bin/bash
+if ! pgrep pgbouncer > /dev/null; then
+    systemctl start pgbouncer
+    echo 'PgBouncer was down, restarted' | logger -t pgbouncer-monitor
+fi
+EOC
+chmod +x /opt/pgbouncer/scripts/check-pgbouncer.sh
+
+# Create a single crontab file
+cat <<'EOC' > /opt/pgbouncer/scripts/crontab.txt
+# PgBouncer health check - run every minute
+* * * * * /opt/pgbouncer/scripts/check-pgbouncer.sh
+# PgBouncer metrics collection - run every minute
+* * * * * /opt/pgbouncer/scripts/pgbouncer-metrics.sh
+EOC
+
+# Install the crontab as the root user
+crontab /opt/pgbouncer/scripts/crontab.txt
+
+# Verify the crontab was installed
+if ! crontab -l; then
+    echo 'Failed to install crontab' | logger -t pgbouncer-setup
+    exit 1
+fi
+
+# Create per-boot script directory
+mkdir -p /var/lib/cloud/scripts/per-boot
+
+# Create per-boot script
+cat <<'EOF' > /var/lib/cloud/scripts/per-boot/00-run-config.sh
+#!/bin/bash
+# Stop existing services
+systemctl stop pgbouncer
+
+# Re-run configuration
+curl -o /tmp/user-data http://169.254.169.254/latest/user-data
+chmod +x /tmp/user-data
+/tmp/user-data
+EOF
+
+chmod +x /var/lib/cloud/scripts/per-boot/00-run-config.sh
+
+# Create CloudWatch configuration directory
+mkdir -p /opt/pgbouncer/cloudwatch
+
+# Install CloudWatch agent
+if ! wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb; then
+    echo 'Failed to download CloudWatch agent' | logger -t pgbouncer-setup
+    exit 1
+fi
+
+if ! dpkg -i amazon-cloudwatch-agent.deb; then
+    echo 'Failed to install CloudWatch agent' | logger -t pgbouncer-setup
+    exit 1
+fi
+
+# Create CloudWatch config
+cat <<EOC > /opt/aws/amazon-cloudwatch-agent/bin/config.json
+{
+  "agent": {
+    "metrics_collection_interval": 60,
+    "run_as_user": "root"
+  },
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": [
+          {
+            "file_path": "/var/log/pgbouncer/pgbouncer.log",
+            "log_group_name": "/pgbouncer/logs",
+            "log_stream_name": "{instance_id}",
+            "timestamp_format": "%Y-%m-%d %H:%M:%S",
+            "multi_line_start_pattern": "{timestamp_format}",
+            "retention_in_days": 14
+          },
+          {
+            "file_path": "/var/log/syslog",
+            "log_group_name": "/pgbouncer/system-logs",
+            "log_stream_name": "{instance_id}",
+            "timestamp_format": "%b %d %H:%M:%S",
+            "retention_in_days": 14
+          }
+        ]
+      }
+    }
+  },
+  "metrics": {
+    "metrics_collected": {
+      "procstat": [
+        {
+          "pattern": "pgbouncer",
+          "measurement": [
+            "cpu_usage",
+            "memory_rss",
+            "read_bytes",
+            "write_bytes",
+            "read_count",
+            "write_count",
+            "open_fd"
+          ]
+        }
+      ],
+      "mem": {
+        "measurement": [
+          "mem_used_percent"
+        ]
+      },
+      "disk": {
+        "measurement": [
+          "used_percent"
+        ]
+      }
+    },
+    "aggregation_dimensions": [["InstanceId"]]
+  }
+}
+EOC
+
+# Verify the config file exists
+if [ ! -f /opt/pgbouncer/cloudwatch/config.json ]; then
+    echo 'CloudWatch config file not created' | logger -t pgbouncer-setup
+    exit 1
+fi
+
+# Start CloudWatch agent
+if ! /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/pgbouncer/cloudwatch/config.json; then
+    echo 'Failed to configure CloudWatch agent' | logger -t pgbouncer-setup
+    exit 1
+fi
+
+systemctl enable amazon-cloudwatch-agent
+systemctl start amazon-cloudwatch-agent
+
+# Verify CloudWatch agent is running
+if ! systemctl is-active amazon-cloudwatch-agent; then
+    echo 'CloudWatch agent failed to start' | logger -t pgbouncer-setup
+    exit 1
+fi
+

From 07e655c504793d8c940ab449d1cfd182486b4495 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Mon, 9 Dec 2024 10:01:04 -0600
Subject: [PATCH 13/28] upgrade to t3.micro

---
 cdk/PgBouncer.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 059e3a5..e123a53 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -75,7 +75,7 @@ export class PgBouncer extends Construct {
       usePublicSubnet = false,
       instanceType = ec2.InstanceType.of(
         ec2.InstanceClass.T3,
-        ec2.InstanceSize.NANO,
+        ec2.InstanceSize.MICRO,
       ),
       pgBouncerConfig = {
         poolMode: "transaction",

From d2989f0d9abb2cb649aec4a2b0fff98b6dd1b91d Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Mon, 9 Dec 2024 10:20:49 -0600
Subject: [PATCH 14/28] don't use file descriptor

---
 cdk/scripts/pgbouncer-setup.sh | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
index 781375a..acac2d2 100644
--- a/cdk/scripts/pgbouncer-setup.sh
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -23,23 +23,17 @@ sleep 5
 DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli
 
 echo "Fetching secret from ARN: ${SECRET_ARN}"
-SECRET=$(aws secretsmanager get-secret-value --secret-id ${SECRET_ARN} --region ${REGION} --query SecretString --output text)
 
 # Before handling secrets, turn off command tracing
 set +x
-
-# Create a read-only file descriptor for sensitive operations
-exec 3<<< "$SECRET"
+SECRET=$(aws secretsmanager get-secret-value --secret-id ${SECRET_ARN} --region ${REGION} --query SecretString --output text)
 
 # Parse database credentials without echoing
-DB_HOST=$(jq -r '.host' <&3)
-DB_PORT=$(jq -r '.port' <&3)
-DB_NAME=$(jq -r '.dbname' <&3)
-DB_USER=$(jq -r '.username' <&3)
-DB_PASSWORD=$(jq -r '.password' <&3)
-
-# Close the file descriptor
-exec 3<&-
+DB_HOST=$(echo "$SECRET" | jq -r '.host')
+DB_PORT=$(echo "$SECRET" | jq -r '.port')
+DB_NAME=$(echo "$SECRET" | jq -r '.dbname')
+DB_USER=$(echo "$SECRET" | jq -r '.username')
+DB_PASSWORD=$(echo "$SECRET" | jq -r '.password')
 
 echo 'Creating PgBouncer configuration...'
 

From d4709a0dfedbf2d971256f618f0b8f362f8493ee Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Mon, 9 Dec 2024 10:32:55 -0600
Subject: [PATCH 15/28] create log file

---
 cdk/scripts/pgbouncer-setup.sh | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
index acac2d2..92a4abe 100644
--- a/cdk/scripts/pgbouncer-setup.sh
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -57,7 +57,7 @@ max_db_connections = ${MAX_DB_CONNECTIONS}
 max_user_connections = ${MAX_USER_CONNECTIONS}
 ignore_startup_parameters = application_name,search_path
 logfile = /var/log/pgbouncer/pgbouncer.log
-pid_file = /var/run/pgbouncer/pgbouncer.pid
+pidfile = /var/run/pgbouncer/pgbouncer.pid
 admin_users = $DB_USER
 stats_users = $DB_USER
 log_connections = 1
@@ -79,15 +79,20 @@ set -x
 chown postgres:postgres /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
 chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
 
-# Enable and start pgbouncer service
-systemctl enable pgbouncer
-systemctl restart pgbouncer
-
 # Configure logging
 mkdir -p /var/log/pgbouncer /var/run/pgbouncer
 chown postgres:postgres /var/log/pgbouncer /var/run/pgbouncer
 chmod 755 /var/log/pgbouncer /var/run/pgbouncer
 
+touch /var/log/pgbouncer/pgbouncer.log
+chown postgres:postgres /var/log/pgbouncer/pgbouncer.log
+chmod 640 /var/log/pgbouncer/pgbouncer.log
+
+# Enable and start pgbouncer service
+systemctl enable pgbouncer
+systemctl restart pgbouncer
+
+
 cat <<EOC > /etc/logrotate.d/pgbouncer
 /var/log/pgbouncer/pgbouncer.log {
     daily

From 590f16791e11ba71eea18c7b8e4a80e04ab33df7 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Mon, 9 Dec 2024 15:12:00 -0600
Subject: [PATCH 16/28] fine tune pgbouncer settings

---
 cdk/PgBouncer.ts               | 10 +++++-----
 cdk/scripts/pgbouncer-setup.sh | 10 ++++++----
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index e123a53..465618c 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -79,13 +79,13 @@ export class PgBouncer extends Construct {
       ),
       pgBouncerConfig = {
         poolMode: "transaction",
-        maxClientConn: 1000,
-        defaultPoolSize: 20,
-        minPoolSize: 10,
+        maxClientConn: 200,
+        defaultPoolSize: 5,
+        minPoolSize: 0,
         reservePoolSize: 5,
         reservePoolTimeout: 5,
-        maxDbConnections: 50,
-        maxUserConnections: 50,
+        maxDbConnections: 40,
+        maxUserConnections: 40,
       },
     } = props;
 
diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
index 92a4abe..860eb7d 100644
--- a/cdk/scripts/pgbouncer-setup.sh
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -12,6 +12,7 @@ RESERVE_POOL_SIZE=${RESERVE_POOL_SIZE}
 RESERVE_POOL_TIMEOUT=${RESERVE_POOL_TIMEOUT}
 MAX_DB_CONNECTIONS=${MAX_DB_CONNECTIONS}
 MAX_USER_CONNECTIONS=${MAX_USER_CONNECTIONS}
+CLOUDWATCH_CONFIG="/opt/aws/amazon-cloudwatch-agent/bin/config.json"
 
 # Add the postgres repository
 curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
@@ -55,6 +56,7 @@ reserve_pool_size = ${RESERVE_POOL_SIZE}
 reserve_pool_timeout = ${RESERVE_POOL_TIMEOUT}
 max_db_connections = ${MAX_DB_CONNECTIONS}
 max_user_connections = ${MAX_USER_CONNECTIONS}
+max_prepared_statements = 10
 ignore_startup_parameters = application_name,search_path
 logfile = /var/log/pgbouncer/pgbouncer.log
 pidfile = /var/run/pgbouncer/pgbouncer.pid
@@ -167,7 +169,7 @@ if ! dpkg -i amazon-cloudwatch-agent.deb; then
 fi
 
 # Create CloudWatch config
-cat <<EOC > /opt/aws/amazon-cloudwatch-agent/bin/config.json
+cat <<EOC > ${CLOUDWATCH_CONFIG}
 {
   "agent": {
     "metrics_collection_interval": 60,
@@ -208,7 +210,7 @@ cat <<EOC > /opt/aws/amazon-cloudwatch-agent/bin/config.json
             "write_bytes",
             "read_count",
             "write_count",
-            "open_fd"
+            "num_fds"
           ]
         }
       ],
@@ -229,13 +231,13 @@ cat <<EOC > /opt/aws/amazon-cloudwatch-agent/bin/config.json
 EOC
 
 # Verify the config file exists
-if [ ! -f /opt/pgbouncer/cloudwatch/config.json ]; then
+if [ ! -f ${CLOUDWATCH_CONFIG} ]; then
     echo 'CloudWatch config file not created' | logger -t pgbouncer-setup
     exit 1
 fi
 
 # Start CloudWatch agent
-if ! /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/pgbouncer/cloudwatch/config.json; then
+if ! /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:${CLOUDWATCH_CONFIG}; then
     echo 'Failed to configure CloudWatch agent' | logger -t pgbouncer-setup
     exit 1
 fi

From 7b8cbffea63eb2e15cb1bfe98f2c0d0e9e56ac3f Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Mon, 9 Dec 2024 20:24:40 -0600
Subject: [PATCH 17/28] wait for dpkg lock

---
 cdk/scripts/pgbouncer-setup.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
index 860eb7d..9554521 100644
--- a/cdk/scripts/pgbouncer-setup.sh
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -20,7 +20,15 @@ sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)
 
 # Install required packages
 apt-get update
-sleep 5
+
+wait_for_dpkg_lock() {
+  while fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock >/dev/null 2>&1; do
+    echo "Waiting for dpkg lock to be released..."
+    sleep 2
+  done
+}
+
+wait_for_dpkg_lock
 DEBIAN_FRONTEND=noninteractive apt-get install -y pgbouncer jq awscli
 
 echo "Fetching secret from ARN: ${SECRET_ARN}"

From 08273d9c2ea8c5ad0cbad09d57e4079e480d763f Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Mon, 9 Dec 2024 20:49:21 -0600
Subject: [PATCH 18/28] make wget quieter

---
 cdk/scripts/pgbouncer-setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
index 9554521..90e18e8 100644
--- a/cdk/scripts/pgbouncer-setup.sh
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -166,7 +166,7 @@ chmod +x /var/lib/cloud/scripts/per-boot/00-run-config.sh
 mkdir -p /opt/pgbouncer/cloudwatch
 
 # Install CloudWatch agent
-if ! wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb; then
+if ! wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb; then
     echo 'Failed to download CloudWatch agent' | logger -t pgbouncer-setup
     exit 1
 fi

From b99dccbe5d77990d9f5e951000551e652b95d057 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Tue, 10 Dec 2024 05:37:52 -0600
Subject: [PATCH 19/28] simplify pgbouncer ingress rules

---
 cdk/PgBouncer.ts   | 15 --------------
 cdk/PgStacInfra.ts | 51 +++++++++++++++++++++++-----------------------
 2 files changed, 25 insertions(+), 41 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 465618c..6fdcde2 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -28,11 +28,6 @@ export interface PgBouncerProps {
     secret: secretsmanager.ISecret;
   };
 
-  /**
-   * Security groups that need access to PgBouncer
-   */
-  clientSecurityGroups: ec2.ISecurityGroup[];
-
   /**
    * Whether to deploy in public subnet
    * @default false
@@ -71,7 +66,6 @@ export class PgBouncer extends Construct {
     const {
       vpc,
       database,
-      clientSecurityGroups,
       usePublicSubnet = false,
       instanceType = ec2.InstanceType.of(
         ec2.InstanceClass.T3,
@@ -96,15 +90,6 @@ export class PgBouncer extends Construct {
       allowAllOutbound: true,
     });
 
-    // Allow incoming PostgreSQL traffic from client security groups
-    clientSecurityGroups.forEach((clientSg, index) => {
-      this.securityGroup.addIngressRule(
-        clientSg,
-        ec2.Port.tcp(5432),
-        `Allow PostgreSQL access from client security group ${index + 1}`,
-      );
-    });
-
     // Allow PgBouncer to connect to RDS
     database.connections.allowFrom(
       this.securityGroup,
diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index 03ac5ed..d02b624 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -81,6 +81,26 @@ export class PgStacInfra extends Stack {
           : ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.SMALL),
     });
 
+    const pgBouncer = new PgBouncer(this, "pgbouncer", {
+      instanceName: `pgbouncer-${stage}`,
+      vpc: props.vpc,
+      database: {
+        connections: db.connections,
+        secret: pgstacSecret,
+      },
+      usePublicSubnet: props.dbSubnetPublic,
+      pgBouncerConfig: {
+        poolMode: "transaction",
+        maxClientConn: 1000,
+        defaultPoolSize: 20,
+        minPoolSize: 10,
+        reservePoolSize: 5,
+        reservePoolTimeout: 5,
+        maxDbConnections: 50,
+        maxUserConnections: 50,
+      },
+    });
+
     const apiSubnetSelection: ec2.SubnetSelection = {
       subnetType: props.dbSubnetPublic
         ? ec2.SubnetType.PUBLIC
@@ -183,34 +203,13 @@ export class PgStacInfra extends Stack {
       titilerPgstacApi.titilerPgstacLambdaFunction.addToRolePolicy(permission);
     });
 
-    const titilerLambdaSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(
-      this,
-      "TitilerLambdaSG",
-      titilerPgstacApi.titilerPgstacLambdaFunction.connections.securityGroups[0]
-        .securityGroupId,
+    // Configure for pgbouncer
+    titilerPgstacApi.titilerPgstacLambdaFunction.connections.allowTo(
+      pgBouncer.instance,
+      ec2.Port.tcp(5432),
+      "allow connections from titiler",
     );
 
-    const pgBouncer = new PgBouncer(this, "pgbouncer", {
-      instanceName: `pgbouncer-${stage}`,
-      vpc: props.vpc,
-      database: {
-        connections: db.connections,
-        secret: pgstacSecret,
-      },
-      clientSecurityGroups: [titilerLambdaSecurityGroup],
-      usePublicSubnet: props.dbSubnetPublic,
-      pgBouncerConfig: {
-        poolMode: "transaction",
-        maxClientConn: 1000,
-        defaultPoolSize: 20,
-        minPoolSize: 10,
-        reservePoolSize: 5,
-        reservePoolTimeout: 5,
-        maxDbConnections: 50,
-        maxUserConnections: 50,
-      },
-    });
-
     titilerPgstacApi.titilerPgstacLambdaFunction.addEnvironment(
       "PGBOUNCER_HOST",
       pgBouncer.endpoint,

From 74d14aed0a26beb1ec24aa0c0f86b2f2839bb68a Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Tue, 10 Dec 2024 06:34:01 -0600
Subject: [PATCH 20/28] Upgrade to eoapi-cdk==7.3.0

---
 cdk/PgStacInfra.ts | 7 ++++++-
 package.json       | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index d02b624..31f8111 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -62,6 +62,7 @@ export class PgStacInfra extends Stack {
       ],
     });
 
+    // Pgstac Database
     const { db, pgstacSecret } = new PgStacDatabase(this, "pgstac-db", {
       vpc,
       allowMajorVersionUpgrade: true,
@@ -81,6 +82,7 @@ export class PgStacInfra extends Stack {
           : ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.SMALL),
     });
 
+    // PgBouncer
     const pgBouncer = new PgBouncer(this, "pgbouncer", {
       instanceName: `pgbouncer-${stage}`,
       vpc: props.vpc,
@@ -107,6 +109,7 @@ export class PgStacInfra extends Stack {
         : ec2.SubnetType.PRIVATE_WITH_EGRESS,
     };
 
+    // STAC API
     const stacApiLambda = new PgStacApiLambda(this, "pgstac-api", {
       apiEnv: {
         NAME: `MAAP STAC API (${stage})`,
@@ -135,6 +138,7 @@ export class PgStacInfra extends Stack {
       sourceArn: props.stacApiIntegrationApiArn,
     });
 
+    // titiler-pgstac
     const fileContents = readFileSync(titilerBucketsPath, "utf8");
     const buckets = load(fileContents) as string[];
 
@@ -203,7 +207,7 @@ export class PgStacInfra extends Stack {
       titilerPgstacApi.titilerPgstacLambdaFunction.addToRolePolicy(permission);
     });
 
-    // Configure for pgbouncer
+    // Configure titiler-pgstac for pgbouncer
     titilerPgstacApi.titilerPgstacLambdaFunction.connections.allowTo(
       pgBouncer.instance,
       ec2.Port.tcp(5432),
@@ -215,6 +219,7 @@ export class PgStacInfra extends Stack {
       pgBouncer.endpoint,
     );
 
+    // STAC Ingestor
     new BastionHost(this, "bastion-host", {
       vpc,
       db,
diff --git a/package.json b/package.json
index 94b8dab..99659fc 100644
--- a/package.json
+++ b/package.json
@@ -24,7 +24,7 @@
   },
   "dependencies": {
     "aws-cdk-lib": "^2.130.0",
-    "eoapi-cdk": "7.2.1",
+    "eoapi-cdk": "7.3.0",
     "constructs": "^10.3.0",
     "source-map-support": "^0.5.16"
   }

From 5d21572e89135591722053b46c97683e6a1530d9 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Tue, 10 Dec 2024 06:49:36 -0600
Subject: [PATCH 21/28] try granting permissions instead of an explicit
 security group

---
 cdk/PgBouncer.ts | 49 ++++++++++--------------------------------------
 1 file changed, 10 insertions(+), 39 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 6fdcde2..0a7c433 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -1,6 +1,5 @@
 import {
   aws_ec2 as ec2,
-  aws_iam as iam,
   aws_secretsmanager as secretsmanager,
   Stack,
 } from "aws-cdk-lib";
@@ -56,7 +55,6 @@ export interface PgBouncerProps {
 }
 
 export class PgBouncer extends Construct {
-  public readonly securityGroup: ec2.ISecurityGroup;
   public readonly instance: ec2.Instance;
   public readonly endpoint: string;
 
@@ -83,41 +81,6 @@ export class PgBouncer extends Construct {
       },
     } = props;
 
-    // Create security group for PgBouncer
-    this.securityGroup = new ec2.SecurityGroup(this, "SecurityGroup", {
-      vpc,
-      description: "Security group for PgBouncer instance",
-      allowAllOutbound: true,
-    });
-
-    // Allow PgBouncer to connect to RDS
-    database.connections.allowFrom(
-      this.securityGroup,
-      ec2.Port.tcp(5432),
-      "Allow PgBouncer to connect to RDS",
-    );
-
-    // Create role for PgBouncer instance
-    const role = new iam.Role(this, "InstanceRole", {
-      assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"),
-      managedPolicies: [
-        iam.ManagedPolicy.fromAwsManagedPolicyName(
-          "AmazonSSMManagedInstanceCore",
-        ),
-        iam.ManagedPolicy.fromAwsManagedPolicyName(
-          "CloudWatchAgentServerPolicy",
-        ),
-      ],
-    });
-
-    // Add policy to allow reading RDS credentials from Secrets Manager
-    role.addToPolicy(
-      new iam.PolicyStatement({
-        actions: ["secretsmanager:GetSecretValue"],
-        resources: [database.secret.secretArn],
-      }),
-    );
-
     // Create PgBouncer instance
     this.instance = new ec2.Instance(this, "Instance", {
       vpc,
@@ -132,8 +95,6 @@ export class PgBouncer extends Construct {
         "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
         { os: ec2.OperatingSystemType.LINUX },
       ),
-      securityGroup: this.securityGroup,
-      role,
       detailedMonitoring: true,
       blockDevices: [
         {
@@ -149,6 +110,16 @@ export class PgBouncer extends Construct {
       userDataCausesReplacement: true,
     });
 
+    // Grant read access to database secret
+    props.database.secret.grantRead(this.instance);
+
+    // Allow PgBouncer to connect to RDS
+    database.connections.allowFrom(
+      this.instance,
+      ec2.Port.tcp(5432),
+      "Allow PgBouncer to connect to RDS",
+    );
+
     // Set the endpoint
     this.endpoint = this.instance.instancePrivateIp;
   }

From 893ff504aa1ba3058ad09d37612c3da790201933 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Tue, 10 Dec 2024 12:14:29 -0600
Subject: [PATCH 22/28] Set pgbouncer maxConnections dynamically based on the
 RDS instance size

---
 cdk/PgStacInfra.ts | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index 31f8111..bd1475f 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -63,6 +63,12 @@ export class PgStacInfra extends Stack {
     });
 
     // Pgstac Database
+    // set instance type to t3.micro if stage is test, otherwise t3.small
+    const dbInstanceType =
+      stage === "test"
+        ? ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.MICRO)
+        : ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.SMALL);
+
     const { db, pgstacSecret } = new PgStacDatabase(this, "pgstac-db", {
       vpc,
       allowMajorVersionUpgrade: true,
@@ -75,14 +81,27 @@ export class PgStacInfra extends Stack {
           : ec2.SubnetType.PRIVATE_ISOLATED,
       },
       allocatedStorage: allocatedStorage,
-      // set instance type to t3.micro if stage is test, otherwise t3.small
-      instanceType:
-        stage === "test"
-          ? ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.MICRO)
-          : ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.SMALL),
+      instanceType: dbInstanceType,
     });
 
     // PgBouncer
+    // total RDS instance memory mapping
+    const instanceMemoryMapMb: Record<string, number> = {
+      "t3.micro": 1024,
+      "t3.small": 2048,
+      "t3.medium": 4096,
+    };
+
+    // Available memory is total instance memory minus some constant for OS overhead (~300?)
+    const memoryInBytes =
+      (instanceMemoryMapMb[dbInstanceType.toString()] - 300) * 1024 ** 2;
+
+    // Max connections for pgbouncer should be the instance max_connections minus 10
+    const maxPgBouncerDbConnections = Math.min(
+      Math.round(memoryInBytes / 9531392) - 10,
+      5000,
+    );
+
     const pgBouncer = new PgBouncer(this, "pgbouncer", {
       instanceName: `pgbouncer-${stage}`,
       vpc: props.vpc,
@@ -98,8 +117,8 @@ export class PgStacInfra extends Stack {
         minPoolSize: 10,
         reservePoolSize: 5,
         reservePoolTimeout: 5,
-        maxDbConnections: 50,
-        maxUserConnections: 50,
+        maxDbConnections: maxPgBouncerDbConnections,
+        maxUserConnections: maxPgBouncerDbConnections,
       },
     });
 

From 10b0c63004a25263de3b5799b0f378abc3a50dfd Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Tue, 10 Dec 2024 20:03:00 -0600
Subject: [PATCH 23/28] fix pgbouncer check, clean up setup script

---
 cdk/scripts/pgbouncer-setup.sh | 72 +++++++++++++++-------------------
 1 file changed, 32 insertions(+), 40 deletions(-)

diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
index 90e18e8..39c14ee 100644
--- a/cdk/scripts/pgbouncer-setup.sh
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -21,6 +21,7 @@ sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)
 # Install required packages
 apt-get update
 
+# Function that makes sure we don't hit a dpkg lock error
 wait_for_dpkg_lock() {
   while fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock >/dev/null 2>&1; do
     echo "Waiting for dpkg lock to be released..."
@@ -119,61 +120,52 @@ EOC
 mkdir -p /opt/pgbouncer/scripts
 
 # Create the health check script
-cat <<'EOC' > /opt/pgbouncer/scripts/check-pgbouncer.sh
+cat <<'EOC' > /opt/pgbouncer/scripts/check.sh
 #!/bin/bash
-if ! pgrep pgbouncer > /dev/null; then
-    systemctl start pgbouncer
-    echo 'PgBouncer was down, restarted' | logger -t pgbouncer-monitor
+echo $(/bin/systemctl is-active pgbouncer)
+if ! /bin/systemctl is-active --quiet pgbouncer; then
+  # If it's not active, attempt to start it
+  echo "$(date): PgBouncer is not running, attempting to restart" | logger -t pgbouncer-monitor
+  /bin/systemctl start pgbouncer
+
+  # Check if the restart was successful
+  if /bin/systemctl is-active --quiet pgbouncer; then
+    echo "$(date): PgBouncer successfully restarted" | logger -t pgbouncer-monitor
+  else
+    echo "$(date): Failed to restart PgBouncer" | logger -t pgbouncer-monitor
+  fi
+else
+  # If it's already active, no action is needed
+  echo "$(date): PgBouncer is running; no action needed" | logger -t pgbouncer-monitor
 fi
 EOC
-chmod +x /opt/pgbouncer/scripts/check-pgbouncer.sh
+chmod +x /opt/pgbouncer/scripts/check.sh
 
-# Create a single crontab file
+# enable cron job
 cat <<'EOC' > /opt/pgbouncer/scripts/crontab.txt
 # PgBouncer health check - run every minute
-* * * * * /opt/pgbouncer/scripts/check-pgbouncer.sh
-# PgBouncer metrics collection - run every minute
-* * * * * /opt/pgbouncer/scripts/pgbouncer-metrics.sh
+* * * * * /opt/pgbouncer/scripts/check.sh
 EOC
 
-# Install the crontab as the root user
 crontab /opt/pgbouncer/scripts/crontab.txt
 
-# Verify the crontab was installed
 if ! crontab -l; then
-    echo 'Failed to install crontab' | logger -t pgbouncer-setup
-    exit 1
+  echo 'Failed to install crontab' | logger -t pgbouncer-setup
+  exit 1
 fi
 
-# Create per-boot script directory
-mkdir -p /var/lib/cloud/scripts/per-boot
-
-# Create per-boot script
-cat <<'EOF' > /var/lib/cloud/scripts/per-boot/00-run-config.sh
-#!/bin/bash
-# Stop existing services
-systemctl stop pgbouncer
-
-# Re-run configuration
-curl -o /tmp/user-data http://169.254.169.254/latest/user-data
-chmod +x /tmp/user-data
-/tmp/user-data
-EOF
-
-chmod +x /var/lib/cloud/scripts/per-boot/00-run-config.sh
-
 # Create CloudWatch configuration directory
 mkdir -p /opt/pgbouncer/cloudwatch
 
 # Install CloudWatch agent
 if ! wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb; then
-    echo 'Failed to download CloudWatch agent' | logger -t pgbouncer-setup
-    exit 1
+  echo 'Failed to download CloudWatch agent' | logger -t pgbouncer-setup
+  exit 1
 fi
 
 if ! dpkg -i amazon-cloudwatch-agent.deb; then
-    echo 'Failed to install CloudWatch agent' | logger -t pgbouncer-setup
-    exit 1
+  echo 'Failed to install CloudWatch agent' | logger -t pgbouncer-setup
+  exit 1
 fi
 
 # Create CloudWatch config
@@ -240,14 +232,14 @@ EOC
 
 # Verify the config file exists
 if [ ! -f ${CLOUDWATCH_CONFIG} ]; then
-    echo 'CloudWatch config file not created' | logger -t pgbouncer-setup
-    exit 1
+  echo 'CloudWatch config file not created' | logger -t pgbouncer-setup
+  exit 1
 fi
 
 # Start CloudWatch agent
 if ! /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:${CLOUDWATCH_CONFIG}; then
-    echo 'Failed to configure CloudWatch agent' | logger -t pgbouncer-setup
-    exit 1
+  echo 'Failed to configure CloudWatch agent' | logger -t pgbouncer-setup
+  exit 1
 fi
 
 systemctl enable amazon-cloudwatch-agent
@@ -255,7 +247,7 @@ systemctl start amazon-cloudwatch-agent
 
 # Verify CloudWatch agent is running
 if ! systemctl is-active amazon-cloudwatch-agent; then
-    echo 'CloudWatch agent failed to start' | logger -t pgbouncer-setup
-    exit 1
+  echo 'CloudWatch agent failed to start' | logger -t pgbouncer-setup
+  exit 1
 fi
 

From d95fe7c4fa39aab0916edbf797dce2e201e440ba Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Tue, 10 Dec 2024 20:32:00 -0600
Subject: [PATCH 24/28] re-instate cloudwatch write permissions

---
 cdk/PgBouncer.ts | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 0a7c433..940ac29 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -1,5 +1,6 @@
 import {
   aws_ec2 as ec2,
+  aws_iam as iam,
   aws_secretsmanager as secretsmanager,
   Stack,
 } from "aws-cdk-lib";
@@ -81,6 +82,27 @@ export class PgBouncer extends Construct {
       },
     } = props;
 
+    // Create role for PgBouncer instance to enable writing to CloudWatch
+    const role = new iam.Role(this, "InstanceRole", {
+      assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"),
+      managedPolicies: [
+        iam.ManagedPolicy.fromAwsManagedPolicyName(
+          "AmazonSSMManagedInstanceCore",
+        ),
+        iam.ManagedPolicy.fromAwsManagedPolicyName(
+          "CloudWatchAgentServerPolicy",
+        ),
+      ],
+    });
+
+    // Add policy to allow reading RDS credentials from Secrets Manager
+    role.addToPolicy(
+      new iam.PolicyStatement({
+        actions: ["secretsmanager:GetSecretValue"],
+        resources: [database.secret.secretArn],
+      }),
+    );
+
     // Create PgBouncer instance
     this.instance = new ec2.Instance(this, "Instance", {
       vpc,
@@ -95,6 +117,7 @@ export class PgBouncer extends Construct {
         "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
         { os: ec2.OperatingSystemType.LINUX },
       ),
+      role,
       detailedMonitoring: true,
       blockDevices: [
         {
@@ -110,9 +133,6 @@ export class PgBouncer extends Construct {
       userDataCausesReplacement: true,
     });
 
-    // Grant read access to database secret
-    props.database.secret.grantRead(this.instance);
-
     // Allow PgBouncer to connect to RDS
     database.connections.allowFrom(
       this.instance,

From e0d1dd27afe5df5d76892fcd58532cc352e26a31 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Wed, 11 Dec 2024 06:06:06 -0600
Subject: [PATCH 25/28] make dbInstanceType a config parameter

---
 cdk/PgStacInfra.ts |  7 +-----
 cdk/app.ts         |  4 +++-
 cdk/config.ts      | 56 ++++++++++++++++++++++++++++++++++++----------
 3 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index bd1475f..16aee93 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -41,6 +41,7 @@ export class PgStacInfra extends Stack {
       vpc,
       stage,
       version,
+      dbInstanceType,
       jwksUrl,
       dataAccessRoleArn,
       allocatedStorage,
@@ -63,12 +64,6 @@ export class PgStacInfra extends Stack {
     });
 
     // Pgstac Database
-    // set instance type to t3.micro if stage is test, otherwise t3.small
-    const dbInstanceType =
-      stage === "test"
-        ? ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.MICRO)
-        : ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.SMALL);
-
     const { db, pgstacSecret } = new PgStacDatabase(this, "pgstac-db", {
       vpc,
       allowMajorVersionUpgrade: true,
diff --git a/cdk/app.ts b/cdk/app.ts
index 86964e3..083bda0 100644
--- a/cdk/app.ts
+++ b/cdk/app.ts
@@ -8,6 +8,7 @@ import { PgStacInfra } from "./PgStacInfra";
 const {
   stage,
   version,
+  dbInstanceType,
   buildStackName,
   tags,
   jwksUrl,
@@ -21,7 +22,7 @@ const {
   titilerPgStacApiCustomDomainName,
   stacBrowserRepoTag,
   stacBrowserCustomDomainName,
-  stacBrowserCertificateArn
+  stacBrowserCertificateArn,
 } = new Config();
 
 export const app = new cdk.App({});
@@ -38,6 +39,7 @@ new PgStacInfra(app, buildStackName("pgSTAC"), {
   stage,
   version,
   jwksUrl,
+  dbInstanceType,
   terminationProtection: false,
   bastionIpv4AllowList: [
     "66.17.119.38/32", // Jamison
diff --git a/cdk/config.ts b/cdk/config.ts
index cda9938..5925029 100644
--- a/cdk/config.ts
+++ b/cdk/config.ts
@@ -1,6 +1,9 @@
+import * as aws_ec2 from "aws-cdk-lib/aws-ec2";
+
 export class Config {
   readonly stage: string;
   readonly version: string;
+  readonly dbInstanceType: aws_ec2.InstanceType;
   readonly tags: Record<string, string>;
   readonly jwksUrl: string;
   readonly dataAccessRoleArn: string;
@@ -18,32 +21,61 @@ export class Config {
   constructor() {
     // These are required environment variables and cannot be undefined
     const requiredVariables = [
-      { name: 'STAGE', value: process.env.STAGE },
-      { name: 'JWKS_URL', value: process.env.JWKS_URL },
-      { name: 'DATA_ACCESS_ROLE_ARN', value: process.env.DATA_ACCESS_ROLE_ARN },
-      { name: 'STAC_API_INTEGRATION_API_ARN', value: process.env.STAC_API_INTEGRATION_API_ARN },
-      { name: 'DB_ALLOCATED_STORAGE', value: process.env.DB_ALLOCATED_STORAGE },
-      { name: 'MOSAIC_HOST', value: process.env.MOSAIC_HOST },
-      { name: 'STAC_BROWSER_REPO_TAG', value: process.env.STAC_BROWSER_REPO_TAG },
-      { name: 'STAC_BROWSER_CUSTOM_DOMAIN_NAME', value: process.env.STAC_BROWSER_CUSTOM_DOMAIN_NAME },
-      { name: 'STAC_BROWSER_CERTIFICATE_ARN', value: process.env.STAC_BROWSER_CERTIFICATE_ARN },
-      { name: 'STAC_API_CUSTOM_DOMAIN_NAME', value: process.env.STAC_API_CUSTOM_DOMAIN_NAME },
+      { name: "STAGE", value: process.env.STAGE },
+      { name: "DB_INSTANCE_TYPE", value: process.env.DB_INSTANCE_TYPE },
+      { name: "JWKS_URL", value: process.env.JWKS_URL },
+      { name: "DATA_ACCESS_ROLE_ARN", value: process.env.DATA_ACCESS_ROLE_ARN },
+      {
+        name: "STAC_API_INTEGRATION_API_ARN",
+        value: process.env.STAC_API_INTEGRATION_API_ARN,
+      },
+      { name: "DB_ALLOCATED_STORAGE", value: process.env.DB_ALLOCATED_STORAGE },
+      { name: "MOSAIC_HOST", value: process.env.MOSAIC_HOST },
+      {
+        name: "STAC_BROWSER_REPO_TAG",
+        value: process.env.STAC_BROWSER_REPO_TAG,
+      },
+      {
+        name: "STAC_BROWSER_CUSTOM_DOMAIN_NAME",
+        value: process.env.STAC_BROWSER_CUSTOM_DOMAIN_NAME,
+      },
+      {
+        name: "STAC_BROWSER_CERTIFICATE_ARN",
+        value: process.env.STAC_BROWSER_CERTIFICATE_ARN,
+      },
+      {
+        name: "STAC_API_CUSTOM_DOMAIN_NAME",
+        value: process.env.STAC_API_CUSTOM_DOMAIN_NAME,
+      },
     ];
 
     for (const variable of requiredVariables) {
       if (!variable.value) {
-      throw new Error(`Must provide ${variable.name}`);
+        throw new Error(`Must provide ${variable.name}`);
       }
     }
 
     this.stage = process.env.STAGE!;
+
     this.jwksUrl = process.env.JWKS_URL!;
     this.dataAccessRoleArn = process.env.DATA_ACCESS_ROLE_ARN!;
     this.stacApiIntegrationApiArn = process.env.STAC_API_INTEGRATION_API_ARN!;
+
+    try {
+      this.dbInstanceType = new aws_ec2.InstanceType(
+        process.env.DB_INSTANCE_TYPE!,
+      );
+    } catch (error) {
+      throw new Error(
+        `Invalid DB_INSTANCE_TYPE: ${process.env.DB_INSTANCE_TYPE!}. Error: ${error}`,
+      );
+    }
+
     this.dbAllocatedStorage = Number(process.env.DB_ALLOCATED_STORAGE!);
     this.mosaicHost = process.env.MOSAIC_HOST!;
     this.stacBrowserRepoTag = process.env.STAC_BROWSER_REPO_TAG!;
-    this.stacBrowserCustomDomainName = process.env.STAC_BROWSER_CUSTOM_DOMAIN_NAME!;
+    this.stacBrowserCustomDomainName =
+      process.env.STAC_BROWSER_CUSTOM_DOMAIN_NAME!;
     this.stacBrowserCertificateArn = process.env.STAC_BROWSER_CERTIFICATE_ARN!;
     this.stacApiCustomDomainName = process.env.STAC_API_CUSTOM_DOMAIN_NAME!;
 

From 5485819832eaac316abe1a61dfef82e151775678 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Wed, 11 Dec 2024 06:06:37 -0600
Subject: [PATCH 26/28] fix up default pgBouncerConfig

---
 cdk/PgBouncer.ts   | 116 +++++++++++++++++++++++++++++++--------------
 cdk/PgStacInfra.ts |  34 +++----------
 2 files changed, 87 insertions(+), 63 deletions(-)

diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 940ac29..14db662 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -9,6 +9,17 @@ import { Construct } from "constructs";
 import * as fs from "fs";
 import * as path from "path";
 
+export interface PgBouncerConfigProps {
+  poolMode?: "transaction" | "session" | "statement";
+  maxClientConn?: number;
+  defaultPoolSize?: number;
+  minPoolSize?: number;
+  reservePoolSize?: number;
+  reservePoolTimeout?: number;
+  maxDbConnections?: number;
+  maxUserConnections?: number;
+}
+
 export interface PgBouncerProps {
   /**
    * Name for the pgbouncer instance
@@ -24,6 +35,7 @@ export interface PgBouncerProps {
    * The RDS instance to connect to
    */
   database: {
+    instanceType: ec2.InstanceType;
     connections: ec2.Connections;
     secret: secretsmanager.ISecret;
   };
@@ -36,51 +48,84 @@ export interface PgBouncerProps {
 
   /**
    * Instance type for PgBouncer
-   * @default t3.small
+   * @default t3.micro
    */
   instanceType?: ec2.InstanceType;
 
   /**
    * PgBouncer configuration options
    */
-  pgBouncerConfig: {
-    poolMode: "transaction" | "session" | "statement";
-    maxClientConn: number;
-    defaultPoolSize: number;
-    minPoolSize: number;
-    reservePoolSize: number;
-    reservePoolTimeout: number;
-    maxDbConnections: number;
-    maxUserConnections: number;
-  };
+  pgBouncerConfig?: PgBouncerConfigProps;
 }
 
 export class PgBouncer extends Construct {
   public readonly instance: ec2.Instance;
   public readonly endpoint: string;
 
+  // The max_connections parameter in PgBouncer determines the maximum number of
+  // connections to open on the actual database instance. We want that number to
+  // be slightly smaller than the actual max_connections value on the RDS instance
+  // so we perform this calculation.
+
+  // TODO: move this to eoapi-cdk where we already have a complete map of instance
+  // type and memory
+  private readonly instanceMemoryMapMb: Record<string, number> = {
+    "t3.micro": 1024,
+    "t3.small": 2048,
+    "t3.medium": 4096,
+  };
+
+  private calculateMaxConnections(dbInstanceType: ec2.InstanceType): number {
+    const memoryMb = this.instanceMemoryMapMb[dbInstanceType.toString()];
+    if (!memoryMb) {
+      throw new Error(
+        `Unsupported instance type: ${dbInstanceType.toString()}`,
+      );
+    }
+
+    // RDS calculates the available memory as the total instance memory minus some
+    // constant for OS overhead
+    const memoryInBytes = (memoryMb - 300) * 1024 ** 2;
+
+    // The default max_connections setting follows this formula:
+    return Math.min(Math.round(memoryInBytes / 9531392), 5000);
+  }
+
+  private getDefaultConfig(
+    dbInstanceType: ec2.InstanceType,
+  ): Required<PgBouncerConfigProps> {
+    // calculate approximate max_connections setting for this RDS instance type
+    const maxConnections = this.calculateMaxConnections(dbInstanceType);
+
+    return {
+      poolMode: "transaction",
+      maxClientConn: 1000,
+      defaultPoolSize: 5,
+      minPoolSize: 0,
+      reservePoolSize: 5,
+      reservePoolTimeout: 5,
+      maxDbConnections: maxConnections - 10,
+      maxUserConnections: maxConnections - 10,
+    };
+  }
+
   constructor(scope: Construct, id: string, props: PgBouncerProps) {
     super(scope, id);
 
-    const {
-      vpc,
-      database,
-      usePublicSubnet = false,
-      instanceType = ec2.InstanceType.of(
-        ec2.InstanceClass.T3,
-        ec2.InstanceSize.MICRO,
-      ),
-      pgBouncerConfig = {
-        poolMode: "transaction",
-        maxClientConn: 200,
-        defaultPoolSize: 5,
-        minPoolSize: 0,
-        reservePoolSize: 5,
-        reservePoolTimeout: 5,
-        maxDbConnections: 40,
-        maxUserConnections: 40,
-      },
-    } = props;
+    // Set defaults for optional props
+    const defaultInstanceType = ec2.InstanceType.of(
+      ec2.InstanceClass.T3,
+      ec2.InstanceSize.MICRO,
+    );
+
+    const instanceType = props.instanceType ?? defaultInstanceType;
+    const defaultConfig = this.getDefaultConfig(props.database.instanceType);
+
+    // Merge provided config with defaults
+    const pgBouncerConfig: Required<PgBouncerConfigProps> = {
+      ...defaultConfig,
+      ...props.pgBouncerConfig,
+    };
 
     // Create role for PgBouncer instance to enable writing to CloudWatch
     const role = new iam.Role(this, "InstanceRole", {
@@ -99,15 +144,15 @@ export class PgBouncer extends Construct {
     role.addToPolicy(
       new iam.PolicyStatement({
         actions: ["secretsmanager:GetSecretValue"],
-        resources: [database.secret.secretArn],
+        resources: [props.database.secret.secretArn],
       }),
     );
 
     // Create PgBouncer instance
     this.instance = new ec2.Instance(this, "Instance", {
-      vpc,
+      vpc: props.vpc,
       vpcSubnets: {
-        subnetType: usePublicSubnet
+        subnetType: props.usePublicSubnet
           ? ec2.SubnetType.PUBLIC
           : ec2.SubnetType.PRIVATE_WITH_EGRESS,
       },
@@ -118,7 +163,6 @@ export class PgBouncer extends Construct {
         { os: ec2.OperatingSystemType.LINUX },
       ),
       role,
-      detailedMonitoring: true,
       blockDevices: [
         {
           deviceName: "/dev/xvda",
@@ -129,12 +173,12 @@ export class PgBouncer extends Construct {
           }),
         },
       ],
-      userData: this.loadUserDataScript(pgBouncerConfig, database),
+      userData: this.loadUserDataScript(pgBouncerConfig, props.database),
       userDataCausesReplacement: true,
     });
 
     // Allow PgBouncer to connect to RDS
-    database.connections.allowFrom(
+    props.database.connections.allowFrom(
       this.instance,
       ec2.Port.tcp(5432),
       "Allow PgBouncer to connect to RDS",
diff --git a/cdk/PgStacInfra.ts b/cdk/PgStacInfra.ts
index 16aee93..4513b5b 100644
--- a/cdk/PgStacInfra.ts
+++ b/cdk/PgStacInfra.ts
@@ -8,14 +8,7 @@ import {
   aws_cloudfront as cloudfront,
   aws_cloudfront_origins as origins,
 } from "aws-cdk-lib";
-import {
-  Aws,
-  Duration,
-  RemovalPolicy,
-  Stack,
-  StackProps,
-  Tags,
-} from "aws-cdk-lib";
+import { Aws, Duration, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib";
 import { Construct } from "constructs";
 import {
   BastionHost,
@@ -80,27 +73,11 @@ export class PgStacInfra extends Stack {
     });
 
     // PgBouncer
-    // total RDS instance memory mapping
-    const instanceMemoryMapMb: Record<string, number> = {
-      "t3.micro": 1024,
-      "t3.small": 2048,
-      "t3.medium": 4096,
-    };
-
-    // Available memory is total instance memory minus some constant for OS overhead (~300?)
-    const memoryInBytes =
-      (instanceMemoryMapMb[dbInstanceType.toString()] - 300) * 1024 ** 2;
-
-    // Max connections for pgbouncer should be the instance max_connections minus 10
-    const maxPgBouncerDbConnections = Math.min(
-      Math.round(memoryInBytes / 9531392) - 10,
-      5000,
-    );
-
     const pgBouncer = new PgBouncer(this, "pgbouncer", {
       instanceName: `pgbouncer-${stage}`,
       vpc: props.vpc,
       database: {
+        instanceType: dbInstanceType,
         connections: db.connections,
         secret: pgstacSecret,
       },
@@ -112,8 +89,6 @@ export class PgStacInfra extends Stack {
         minPoolSize: 10,
         reservePoolSize: 5,
         reservePoolTimeout: 5,
-        maxDbConnections: maxPgBouncerDbConnections,
-        maxUserConnections: maxPgBouncerDbConnections,
       },
     });
 
@@ -362,6 +337,11 @@ export interface Props extends StackProps {
    */
   version: string;
 
+  /**
+   * RDS Instance type
+   */
+  dbInstanceType: ec2.InstanceType;
+
   /**
    * Flag to control whether database should be deployed into a
    * public subnet.

From 7a3aff0312e2069df13e7e95d89e571dc6ebc758 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Wed, 11 Dec 2024 06:30:29 -0600
Subject: [PATCH 27/28] add DB_INSTANCE_TYPE to env in deploy.yml

---
 .github/workflows/deploy.yml | 1 +
 cdk/PgBouncer.ts             | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 2a62c83..b823b10 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -51,6 +51,7 @@ jobs:
         JWKS_URL: ${{ steps.import-stacks-vars-to-output.outputs.JWKS_URL }}
         DATA_ACCESS_ROLE_ARN: ${{ steps.import-stacks-vars-to-output.outputs.DATA_ACCESS_ROLE_ARN }}
         DB_ALLOCATED_STORAGE: ${{ vars.DB_ALLOCATED_STORAGE }}
+        DB_INSTANCE_TYPE: ${{ vars.DB_INSTANCE_TYPE }}
         GIT_REPOSITORY: ${{ github.repository}}
         COMMIT_SHA: ${{ github.sha }}
         AUTHOR: ${{ github.actor }}
diff --git a/cdk/PgBouncer.ts b/cdk/PgBouncer.ts
index 14db662..afca8a5 100644
--- a/cdk/PgBouncer.ts
+++ b/cdk/PgBouncer.ts
@@ -9,6 +9,8 @@ import { Construct } from "constructs";
 import * as fs from "fs";
 import * as path from "path";
 
+// used to populate pgbouncer config:
+// see https://www.pgbouncer.org/config.html for details
 export interface PgBouncerConfigProps {
   poolMode?: "transaction" | "session" | "statement";
   maxClientConn?: number;
@@ -97,6 +99,8 @@ export class PgBouncer extends Construct {
     // calculate approximate max_connections setting for this RDS instance type
     const maxConnections = this.calculateMaxConnections(dbInstanceType);
 
+    // maxDbConnections (and maxUserConnections) are the only settings that need
+    // to be responsive to the database size/max_connections setting
     return {
       poolMode: "transaction",
       maxClientConn: 1000,

From e874466cbfaa5715a6f29087aab2fe93cacd78b5 Mon Sep 17 00:00:00 2001
From: hrodmn <henry.rodman@gmail.com>
Date: Wed, 11 Dec 2024 07:00:16 -0600
Subject: [PATCH 28/28] make sure /var/run/pgbouncer directory is created on
 each boot

---
 cdk/scripts/pgbouncer-setup.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdk/scripts/pgbouncer-setup.sh b/cdk/scripts/pgbouncer-setup.sh
index 39c14ee..bcbcb1b 100644
--- a/cdk/scripts/pgbouncer-setup.sh
+++ b/cdk/scripts/pgbouncer-setup.sh
@@ -91,6 +91,11 @@ chown postgres:postgres /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
 chmod 600 /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/userlist.txt
 
 # Configure logging
+# ensure /var/run/pgbouncer gets created on boot
+cat <<EOC > /etc/tmpfiles.d/pgbouncer.conf
+d /var/run/pgbouncer 0755 postgres postgres -
+EOC
+
 mkdir -p /var/log/pgbouncer /var/run/pgbouncer
 chown postgres:postgres /var/log/pgbouncer /var/run/pgbouncer
 chmod 755 /var/log/pgbouncer /var/run/pgbouncer