PMM-14517: Percona Distribution for Postgresql 18 and support for cgroups

pmm_qa/percona-distribution-postgresql/data/etcd.conf.yaml.j2

@@ -4,9 +4,9 @@ initial-cluster-token: PostgreSQL_HA_Cluster_1
 initial-cluster-state: new
 
 listen-peer-urls: http://0.0.0.0:2380
-initial-advertise-peer-urls: http://pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:2380
+initial-advertise-peer-urls: http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_{{ item }}:2380
 
 listen-client-urls: http://0.0.0.0:2379
-advertise-client-urls: http://pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:2379
+advertise-client-urls: http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_{{ item }}:2379
 
-initial-cluster: node1=http://pdpgsql_pmm_patroni_{{ pg_version }}_1:2380,node2=http://pdpgsql_pmm_patroni_{{ pg_version }}_2:2380,node3=http://pdpgsql_pmm_patroni_{{ pg_version }}_3:2380
+initial-cluster: node1=http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_1:2380,node2=http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_2:2380,node3=http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_3:2380

pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2

@@ -1,13 +1,13 @@
 scope: patroni_cls
 namespace: /var/lib/pgsql/config/
-name: pg_node_{{ item }}
+name: {{ container_prefix }}{{ item }}
 
 restapi:
   listen: 0.0.0.0:8008
-  connect_address: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:8008"
+  connect_address: "{{ container_prefix }}{{ item }}:8008"
 
 etcd3:
-  host: "pdpgsql_pmm_patroni_{{ pg_version }}_1:2379"
+  host: "{{ container_prefix }}1:2379"
 
 bootstrap:
   dcs:
@@ -29,7 +29,7 @@ bootstrap:
         wal_log_hints: "on"
         archive_mode: "on"
         archive_timeout: 600s
-        archive_command: "cp -f %p /home/postgres/archived/%f"
+        archive_command: pgbackrest --stanza=patroni_backup --log-level-console=info archive-push /var/lib/postgresql/{{ pdpgsql_version }}/main/pg_wal/%f
       pg_hba:
         - host replication replicator 127.0.0.1/32 md5
         - host replication replicator 172.18.0.0/16 md5
@@ -51,9 +51,9 @@ bootstrap:
 
 postgresql:
   listen: 0.0.0.0:5432
-  connect_address: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:5432"
-  data_dir: /var/lib/pgsql/{{ pg_version}}/data
-  bin_dir: /usr/pgsql-{{ pg_version }}/bin
+  connect_address: "{{ container_prefix }}{{ item }}:5432"
+  data_dir: /var/lib/postgresql/{{ pdpgsql_version }}/main
+  bin_dir: /usr/lib/postgresql/{{ pdpgsql_version }}/bin
   pgpass: /tmp/pgpass
 
   authentication:
@@ -62,14 +62,14 @@ postgresql:
       password: replPasswd
     superuser:
       username: postgres
-      password: postgres
+      password: pass+this
 
   create_replica_methods:
     - pgbackrest
     - basebackup
 
   pgbackrest:
-    command: pgbackrest --stanza=patroni_backup restore --type=none
+    command: pgbackrest --stanza=patroni_backup restore --pg1-path=/var/lib/postgresql/{{ pdpgsql_version }}/main --type=none
     keep_data: true
     no_params: true
 

pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2

@@ -1,3 +1,4 @@
 host    replication     {{ replication_user }}      0.0.0.0/0       md5
 host    all             all                         0.0.0.0/0       md5
 local   all             postgres                    trust
+local   all             all                         md5

pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2

@@ -1,3 +1,4 @@
 host    replication     {{ replication_user }}      0.0.0.0/0       md5
 host    all             all                         0.0.0.0/0       md5
 local   all             postgres                    trust
+local   all             all                         md5

pmm_qa/percona-distribution-postgresql/data/pgsm_run_queries.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+## Running Queries
+wget https://raw.githubusercontent.com/percona/pmm-agent/main/testqueries/postgres/pg_stat_monitor_load.sql
+while true
+do
+  su postgres bash -c 'psql -d contrib_regression -f pg_stat_monitor_load.sql'
+  su postgres bash -c 'psql -d sbtest1 -f pg_stat_monitor_load.sql'
+  sleep 30
+done

pmm_qa/percona-distribution-postgresql/data/pgsql_load.sql

@@ -0,0 +1,115 @@
+-- ========================================
+-- CREATE TABLES
+-- ========================================
+
+CREATE TABLE students (
+    student_id SERIAL PRIMARY KEY,
+    first_name VARCHAR(50),
+    last_name VARCHAR(50),
+    birth_date DATE
+);
+
+CREATE TABLE classes (
+    class_id SERIAL PRIMARY KEY,
+    name VARCHAR(100),
+    teacher VARCHAR(100)
+);
+
+CREATE TABLE enrollments (
+    enrollment_id SERIAL PRIMARY KEY,
+    student_id INTEGER REFERENCES students(student_id),
+    class_id INTEGER REFERENCES classes(class_id),
+    enrollment_date DATE DEFAULT CURRENT_DATE
+);
+
+-- ========================================
+-- INSERT MOCK DATA
+-- ========================================
+
+INSERT INTO students (first_name, last_name, birth_date) VALUES
+('Alice', 'Smith', '2005-04-10'),
+('Bob', 'Johnson', '2006-08-15'),
+('Charlie', 'Brown', '2004-12-01');
+
+INSERT INTO classes (name, teacher) VALUES
+('Mathematics', 'Mrs. Taylor'),
+('History', 'Mr. Anderson'),
+('Science', 'Dr. Reynolds');
+
+INSERT INTO enrollments (student_id, class_id) VALUES
+(1, 1),
+(1, 2),
+(2, 2),
+(3, 1),
+(3, 3);
+
+-- ========================================
+-- SIMULATE DEAD TUPLES
+-- ========================================
+
+
+INSERT INTO students (first_name, last_name, birth_date)
+SELECT 'John', 'Doe', CURRENT_DATE - (random() * 5000)::int
+FROM generate_series(1, 100000);
+
+-- These updates and deletes will create dead tuples
+
+-- Update records (old versions become dead)
+UPDATE students
+SET last_name = last_name || '_updated'
+WHERE student_id IN (1, 2);
+
+-- Delete records (deleted rows become dead)
+DELETE FROM enrollments
+WHERE enrollment_id IN (SELECT enrollment_id FROM enrollments LIMIT 2);
+
+-- Disable autovacuum temporarily (for demo)
+ALTER TABLE students SET (autovacuum_enabled = false);
+ALTER TABLE enrollments SET (autovacuum_enabled = false);
+
+-- ========================================
+-- SELECT QUERIES
+-- ========================================
+
+-- Get all students
+SELECT * FROM students;
+
+-- Get all students enrolled in Mathematics
+SELECT s.first_name, s.last_name
+FROM students s
+JOIN enrollments e ON s.student_id = e.student_id
+JOIN classes c ON e.class_id = c.class_id
+WHERE c.name = 'Mathematics';
+
+-- Count students per class
+SELECT c.name, COUNT(e.student_id) AS student_count
+FROM classes c
+LEFT JOIN enrollments e ON c.class_id = e.class_id
+GROUP BY c.name;
+
+-- ========================================
+-- UPDATE QUERIES
+-- ========================================
+
+-- Change Bob's last name
+UPDATE students
+SET last_name = 'Williams'
+WHERE first_name = 'Bob' AND last_name = 'Johnson';
+
+-- Update the teacher for the History class
+UPDATE classes
+SET teacher = 'Ms. Carter'
+WHERE name = 'History';
+
+-- ========================================
+-- DELETE QUERIES
+-- ========================================
+
+-- Remove Charlie from Science class
+DELETE FROM enrollments
+WHERE student_id = (SELECT student_id FROM students WHERE first_name = 'Charlie')
+  AND class_id = (SELECT class_id FROM classes WHERE name = 'Science');
+
+-- Delete a student completely
+DELETE FROM students
+WHERE first_name = 'Alice' AND last_name = 'Smith';

pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf → pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf.j2

@@ -3,7 +3,8 @@ max_wal_senders = 10
 wal_keep_size = 64MB
 hot_standby = on
 listen_addresses = '*'
-hba_file = '/etc/postgresql/pg_hba.conf'
+hba_file = '/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf'
+data_directory = '/var/lib/postgresql/{{ pdpgsql_version }}/main'
 shared_preload_libraries = 'pg_stat_monitor'
 track_activity_query_size=2048
 track_io_timing=ON

pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf → pmm_qa/percona-distribution-postgresql/data/postgresql-replica.conf.j2

@@ -1,4 +1,5 @@
-hba_file = '/etc/postgresql/pg_hba.conf'
+hba_file = '/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf'
+data_directory = '/var/lib/postgresql/{{ pdpgsql_version }}/main'
 shared_preload_libraries = 'pg_stat_monitor'
 track_activity_query_size=2048
 track_io_timing=ON
@@ -11,4 +12,4 @@ pg_stat_monitor.pgsm_enable_query_plan=1
 log_connections = on
 log_disconnections = on
 log_replication_commands = on
-log_statement = 'all'
+log_statement = 'all'

pmm_qa/percona-distribution-postgresql/data/postgresql-single.conf.j2

@@ -0,0 +1,12 @@
+listen_addresses = '*'
+hba_file = '/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf'
+data_directory = '/var/lib/postgresql/{{ pdpgsql_version }}/main'
+shared_preload_libraries = 'pg_stat_monitor'
+track_activity_query_size=2048
+track_io_timing=ON
+max_connections=1000
+pg_stat_monitor.pgsm_enable_query_plan = 'yes'
+log_connections = on
+log_disconnections = on
+log_replication_commands = on
+log_statement = 'all'

pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml

@@ -5,12 +5,12 @@
   connection: local
   gather_facts: yes
   vars:
-    pg_version: "{{ lookup('env', 'PGSQL_VERSION') | default('17', true) }}"
+    pdpgsql_version: "{{ lookup('env', 'PDPGSQL_VERSION') | default('17', true) }}"
     replication_user: "repl_user"
     replication_password: "GRgrO9301RuF"
     root_password: "GRgrO9301RuF"
     pdpgsql_port: 5432
-    nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}"
+    nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('1', true)) | int }}"
     network_name: "pmm-qa"
     data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data"
     pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}"
@@ -21,12 +21,22 @@
     setup_type: "{{ lookup('env', 'SETUP_TYPE') }}"
     random_service_name_value: ""
     docker_repo: "percona/percona-distribution-postgresql"
+    container_prefix: "pdpgsql_pmm{{ (setup_type|default('')) and '_' ~ setup_type }}_{{ pdpgsql_version }}_"
 
   tasks:
-    - name: Display binary log status for primary
+    - name: Display setup type selected
       debug:
         msg: "{{ setup_type }}"
 
+    - name: Create Docker network
+      shell: docker network create {{ network_name }}
+      ignore_errors: yes
+      become: true
+
+    - name: Install Percona Distribution for Postgres
+      include_tasks: tasks/install-percona-distribution-postgresql-single.yml
+      when: setup_type != "replication" and setup_type != "patroni"
+
     - name: Install Percona Distribution for Postgres with Replication
       include_tasks: tasks/percona-distribution-postgres-replication-setup.yml
       when: setup_type == "replication"

pmm_qa/percona-distribution-postgresql/tasks/install-pdpgsql.yml

@@ -0,0 +1,44 @@
+- name: Start containers for Percona Distribution for Postgresql
+  shell: |
+    docker run -d \
+      --name={{ container_prefix }}{{ item }} \
+      --restart=always \
+      --network {{ network_name }} \
+      -e POSTGRES_PASSWORD={{ root_password }} \
+      --privileged --cgroupns=host \
+      -v /sys/fs/cgroup:/sys/fs/cgroup:rw \
+      -v /var/lib/containerd \
+      -p {{ pdpgsql_port + (item - 1) }}:5432 \
+      antmelekhin/docker-systemd:ubuntu-24.04
+  become: true
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Install dependencies
+  shell: |
+    docker exec {{ container_prefix }}{{ item }} apt-get update
+    docker exec {{ container_prefix }}{{ item }} apt-get -y install wget gnupg2 lsb-release curl
+  become: true
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Install Percona Release
+  shell: |
+    docker exec {{ container_prefix }}{{ item }} wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb
+    docker exec {{ container_prefix }}{{ item }} dpkg -i ./percona-release_latest.generic_all.deb
+    docker exec {{ container_prefix }}{{ item }} percona-release enable ppg-{{ pdpgsql_version }}
+  become: true
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Install Percona Distribution for PostgreSQL
+  shell: |
+    docker exec -u root {{ container_prefix }}{{ item }} apt-get install -y percona-postgresql-{{ pdpgsql_version }} percona-pgbackrest percona-pg-stat-monitor{{ pdpgsql_version }}
+  become: true
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Start Percona distribution for Postgresql
+  shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl start postgresql
+  become: true
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Wait 5 seconds for Percona Distribution for postgresql to start
+  pause:
+    seconds: 5