From dcb0a25c111a843919f907ba6fe6ccd3e878b0a6 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Fri, 30 Aug 2024 23:42:37 -0500 Subject: [PATCH 1/3] Improve Prometheus stats + Add Grafana dashboard --- grafana_dashboard.json | 2124 ++++++++++++++++++++++++++++++++++++++++ src/prometheus.rs | 113 ++- 2 files changed, 2209 insertions(+), 28 deletions(-) create mode 100644 grafana_dashboard.json diff --git a/grafana_dashboard.json b/grafana_dashboard.json new file mode 100644 index 00000000..a5f7f835 --- /dev/null +++ b/grafana_dashboard.json @@ -0,0 +1,2124 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard to monitor PgCat deployments. It is based on the metrics exported by the Prometheus exporter that comes bundled with PgCat\n\nPlease visit https://github.com/postgresml/pgcat for more information ", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 4, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 39, + "title": "Throughput Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "For this metric, each individual query outside a transaction is counted as one transaction. All queries that run inside a transaction are counted as one transaction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_transaction_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Transaction Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of individual queries run", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_query_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Query Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Average Query Latency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_stats_avg_query_time{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average Query Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Average latency of transactions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_stats_avg_xact_time{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average Transaction Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Data received in bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_bytes_received{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Data received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Data sent in bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_bytes_sent{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Data Sent", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 22, + "panels": [], + "title": "Capacity metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "This is the ratio between the number of active server connections to the pool size. Persistently high ratio (e.g. 80%-100%) may suggest the need for a larger pool or a more performant Database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 150, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 0, + "y": 14 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "(pgcat_servers_active_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"} / pgcat_databases_pool_size{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}) * 100 ", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Percentage Server Pool Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently waiting in the queue waiting for a server connection to be assigned. This should remain close to 0. Any persistent deviation from zero means clients are blocked from querying the database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 13, + "y": 14 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_waiting{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Waiting Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Banned connections won't get queries routed to them until they are unbanned. Instances are unbanned after the ban timer expires or if all replicas in the pool are banned so they are all unbanned as a failsafe. Primary instances are never banned", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_is_banned{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Banned Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Paused connection are connections that belong to pool that was paused by the administrator", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 22 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_is_paused{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Paused Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 22 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "(pgcat_pools_maxwait{pool=~\"$pool\", user=~\"$user\"} * 1000 * 1000) + pgcat_pools_maxwait_us{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Maximum wait time by Pool", + "type": "timeseries" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 16, + "title": "Server Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 30 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_idle_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Idle Server Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 30 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_active_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Server Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 30 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_login_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Server Connection in Login State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 30 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_tested_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Server Connection in Tested State", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 12, + "panels": [], + "title": "Client Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently connected but not assigned a server connection nor are they seeking one", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_idle{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Idle Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently assigned a server connection", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 37 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_active{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently waiting in the queue waiting for a server connection to be assigned. This should remain close to 0. Any persistent deviation from zero means clients are blocked from querying the database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 37 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_waiting{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Waiting Clients", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "Databases", + "PostgreSQL" + ], + "templating": { + "list": [ + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,role)", + "description": "Instance Role in the pool", + "hide": 0, + "includeAll": true, + "label": "Role", + "multi": false, + "name": "role", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,role)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,shard)", + "description": "", + "hide": 0, + "includeAll": true, + "label": "Shard ID", + "multi": true, + "name": "shard_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,shard)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,index)", + "description": "The instance index in the role (e.g. a pool with two replicas will have instances with indices 0 and 1", + "hide": 0, + "includeAll": true, + "label": "Instance Index", + "multi": true, + "name": "instance_index", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,index)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,pool)", + "description": "The PgCat Connection Pool ", + "hide": 0, + "includeAll": true, + "label": "Pool Name", + "multi": true, + "name": "pool", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,pool)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,host)", + "description": "The underlying Database host name", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(usename)", + "description": "PostgreSQL username used by the pool", + "hide": 0, + "includeAll": true, + "label": "Username", + "multi": true, + "name": "user", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(usename)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "PgCat Dashboard", + "uid": "ddwejyl5j6jnkb", + "version": 46, + "weekStart": "" +} diff --git a/src/prometheus.rs b/src/prometheus.rs index d8e23464..8c2dcc38 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -8,23 +8,34 @@ use hyper::{Method, Request, Response, StatusCode}; use hyper_util::rt::TokioIo; use log::{debug, error, info}; use phf::phf_map; +use tokio::net::TcpListener; use std::collections::HashMap; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::Ordering; -use std::sync::Arc; -use tokio::net::TcpListener; use crate::config::Address; use crate::pool::{get_all_pools, PoolIdentifier}; +use crate::stats::get_server_stats; use crate::stats::pool::PoolStats; -use crate::stats::{get_server_stats, ServerStats}; struct MetricHelpType { help: &'static str, ty: &'static str, } +struct ServerPrometheusStats { + bytes_received: u64, + bytes_sent: u64, + transaction_count: u64, + query_count: u64, + error_count: u64, + active_count: u64, + idle_count: u64, + login_count: u64, + tested_count: u64, +} + // reference for metric types: https://prometheus.io/docs/concepts/metric_types/ // counters only increase // gauges can arbitrarily increase or decrease @@ -127,22 +138,46 @@ static METRIC_HELP_AND_TYPES_LOOKUP: phf::Map<&'static str, MetricHelpType> = ph }, "servers_bytes_received" => MetricHelpType { help: "Volume in bytes of network traffic received by server", - ty: "gauge", + ty: "counter", }, "servers_bytes_sent" => MetricHelpType { help: "Volume in bytes of network traffic sent by server", - ty: "gauge", + ty: "counter", }, "servers_transaction_count" => MetricHelpType { help: "Number of transactions executed by server", - ty: "gauge", + ty: "counter", }, "servers_query_count" => MetricHelpType { help: "Number of queries executed by server", - ty: "gauge", + ty: "counter", }, "servers_error_count" => MetricHelpType { help: "Number of errors", + ty: "counter", + }, + "servers_idle_count" => MetricHelpType { + help: "Number of server connection in idle state", + ty: "gauge", + }, + "servers_active_count" => MetricHelpType { + help: "Number of server connection in active state", + ty: "gauge", + }, + "servers_tested_count" => MetricHelpType { + help: "Number of server connection in tested state", + ty: "gauge", + }, + "servers_login_count" => MetricHelpType { + help: "Number of server connection in login state", + ty: "gauge", + }, + "servers_is_banned" => MetricHelpType { + help: "0 if server is not banned, 1 if server is banned", + ty: "gauge", + }, + "servers_is_paused" => MetricHelpType { + help: "0 if server is not paused, 1 if server is paused", ty: "gauge", }, "databases_pool_size" => MetricHelpType { @@ -210,7 +245,9 @@ impl PrometheusMetric { labels.insert("shard", address.shard.to_string()); labels.insert("role", address.role.to_string()); labels.insert("pool", address.pool_name.clone()); + labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); + labels.insert("user", address.username.clone()); Self::from_name(&format!("databases_{}", name), value, labels) } @@ -225,8 +262,9 @@ impl PrometheusMetric { labels.insert("shard", address.shard.to_string()); labels.insert("role", address.role.to_string()); labels.insert("pool", address.pool_name.clone()); + labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); - + labels.insert("user", address.username.clone()); Self::from_name(&format!("servers_{}", name), value, labels) } @@ -236,7 +274,9 @@ impl PrometheusMetric { labels.insert("shard", address.shard.to_string()); labels.insert("pool", address.pool_name.clone()); labels.insert("role", address.role.to_string()); + labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); + labels.insert("user", address.username.clone()); Self::from_name(&format!("stats_{}", name), value, labels) } @@ -338,34 +378,51 @@ fn push_database_stats(lines: &mut Vec) { // Adds relevant metrics shown in a SHOW SERVERS admin command. fn push_server_stats(lines: &mut Vec) { let server_stats = get_server_stats(); - let mut server_stats_by_addresses = HashMap::>::new(); + let mut prom_stats = HashMap::::new(); for (_, stats) in server_stats { - server_stats_by_addresses.insert(stats.address_name(), stats); + let entry = prom_stats + .entry(stats.address_name()) + .or_insert(ServerPrometheusStats { + bytes_received: 0, + bytes_sent: 0, + transaction_count: 0, + query_count: 0, + error_count: 0, + active_count: 0, + idle_count: 0, + login_count: 0, + tested_count: 0, + }); + entry.bytes_received += stats.bytes_received.load(Ordering::Relaxed); + entry.bytes_sent += stats.bytes_sent.load(Ordering::Relaxed); + entry.transaction_count += stats.transaction_count.load(Ordering::Relaxed); + entry.query_count += stats.query_count.load(Ordering::Relaxed); + entry.error_count += stats.error_count.load(Ordering::Relaxed); + match stats.state.load(Ordering::Relaxed) { + crate::stats::ServerState::Login => entry.login_count += 1, + crate::stats::ServerState::Active => entry.active_count += 1, + crate::stats::ServerState::Tested => entry.tested_count += 1, + crate::stats::ServerState::Idle => entry.idle_count += 1, + } } for (_, pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); - if let Some(server_info) = server_stats_by_addresses.get(&address.name()) { + if let Some(server_info) = prom_stats.get(&address.name()) { let metrics = [ - ( - "bytes_received", - server_info.bytes_received.load(Ordering::Relaxed), - ), - ("bytes_sent", server_info.bytes_sent.load(Ordering::Relaxed)), - ( - "transaction_count", - server_info.transaction_count.load(Ordering::Relaxed), - ), - ( - "query_count", - server_info.query_count.load(Ordering::Relaxed), - ), - ( - "error_count", - server_info.error_count.load(Ordering::Relaxed), - ), + ("bytes_received", server_info.bytes_received), + ("bytes_sent", server_info.bytes_sent), + ("transaction_count", server_info.transaction_count), + ("query_count", server_info.query_count), + ("error_count", server_info.error_count), + ("idle_count", server_info.idle_count), + ("active_count", server_info.active_count), + ("login_count", server_info.login_count), + ("tested_count", server_info.tested_count), + ("is_banned", if pool.is_banned(address) { 1 } else { 0 }), + ("is_paused", if pool.paused() { 1 } else { 0 }), ]; for (key, value) in metrics { if let Some(prometheus_metric) = From 6c63a5880b157158ee45b75415a91dcdbe5c3d2b Mon Sep 17 00:00:00 2001 From: Mostafa Date: Fri, 30 Aug 2024 23:46:05 -0500 Subject: [PATCH 2/3] fmt --- src/prometheus.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prometheus.rs b/src/prometheus.rs index 8c2dcc38..9a14b8fe 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -8,11 +8,11 @@ use hyper::{Method, Request, Response, StatusCode}; use hyper_util::rt::TokioIo; use log::{debug, error, info}; use phf::phf_map; -use tokio::net::TcpListener; use std::collections::HashMap; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::Ordering; +use tokio::net::TcpListener; use crate::config::Address; use crate::pool::{get_all_pools, PoolIdentifier}; From 06976a4f8dd5d4a02e16ff5cc31cb54814bf1151 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Sat, 31 Aug 2024 06:57:16 -0500 Subject: [PATCH 3/3] Update Readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2925fb04..21e6da70 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,8 @@ psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' Additionally, Prometheus statistics are available at `/metrics` via HTTP. +We also have a [basic Grafana dashboard](https://github.com/postgresml/pgcat/blob/main/grafana_dashboard.json) based on Prometheus metrics that you can import into Grafana and build on it or use it for monitoring. + ### Live configuration reloading The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. All settings except the `host` and `port` can be reloaded without restarting the pooler, including sharding and replicas configurations.