diff --git a/accountant/accountant-app/pom.xml b/accountant/accountant-app/pom.xml index 0c663aa6c..2a49ffc42 100644 --- a/accountant/accountant-app/pom.xml +++ b/accountant/accountant-app/pom.xml @@ -63,6 +63,11 @@ org.springframework.boot spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + runtime + co.nilin.opex.utility.preferences preferences diff --git a/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/utils/PrometheusHealthExtension.kt b/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..41763d3c0 --- /dev/null +++ b/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.accountant.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "ACCOUNTANT" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/accountant/accountant-app/src/main/resources/application.yml b/accountant/accountant-app/src/main/resources/application.yml index 1f71cb1b8..0bf62e5a8 100644 --- a/accountant/accountant-app/src/main/resources/application.yml +++ b/accountant/accountant-app/src/main/resources/application.yml @@ -44,6 +44,22 @@ spring: prefer-ip-address: true config: import: vault://secret/${spring.application.name} +management: + health: + db: + enabled: true + endpoints: + web: + base-path: /actuator + exposure: + include: [ "health", "prometheus", "metrics" ] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true app: address: 1 wallet: diff --git a/api/api-app/pom.xml b/api/api-app/pom.xml index 1d3cde562..c58df1517 100644 --- a/api/api-app/pom.xml +++ b/api/api-app/pom.xml @@ -72,6 +72,11 @@ co.nilin.opex.utility.preferences preferences + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/api/api-app/src/main/kotlin/co/nilin/opex/api/app/ApiApp.kt b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/ApiApp.kt index c323a9e19..35df759a8 100644 --- a/api/api-app/src/main/kotlin/co/nilin/opex/api/app/ApiApp.kt +++ b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/ApiApp.kt @@ -4,10 +4,12 @@ import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.cache.annotation.EnableCaching import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling import springfox.documentation.swagger2.annotations.EnableSwagger2 @SpringBootApplication @ComponentScan("co.nilin.opex") +@EnableScheduling class ApiApp fun main(args: Array) { diff --git a/api/api-app/src/main/kotlin/co/nilin/opex/api/app/utils/PrometheusHealthExtension.kt b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..9f3fbc902 --- /dev/null +++ b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.api.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "API" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/api/api-app/src/main/resources/application.yml b/api/api-app/src/main/resources/application.yml index e78bde202..f9c9a1a6f 100644 --- a/api/api-app/src/main/resources/application.yml +++ b/api/api-app/src/main/resources/application.yml @@ -44,6 +44,19 @@ spring: prefer-ip-address: true config: import: vault://secret/${spring.application.name} +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true app: accountant: url: lb://opex-accountant diff --git a/bc-gateway/bc-gateway-app/pom.xml b/bc-gateway/bc-gateway-app/pom.xml index 617e5ad65..d9aaf6386 100644 --- a/bc-gateway/bc-gateway-app/pom.xml +++ b/bc-gateway/bc-gateway-app/pom.xml @@ -94,6 +94,11 @@ org.springframework.cloud spring-cloud-starter-vault-config + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/utils/PrometheusHealthExtension.kt b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..79c77b4a4 --- /dev/null +++ b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.bcgateway.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "BC_GATEWAY" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/bc-gateway/bc-gateway-app/src/main/resources/application.yml b/bc-gateway/bc-gateway-app/src/main/resources/application.yml index 52bb87857..ed25c2298 100644 --- a/bc-gateway/bc-gateway-app/src/main/resources/application.yml +++ b/bc-gateway/bc-gateway-app/src/main/resources/application.yml @@ -42,6 +42,19 @@ spring: import: vault://secret/${spring.application.name} codec: max-in-memory-size: 20MB +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true logging: level: org.apache.kafka: ERROR diff --git a/captcha/captcha-app/pom.xml b/captcha/captcha-app/pom.xml index 9b1a80782..bb347a017 100644 --- a/captcha/captcha-app/pom.xml +++ b/captcha/captcha-app/pom.xml @@ -64,6 +64,11 @@ springfox-boot-starter 3.0.0 + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/captcha/captcha-app/src/main/kotlin/co/nilin/opex/captcha/app/App.kt b/captcha/captcha-app/src/main/kotlin/co/nilin/opex/captcha/app/App.kt index a9d4ef274..1f7c7cac6 100644 --- a/captcha/captcha-app/src/main/kotlin/co/nilin/opex/captcha/app/App.kt +++ b/captcha/captcha-app/src/main/kotlin/co/nilin/opex/captcha/app/App.kt @@ -4,10 +4,12 @@ import co.nilin.opex.utility.error.EnableOpexErrorHandler import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling @SpringBootApplication @ComponentScan("co.nilin.opex") @EnableOpexErrorHandler +@EnableScheduling class App fun main(args: Array) { diff --git a/captcha/captcha-app/src/main/kotlin/co/nilin/opex/captcha/app/utils/PrometheusHealthExtension.kt b/captcha/captcha-app/src/main/kotlin/co/nilin/opex/captcha/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..bc52154b5 --- /dev/null +++ b/captcha/captcha-app/src/main/kotlin/co/nilin/opex/captcha/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.captcha.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "CAPTCHA" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/captcha/captcha-app/src/main/resources/application.yml b/captcha/captcha-app/src/main/resources/application.yml index 67129ca4e..720d3186b 100644 --- a/captcha/captcha-app/src/main/resources/application.yml +++ b/captcha/captcha-app/src/main/resources/application.yml @@ -21,5 +21,18 @@ spring: instance-id: ${spring.application.name}:${server.port} healthCheckInterval: 20s prefer-ip-address: true +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true app: captcha-window-seconds: 600 diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 6d610a59b..f9a1fa115 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -1,5 +1,14 @@ version: '3.8' services: + kafka-1: + image: ghcr.io/opexdev/kafka:$TAG + build: docker-images/kafka + kafka-2: + image: ghcr.io/opexdev/kafka:$TAG + build: docker-images/kafka + kafka-3: + image: ghcr.io/opexdev/kafka:$TAG + build: docker-images/kafka vault: image: ghcr.io/opexdev/vault-opex:$TAG build: docker-images/vault @@ -48,6 +57,12 @@ services: admin: image: ghcr.io/opexdev/admin:$TAG build: admin/admin-app + logstash: + image: ghcr.io/opexdev/opex-logstash:$TAG + build: docker-images/logstash filebeat: image: ghcr.io/opexdev/opex-filebeat:$TAG build: docker-images/filebeat + prometheus: + image: ghcr.io/opexdev/opex-prometheus:$TAG + build: docker-images/prometheus diff --git a/docker-compose.override.yml b/docker-compose.override.yml index c1e63580e..423035d26 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,5 +1,11 @@ version: '3.8' services: + kafka-1: + build: docker-images/kafka + kafka-2: + build: docker-images/kafka + kafka-3: + build: docker-images/kafka vault: build: docker-images/vault postgres-opex: @@ -48,5 +54,9 @@ services: build: captcha/captcha-app admin: build: admin/admin-app + logstash: + build: docker-images/logstash filebeat: build: docker-images/filebeat + prometheus: + build: docker-images/prometheus diff --git a/docker-compose.yml b/docker-compose.yml index 4505e47a8..5ea7748d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: restart_policy: condition: on-failure kafka-1: - image: confluentinc/cp-kafka:7.1.1 + image: ghcr.io/opexdev/kafka hostname: kafka-1 volumes: - kafka-1:/var/lib/kafka/data @@ -41,6 +41,7 @@ services: - KAFKA_ADVERTISED_LISTENERS=CLIENT://kafka-1:29092,EXTERNAL://kafka-1:9092 - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false + - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter-0.15.0.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml depends_on: - zookeeper networks: @@ -49,7 +50,7 @@ services: restart_policy: condition: on-failure kafka-2: - image: confluentinc/cp-kafka:7.1.1 + image: ghcr.io/opexdev/kafka hostname: kafka-2 volumes: - kafka-2:/var/lib/kafka/data @@ -61,6 +62,7 @@ services: - KAFKA_ADVERTISED_LISTENERS=CLIENT://kafka-2:29092,EXTERNAL://kafka-2:9092 - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false + - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter-0.15.0.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml depends_on: - zookeeper networks: @@ -69,7 +71,7 @@ services: restart_policy: condition: on-failure kafka-3: - image: confluentinc/cp-kafka:7.1.1 + image: ghcr.io/opexdev/kafka hostname: kafka-3 volumes: - kafka-3:/var/lib/kafka/data @@ -81,6 +83,7 @@ services: - KAFKA_ADVERTISED_LISTENERS=CLIENT://kafka-3:29092,EXTERNAL://kafka-3:9092 - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false + - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter-0.15.0.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml depends_on: - zookeeper networks: @@ -206,6 +209,8 @@ services: - consul - vault - postgres-accountant + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -227,6 +232,8 @@ services: - consul - vault - postgres-eventlog + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -246,6 +253,8 @@ services: - kafka-2 - kafka-3 - redis + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -264,6 +273,8 @@ services: - auth - consul - matching-engine + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -294,6 +305,8 @@ services: - postgres-auth networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -320,6 +333,8 @@ services: - postgres-wallet networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -344,6 +359,8 @@ services: - postgres-market networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -372,6 +389,8 @@ services: - postgres-api networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -394,6 +413,8 @@ services: - postgres-api networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -421,6 +442,8 @@ services: - postgres-bc-gateway networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -438,6 +461,8 @@ services: - consul networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -460,6 +485,8 @@ services: - consul - postgres-referral - vault + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -471,6 +498,8 @@ services: - SWAGGER_AUTH_URL=$KEYCLOAK_FRONTEND_URL depends_on: - consul + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure @@ -494,36 +523,73 @@ services: - vault networks: - default + labels: + collect_logs: "true" deploy: restart_policy: condition: on-failure elasticsearch: - image: elastic/elasticsearch:8.3.2 + image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2 environment: - ES_JAVA_OPTS=-Xms1g -Xmx1g - discovery.type=single-node - - ingest.geoip.downloader.enabled=false - node.name=elasticsearch - network.publish_host=elasticsearch + - xpack.security.enabled=true volumes: - - elasticsearch_data:/usr/share/elasticsearch/data + - elasticsearch-data:/usr/share/elasticsearch/data networks: - default - kibana: - image: elastic/kibana:8.3.2 + logstash: + image: ghcr.io/opexdev/opex-logstash environment: - - SERVER_PUBLICBASEURL=$KIBANA_PUBLIC_URL - - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate - networks: - - default + - LS_JAVA_OPTS=-Xmx256m -Xms256m + - xpack.monitoring.elasticsearch.url=elasticsearch:9200 + - xpack.monitoring.elasticsearch.username=${LOGSTASH_ELASTIC_USER} + - xpack.monitoring.elasticsearch.password=${LOGSTASH_ELASTIC_PASSWORD} + - ELASTIC_USER=${LOGSTASH_ELASTIC_USER} + - ELASTIC_PASSWORD=${LOGSTASH_ELASTIC_PASSWORD} + restart: on-failure + depends_on: + - elasticsearch filebeat: image: ghcr.io/opexdev/opex-filebeat user: root - environment: - - FILEBEAT_API_KEY=$FILEBEAT_API_KEY volumes: - - /var/lib/docker:/var/lib/docker:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - logstash + networks: + - default + kibana: + image: docker.elastic.co/kibana/kibana:8.6.2 + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + - ELASTICSEARCH_USERNAME=${KIBANA_ELASTIC_USER} + - ELASTICSEARCH_PASSWORD=${KIBANA_ELASTIC_PASSWORD} + - XPACK_MONITORING_ENABLED=true + - XPACK_MONITORING_COLLECTION_ENABLED=true + - XPACK_SECURITY_ENABLED=true + networks: + - default + depends_on: + - elasticsearch + prometheus: + image: ghcr.io/opexdev/opex-prometheus + restart: unless-stopped + command: + - '--config.file=/etc/prometheus/prometheus.yaml' + networks: + - default + grafana: + image: grafana/grafana-oss:8.5.2 + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + - GF_SERVER_DOMAIN=localhost networks: - default volumes: @@ -545,7 +611,8 @@ volumes: referral-data: storage-data: admin-data: - elasticsearch_data: + elasticsearch-data: + grafana-data: networks: default: driver: bridge diff --git a/docker-images/filebeat/Dockerfile b/docker-images/filebeat/Dockerfile index 17c6916d4..b9a3cd6cf 100644 --- a/docker-images/filebeat/Dockerfile +++ b/docker-images/filebeat/Dockerfile @@ -1,2 +1,2 @@ -FROM elastic/filebeat:8.3.2 +FROM docker.elastic.co/beats/filebeat:8.6.2 COPY filebeat.yml /usr/share/filebeat/filebeat.yml diff --git a/docker-images/filebeat/filebeat.yml b/docker-images/filebeat/filebeat.yml index ca28f7289..058a3f6f8 100644 --- a/docker-images/filebeat/filebeat.yml +++ b/docker-images/filebeat/filebeat.yml @@ -1,24 +1,25 @@ -filebeat.inputs: - - type: container - paths: - - '/var/lib/docker/containers/*/*.log' +filebeat.autodiscover: + providers: + - type: docker + labels.dedot: true + templates: + - condition: + contains: + container.labels.collect_logs: "true" + config: + - type: container + format: docker + paths: + - "/var/lib/docker/containers/${data.docker.container.id}/*.log" + processors: + - decode_json_fields: + when.equals: + docker.container.labels.decode_log_event_to_json_object: "true" + fields: [ "message" ] + target: "" + overwrite_keys: true -processors: - - add_docker_metadata: - host: "unix:///var/run/docker.sock" +output.logstash: + hosts: "logstash:5044" - - decode_json_fields: - fields: [ "message" ] - target: "json" - overwrite_keys: true - -output.elasticsearch: - hosts: [ "elasticsearch:9200" ] - protocol: https - api_key: ${FILEBEAT_API_KEY} - ssl.verification_mode: "none" - indices: - - index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" - -logging.json: true -logging.metrics.enabled: false +logging.metrics.enabled: false \ No newline at end of file diff --git a/docker-images/kafka/Dockerfile b/docker-images/kafka/Dockerfile new file mode 100644 index 000000000..e3a565f44 --- /dev/null +++ b/docker-images/kafka/Dockerfile @@ -0,0 +1,6 @@ +FROM confluentinc/cp-kafka:7.1.1 +USER root +RUN mkdir /opt/prometheus +RUN chmod +rx /opt/prometheus +COPY jmx-exporter-0.15.0.jar /opt/prometheus +COPY kafka-jmx-exporter.yml /opt/prometheus \ No newline at end of file diff --git a/docker-images/kafka/jmx-exporter-0.15.0.jar b/docker-images/kafka/jmx-exporter-0.15.0.jar new file mode 100644 index 000000000..d896a2170 Binary files /dev/null and b/docker-images/kafka/jmx-exporter-0.15.0.jar differ diff --git a/docker-images/kafka/kafka-jmx-exporter.yml b/docker-images/kafka/kafka-jmx-exporter.yml new file mode 100644 index 000000000..4de5dad60 --- /dev/null +++ b/docker-images/kafka/kafka-jmx-exporter.yml @@ -0,0 +1,192 @@ +--- +startDelaySeconds: 120 +lowercaseOutputName: true +lowercaseOutputLabelNames: true +blacklistObjectNames: + - "kafka.consumer:type=*,id=*" + - "kafka.consumer:type=*,client-id=*" + - "kafka.consumer:type=*,client-id=*,node-id=*" + - "kafka.producer:type=*,id=*" + - "kafka.producer:type=*,client-id=*" + - "kafka.producer:type=*,client-id=*,node-id=*" + - "kafka.*:type=kafka-metrics-count,*" + # This will ignore the admin client metrics from Kafka Brokers and will blacklist certain metrics + # that do not make sense for ingestion. + # "kafka.admin.client:type=*, node-id=*, client-id=*" + # "kafka.admin.client:type=*, client-id=*" + # "kafka.admin.client:type=*, id=*" + - "kafka.admin.client:*" + - "kafka.server:type=*,cipher=*,protocol=*,listener=*,networkProcessor=*" + - "kafka.server:type=*" +rules: + # This is by far the biggest contributor to the number of sheer metrics being produced. + # Always keep it on the top for the case of probability when so many metrics will hit the first condition and exit. + # "kafka.cluster:type=*, name=*, topic=*, partition=*" + # "kafka.log:type=*,name=*, topic=*, partition=*" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + topic: "$4" + partition: "$5" + # "kafka.server:type=*,name=*, client-id=*, topic=*, partition=*" + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + # "kafka.network:type=*, name=*, request=*, error=*" + # "kafka.network:type=*, name=*, request=*, version=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2_$3 + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + # "kafka.rest:type=*, topic=*, partition=*, client-id=*" + # "kafka.rest:type=*, cipher=*, protocol=*, client-id=*" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2 + labels: + "$3": "$4" + "$5": "$6" + "$7": "$8" + # Count and Value + # "kafka.server:type=*, name=*, topic=*" + # "kafka.server:type=*, name=*, clientId=*" + # "kafka.server:type=*, name=*, delayedOperation=*" + # "kafka.server:type=*, name=*, fetcherType=*" + # "kafka.network:type=*, name=*, networkProcessor=*" + # "kafka.network:type=*, name=*, processor=*" + # "kafka.network:type=*, name=*, request=*" + # "kafka.network:type=*, name=*, listener=*" + # "kafka.log:type=*, name=*, logDirectory=*" + # "kafka.log:type=*, name=*, op=*" + # "kafka.rest:type=*, node-id=*, client-id=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2_$3 + labels: + "$4": "$5" + # "kafka.consumer:type=*, topic=*, client-id=*" + # "kafka.producer:type=*, topic=*, client-id=*" + # "kafka.rest:type=*, topic=*, client-id=*" + # "kafka.server:type=*, broker-id=*, fetcher-id=*" + # "kafka.server:type=*, listener=*, networkProcessor=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2 + labels: + "$3": "$4" + "$5": "$6" + # "kafka.network:type=*, name=*" + # "kafka.server:type=*, name=*" + # "kafka.controller:type=*, name=*" + # "kafka.databalancer:type=*, name=*" + # "kafka.log:type=*, name=*" + # "kafka.utils:type=*, name=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2_$3 + # "kafka.producer:type=*, client-id=*" + # "kafka.producer:type=*, id=*" + # "kafka.rest:type=*, client-id=*" + # "kafka.rest:type=*, http-status-code=*" + # "kafka.server:type=*, BrokerId=*" + # "kafka.server:type=*, listener=*" + # "kafka.server:type=*, id=*" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2 + labels: + "$3": "$4" + + - pattern: kafka.server<>OneMinuteRate + name: kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total + type: GAUGE + # "kafka.server:type=*, listener=*, networkProcessor=*, clientSoftwareName=*, clientSoftwareVersion=*" + - pattern: kafka.server<>connections + name: kafka_server_socketservermetrics_connections + type: GAUGE + labels: + client_software_name: "$1" + client_software_version: "$2" + listener: "$3" + network_processor: "$4" + - pattern: "kafka.server<>(.+):" + name: kafka_server_socketservermetrics_$3 + type: GAUGE + labels: + listener: "$1" + network_processor: "$2" + # "kafka.coordinator.group:type=*, name=*" + # "kafka.coordinator.transaction:type=*, name=*" + - pattern: kafka.coordinator.(\w+)<>(Count|Value) + name: kafka_coordinator_$1_$2_$3 + # Percentile + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" + # Additional Rules for Confluent Server Metrics + # 'confluent.metadata:type=*, name=*, topic=*, partition=*' + - pattern: confluent.(\w+)<>Value + name: confluent_$1_$2 + type: GAUGE + labels: + "$3": "$4" + "$5": "$6" + "$7": "$8" + # 'confluent.metadata.service:type=*, node-id=*, client-id=*' + - pattern: confluent.(.+)<>Value + name: confluent_$1_$2 + type: GAUGE + labels: + "$3": "$4" + "$5": "$6" + # 'confluent.metadata.service:type=*, client-id=*' + # 'confluent.metadata.service:type=*, id=*' + # 'confluent.metadata:type=*, name=*' + # 'confluent.license:type=*, name=*' + - pattern: confluent.(.+)<>Value + name: confluent_$1_$2 + type: GAUGE + labels: + "$3": "$4" + + # Quotas + - pattern : 'kafka.server<>(.+):' + name: kafka_server_$1_$4 + type: GAUGE + labels: + user: "$2" + client-id: "$3" + + - pattern : 'kafka.server<>(.+):' + name: kafka_server_$1_$3 + type: GAUGE + labels: + user: "$2" + + - pattern : 'kafka.server<>(.+):' + name: kafka_server_$1_$3 + type: GAUGE + labels: + client-id: "$2" diff --git a/docker-images/logstash/Dockerfile b/docker-images/logstash/Dockerfile new file mode 100644 index 000000000..49d773bb9 --- /dev/null +++ b/docker-images/logstash/Dockerfile @@ -0,0 +1,2 @@ +FROM docker.elastic.co/logstash/logstash:8.6.2 +COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf \ No newline at end of file diff --git a/docker-images/logstash/logstash.conf b/docker-images/logstash/logstash.conf new file mode 100644 index 000000000..e4d29b0fd --- /dev/null +++ b/docker-images/logstash/logstash.conf @@ -0,0 +1,41 @@ +input { + beats { + port => 5044 + } +} + +filter { + grok { + match => {"message" => "%{DATESTAMP:dateTime} \s*%{DATA:logLevel} %{INT:pid} --- \[\s*%{DATA:thread}\] %{DATA:class}: %{GREEDYDATA:msg}" } + } + + mutate { + rename => { "dateTime" => "[logback][dateTime]" } + rename => { "logLevel" => "[logback][logLevel]" } + rename => { "pid" => "[logback][pid]" } + rename => { "thread" => "[logback][thread]" } + rename => { "class" => "[logback][class]" } + rename => { "msg" => "[logback][message]" } + + rename => { "[docker][container][labels][com_docker_compose_container-number]" => "[container][number]" } + rename => { "[docker][container][labels][com_docker_compose_project]" => "[container][compose][project]" } + rename => { "[docker][container][labels][com_docker_compose_service]" => "[container][compose][service]" } + remove_field => [ "docker" ] + + add_tag => [ "logstash_filter_applied" ] + } + + if "_grokparsefailure" in [tags] { + mutate { add_field => { "formatted" => "No" } } + } else { + mutate { add_field => { "formatted" => "Yes" } } + } +} + +output { + elasticsearch { + hosts => "http://elasticsearch:9200" + user => "${ELASTIC_USER}" + password => "${ELASTIC_PASSWORD}" + } +} \ No newline at end of file diff --git a/docker-images/prometheus/Dockerfile b/docker-images/prometheus/Dockerfile new file mode 100644 index 000000000..d66a644f1 --- /dev/null +++ b/docker-images/prometheus/Dockerfile @@ -0,0 +1,2 @@ +FROM prom/prometheus:v2.35.0 +COPY prometheus.yaml /etc/prometheus \ No newline at end of file diff --git a/docker-images/prometheus/prometheus.yaml b/docker-images/prometheus/prometheus.yaml new file mode 100644 index 000000000..cc8dd81f0 --- /dev/null +++ b/docker-images/prometheus/prometheus.yaml @@ -0,0 +1,50 @@ +scrape_configs: + - job_name: "Kafka cluster" + scrape_interval: 5s + static_configs: + - targets: + - "kafka-1:1234" + - "kafka-2:1234" + - "kafka-3:1234" + labels: + env: "dev" + - job_name: 'Spring Boot Application input' + metrics_path: '/actuator/prometheus' + scrape_interval: 5s + static_configs: + - targets: [ 'accountant:8080' ] + labels: + application: 'Accountant' + - targets: [ 'wallet:8080' ] + labels: + application: 'Wallet' + - targets: [ 'api:8080' ] + labels: + application: 'API' + - targets: [ 'bc-gateway:8080' ] + labels: + application: 'Blockchain Gateway' + - targets: [ 'captcha:8080' ] + labels: + application: 'Captcha' + - targets: [ 'market:8080' ] + labels: + application: 'Market' + - targets: [ 'eventlog:8080' ] + labels: + application: 'Eventlog' + - targets: [ 'matching-engine:8080' ] + labels: + application: 'Matching Engine' + - targets: [ 'matching-gateway:8080' ] + labels: + application: 'Matching Gateway' + - targets: [ 'storage:8080' ] + labels: + application: 'Storage' + - targets: [ 'auth:8080' ] + labels: + application: 'Auth' + - targets: [ 'websocket:8080' ] + labels: + application: 'Websocket' \ No newline at end of file diff --git a/eventlog/eventlog-app/pom.xml b/eventlog/eventlog-app/pom.xml index 8eb8aa4d4..d619a363b 100644 --- a/eventlog/eventlog-app/pom.xml +++ b/eventlog/eventlog-app/pom.xml @@ -55,6 +55,11 @@ org.springframework.boot spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/eventlog/eventlog-app/src/main/kotlin/co/nilin/opex/eventlog/app/EventLogApp.kt b/eventlog/eventlog-app/src/main/kotlin/co/nilin/opex/eventlog/app/EventLogApp.kt index eb6111caf..2eb345511 100644 --- a/eventlog/eventlog-app/src/main/kotlin/co/nilin/opex/eventlog/app/EventLogApp.kt +++ b/eventlog/eventlog-app/src/main/kotlin/co/nilin/opex/eventlog/app/EventLogApp.kt @@ -3,9 +3,11 @@ package co.nilin.opex.eventlog.app import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling @SpringBootApplication @ComponentScan("co.nilin.opex") +@EnableScheduling class EventLogApp fun main(args: Array) { diff --git a/eventlog/eventlog-app/src/main/kotlin/co/nilin/opex/eventlog/app/utils/PrometheusHealthExtension.kt b/eventlog/eventlog-app/src/main/kotlin/co/nilin/opex/eventlog/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..b455d93b6 --- /dev/null +++ b/eventlog/eventlog-app/src/main/kotlin/co/nilin/opex/eventlog/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.eventlog.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "EVENTLOG" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/eventlog/eventlog-app/src/main/resources/application.yml b/eventlog/eventlog-app/src/main/resources/application.yml index 5289bf242..4132715e8 100644 --- a/eventlog/eventlog-app/src/main/resources/application.yml +++ b/eventlog/eventlog-app/src/main/resources/application.yml @@ -28,4 +28,17 @@ spring: profile-separator: '/' application-name: ${spring.application.name} config: - import: vault://secret/${spring.application.name} \ No newline at end of file + import: vault://secret/${spring.application.name} +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true \ No newline at end of file diff --git a/market/market-app/pom.xml b/market/market-app/pom.xml index a264a8522..a867c9541 100644 --- a/market/market-app/pom.xml +++ b/market/market-app/pom.xml @@ -88,6 +88,11 @@ co.nilin.opex.utility.preferences preferences + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/market/market-app/src/main/kotlin/co/nilin/opex/market/app/MarketAppApplication.kt b/market/market-app/src/main/kotlin/co/nilin/opex/market/app/MarketAppApplication.kt index 251dfcc6b..c8413c59d 100644 --- a/market/market-app/src/main/kotlin/co/nilin/opex/market/app/MarketAppApplication.kt +++ b/market/market-app/src/main/kotlin/co/nilin/opex/market/app/MarketAppApplication.kt @@ -3,9 +3,11 @@ package co.nilin.opex.market.app import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling @SpringBootApplication @ComponentScan("co.nilin.opex") +@EnableScheduling class MarketAppApplication fun main(args: Array) { diff --git a/market/market-app/src/main/kotlin/co/nilin/opex/market/app/utils/PrometheusHealthExtension.kt b/market/market-app/src/main/kotlin/co/nilin/opex/market/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..90fe2b630 --- /dev/null +++ b/market/market-app/src/main/kotlin/co/nilin/opex/market/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,66 @@ +package co.nilin.opex.market.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.context.annotation.Profile +import org.springframework.scheduling.annotation.EnableScheduling +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +@Profile("scheduled") +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "MARKET" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/market/market-app/src/main/resources/application.yml b/market/market-app/src/main/resources/application.yml index 776147a09..cb8c5ac61 100644 --- a/market/market-app/src/main/resources/application.yml +++ b/market/market-app/src/main/resources/application.yml @@ -43,6 +43,19 @@ spring: prefer-ip-address: true config: import: vault://secret/${spring.application.name} +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true app: auth: cert-url: lb://opex-auth/auth/realms/opex/protocol/openid-connect/certs diff --git a/matching-engine/matching-engine-app/pom.xml b/matching-engine/matching-engine-app/pom.xml index d70d096d6..0b3c52b29 100644 --- a/matching-engine/matching-engine-app/pom.xml +++ b/matching-engine/matching-engine-app/pom.xml @@ -51,6 +51,11 @@ co.nilin.opex.utility.preferences preferences + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/MatchingEngineApp.kt b/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/MatchingEngineApp.kt index bacefbbe5..bf4fb80b1 100644 --- a/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/MatchingEngineApp.kt +++ b/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/MatchingEngineApp.kt @@ -3,9 +3,11 @@ package co.nilin.opex.matching.engine.app import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling @SpringBootApplication @ComponentScan("co.nilin.opex") +@EnableScheduling class MatchingEngineApp fun main(args: Array) { diff --git a/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/utils/PrometheusHealthExtension.kt b/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..e0e4515f8 --- /dev/null +++ b/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.matching.engine.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "MATCHING_ENGINE" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/matching-engine/matching-engine-app/src/main/resources/application.yml b/matching-engine/matching-engine-app/src/main/resources/application.yml index 694e6c8a0..cd493c173 100644 --- a/matching-engine/matching-engine-app/src/main/resources/application.yml +++ b/matching-engine/matching-engine-app/src/main/resources/application.yml @@ -10,3 +10,16 @@ spring: redis: host: ${REDIS_HOST:localhost} port: 6379 +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true diff --git a/matching-gateway/matching-gateway-app/pom.xml b/matching-gateway/matching-gateway-app/pom.xml index a23da041b..552cfefcd 100644 --- a/matching-gateway/matching-gateway-app/pom.xml +++ b/matching-gateway/matching-gateway-app/pom.xml @@ -77,6 +77,11 @@ io.mockk mockk + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/matching-gateway/matching-gateway-app/src/main/kotlin/co/nilin/opex/matching/gateway/app/utils/PrometheusHealthExtension.kt b/matching-gateway/matching-gateway-app/src/main/kotlin/co/nilin/opex/matching/gateway/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..38ee9bc74 --- /dev/null +++ b/matching-gateway/matching-gateway-app/src/main/kotlin/co/nilin/opex/matching/gateway/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.matching.gateway.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "MATCHING_GATEWAY" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/matching-gateway/matching-gateway-app/src/main/resources/application.yml b/matching-gateway/matching-gateway-app/src/main/resources/application.yml index ab9e6a21d..558d4890f 100644 --- a/matching-gateway/matching-gateway-app/src/main/resources/application.yml +++ b/matching-gateway/matching-gateway-app/src/main/resources/application.yml @@ -23,6 +23,19 @@ spring: instance-id: ${spring.application.name}:${server.port} healthCheckInterval: 20s prefer-ip-address: true +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true swagger.authUrl: ${SWAGGER_AUTH_URL:https://api.opex.dev/auth}/realms/opex/protocol/openid-connect/token app: accountant: diff --git a/storage/storage-app/pom.xml b/storage/storage-app/pom.xml index ca0ec9df8..be4dbbfdd 100644 --- a/storage/storage-app/pom.xml +++ b/storage/storage-app/pom.xml @@ -68,6 +68,11 @@ springfox-boot-starter 3.0.0 + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/storage/storage-app/src/main/kotlin/co/nilin/opex/storage/app/StorageApp.kt b/storage/storage-app/src/main/kotlin/co/nilin/opex/storage/app/StorageApp.kt index 3a469332f..4ba3a4ab4 100644 --- a/storage/storage-app/src/main/kotlin/co/nilin/opex/storage/app/StorageApp.kt +++ b/storage/storage-app/src/main/kotlin/co/nilin/opex/storage/app/StorageApp.kt @@ -4,11 +4,13 @@ import co.nilin.opex.utility.error.EnableOpexErrorHandler import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling import springfox.documentation.swagger2.annotations.EnableSwagger2 @SpringBootApplication @ComponentScan("co.nilin.opex") @EnableOpexErrorHandler +@EnableScheduling class StorageApp fun main(args: Array) { diff --git a/storage/storage-app/src/main/kotlin/co/nilin/opex/storage/app/utils/PrometheusHealthExtension.kt b/storage/storage-app/src/main/kotlin/co/nilin/opex/storage/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..61118ffbe --- /dev/null +++ b/storage/storage-app/src/main/kotlin/co/nilin/opex/storage/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.storage.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "STORAGE" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/storage/storage-app/src/main/resources/application.yml b/storage/storage-app/src/main/resources/application.yml index d2669ffe3..c1a77cf60 100644 --- a/storage/storage-app/src/main/resources/application.yml +++ b/storage/storage-app/src/main/resources/application.yml @@ -21,6 +21,19 @@ spring: instance-id: ${spring.application.name}:${server.port} healthCheckInterval: 20s prefer-ip-address: true +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true app: root-dir: ${ROOT_DIR} diff --git a/user-management/keycloak-gateway/pom.xml b/user-management/keycloak-gateway/pom.xml index eae08a7a1..f86de02b1 100644 --- a/user-management/keycloak-gateway/pom.xml +++ b/user-management/keycloak-gateway/pom.xml @@ -106,6 +106,11 @@ spring-kafka-test test + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/user-management/keycloak-gateway/src/main/kotlin/co/nilin/opex/auth/gateway/KeycloakGatewayApp.kt b/user-management/keycloak-gateway/src/main/kotlin/co/nilin/opex/auth/gateway/KeycloakGatewayApp.kt index a9edbf388..7dad417e6 100644 --- a/user-management/keycloak-gateway/src/main/kotlin/co/nilin/opex/auth/gateway/KeycloakGatewayApp.kt +++ b/user-management/keycloak-gateway/src/main/kotlin/co/nilin/opex/auth/gateway/KeycloakGatewayApp.kt @@ -6,11 +6,13 @@ import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfigurati import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling @SpringBootApplication(exclude = [LiquibaseAutoConfiguration::class]) @ComponentScan("co.nilin.opex") @EnableOpexErrorHandler @EnableConfigurationProperties +@EnableScheduling class KeycloakGatewayApp fun main(args: Array) { diff --git a/user-management/keycloak-gateway/src/main/kotlin/co/nilin/opex/auth/gateway/utils/PrometheusHealthExtension.kt b/user-management/keycloak-gateway/src/main/kotlin/co/nilin/opex/auth/gateway/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..79770e81b --- /dev/null +++ b/user-management/keycloak-gateway/src/main/kotlin/co/nilin/opex/auth/gateway/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.auth.gateway.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "AUTH" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/user-management/keycloak-gateway/src/main/resources/application.yml b/user-management/keycloak-gateway/src/main/resources/application.yml index 0a1e3e74c..463fff843 100644 --- a/user-management/keycloak-gateway/src/main/resources/application.yml +++ b/user-management/keycloak-gateway/src/main/resources/application.yml @@ -50,6 +50,19 @@ spring: prefer-ip-address: true config: import: vault://secret/${spring.application.name} +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true keycloak: adminUrl: ${ADMIN_URL:http://localhost:8080/auth} frontendUrl: ${FRONTEND_URL:http://localhost:8080/auth} diff --git a/wallet/wallet-app/pom.xml b/wallet/wallet-app/pom.xml index 5acbbd196..516dff0e4 100644 --- a/wallet/wallet-app/pom.xml +++ b/wallet/wallet-app/pom.xml @@ -111,6 +111,11 @@ co.nilin.opex.utility.preferences preferences + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/WalletApp.kt b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/WalletApp.kt index 24f105e0b..f5f5f2e93 100644 --- a/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/WalletApp.kt +++ b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/WalletApp.kt @@ -4,10 +4,12 @@ import co.nilin.opex.utility.error.EnableOpexErrorHandler import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling @SpringBootApplication @ComponentScan("co.nilin.opex") @EnableOpexErrorHandler +@EnableScheduling class WalletApp fun main(args: Array) { diff --git a/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/utils/PrometheusHealthExtension.kt b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/utils/PrometheusHealthExtension.kt new file mode 100644 index 000000000..02dc4fcfc --- /dev/null +++ b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/utils/PrometheusHealthExtension.kt @@ -0,0 +1,63 @@ +package co.nilin.opex.wallet.app.utils + +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.actuate.health.HealthComponent +import org.springframework.boot.actuate.health.HealthEndpoint +import org.springframework.boot.actuate.health.SystemHealth +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component + +@Component +class PrometheusHealthExtension( + private val registry: MeterRegistry, + private val endpoint: HealthEndpoint +) { + + private var consulHealth = -1 + private var r2dbcHealth = -1 + private var vaultHealth = -1 + private var vaultReactiveHealth = -1 + private val service = "WALLET" + + init { + Gauge.builder("consul_health", consulHealth) { consulHealth.toDouble() } + .description("Health of consul connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("r2dbc_health", r2dbcHealth) { r2dbcHealth.toDouble() } + .description("Health of r2dbc connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vault_health", vaultHealth) { vaultHealth.toDouble() } + .description("Health of vault connection") + .tag("Service", service) + .register(registry) + + Gauge.builder("vaultReactive_health", vaultReactiveHealth) { vaultReactiveHealth.toDouble() } + .description("Health of vaultReactive connection") + .tag("Service", service) + .register(registry) + } + + @Scheduled(initialDelay = 1000, fixedDelay = 5000) + fun updateHealth() { + try { + val health = endpoint.health() as SystemHealth + consulHealth = getHealthValue(health.components["consul"]) + r2dbcHealth = getHealthValue(health.components["r2dbc"]) + vaultHealth = getHealthValue(health.components["vault"]) + vaultReactiveHealth = getHealthValue(health.components["vaultReactive"]) + } catch (e: Exception) { + e.printStackTrace() + } + } + + private fun getHealthValue(health: HealthComponent?): Int { + health ?: return -1 + return if (health.status.code == "UP") 1 else 0 + } + +} \ No newline at end of file diff --git a/wallet/wallet-app/src/main/resources/application.yml b/wallet/wallet-app/src/main/resources/application.yml index 87eecd1f2..95fb7eb9f 100644 --- a/wallet/wallet-app/src/main/resources/application.yml +++ b/wallet/wallet-app/src/main/resources/application.yml @@ -1,4 +1,17 @@ server.port: 8080 +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true spring: jackson: serialization: diff --git a/websocket/websocket-app/pom.xml b/websocket/websocket-app/pom.xml index bbe08610b..878f99149 100644 --- a/websocket/websocket-app/pom.xml +++ b/websocket/websocket-app/pom.xml @@ -80,6 +80,11 @@ reactor-test test + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/websocket/websocket-app/src/main/resources/application.yml b/websocket/websocket-app/src/main/resources/application.yml index 9c3d6d93e..f38ee2589 100644 --- a/websocket/websocket-app/src/main/resources/application.yml +++ b/websocket/websocket-app/src/main/resources/application.yml @@ -40,6 +40,19 @@ spring: prefer-ip-address: true config: import: vault://secret/${spring.application.name} +management: + endpoints: + web: + base-path: /actuator + exposure: + include: ["health", "prometheus", "metrics"] + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true app: address: 1 auth: