diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a58e36ecb962..968024394f25b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -711,7 +711,10 @@ repos: language: python pass_filenames: true files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$ - exclude: ^scripts/ci/docker-compose/grafana/.|^scripts/ci/docker-compose/prometheus.yml + exclude: > + (?x) + ^scripts/ci/docker-compose/grafana/.| + ^scripts/ci/docker-compose/.+-config\.ya?ml require_serial: true additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0'] - id: lint-json-schema diff --git a/BREEZE.rst b/BREEZE.rst index 82ade6c028359..ffbb5fa2e1a48 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -569,9 +569,11 @@ Those are all available flags of ``shell`` command: :width: 100% :alt: Breeze shell +Running Breeze with Metrics +--------------------------- Running Breeze with a StatsD Metrics Stack ------------------------------------------- +.......................................... You can launch an instance of Breeze pre-configured to emit StatsD metrics using ``breeze start-airflow --integration statsd``. This will launch an Airflow webserver @@ -592,6 +594,37 @@ You can connect to these ports/databases using: * Prometheus Targets: http://127.0.0.1:29090/targets * Grafana Dashboards: http://127.0.0.1:23000/dashboards +Running Breeze with an OpenTelemetry Metrics Stack +.................................................. + +---- + +[Work in Progress] +NOTE: This will launch the stack as described below but Airflow integration is +still a Work in Progress. This should be considered experimental and likely to +change by the time Airflow fully supports emitting metrics via OpenTelemetry. + +---- + +You can launch an instance of Breeze pre-configured to emit OTel metrics using +``breeze start-airflow --integration otel``. This will launch an Airflow webserver +within the Breeze environment as well as containers running OpenTelemetry-Collector, +Prometheus, and Grafana. The integration configures the "Targets" in Prometheus, +the "Datasources" in Grafana, and includes a default dashboard in Grafana. + +When you run Airflow Breeze with this integration, in addition to the standard ports +(See "Port Forwarding" below), the following are also automatically forwarded: + +* 28888 -> forwarded to OpenTelemetry Collector -> breeze-otel_collector:8888 +* 29090 -> forwarded to Prometheus -> breeze-prometheus:9090 +* 23000 -> forwarded to Grafana -> breeze-grafana:3000 + +You can connect to these ports/databases using: + +* OpenTelemetry Collector: http://127.0.0.1:28888/metrics +* Prometheus Targets: http://127.0.0.1:29090/targets +* Grafana Dashboards: http://127.0.0.1:23000/dashboards + Stopping the environment ------------------------ diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 7b862d552ea02..589c51af4631f 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -54,6 +54,7 @@ [ *ALL_INTEGRATIONS, "all", + "otel", "statsd", ] ) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index eeda29963e285..874a992d15c0f 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,7 +1,7 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:2b27f16d64b13766bdbc9e7979f75601 +main:2e1b4e5838838f513750d1352798b006 build-docs:18235f12f85f8df82f3eb245e429f62d ci:find-newer-dependencies:8fa2b57f5f0523c928743b235ee3ab5a ci:fix-ownership:fee2c9ec9ef19686792002ae054fecdd @@ -53,12 +53,12 @@ setup:regenerate-command-images:15215e52342dd2f2e27a85726f40a820 setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:123b462a421884dc2320ffc5e54b2478 setup:56a2ef337c354362760d247df5d05365 -shell:42c48eb6f0b9d2b5e3e7602de2600341 -start-airflow:91a1c8ef99573266e669acb678aaa857 +shell:e34d2512360503b70e6a05e5b8b59986 +start-airflow:e096fa493a810935c012ef3d95775aa2 static-checks:12e8fed2acbed0d823efc5121fd0eb58 stop:e5aa686b4e53707ced4039d8414d5cd6 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c testing:helm-tests:94a442e7f3f63b34c4831a84d165690a -testing:integration-tests:eee580f73de750c1229c5f2b0d9e500f -testing:tests:200916b63e40d9c72b58236c925b2589 -testing:3627d67696a9a2a31aec76ec745f7004 +testing:integration-tests:fa4f9be38ee380f6f8feeb9dd8ba7669 +testing:tests:8dd0496fddb37b207fc2b74add201d43 +testing:1e4ea0f58b4814342911355d35efd966 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 8bd2e15f6152f..816cc9693af74 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -196,8 +196,8 @@ │--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ │--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ │--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).         │ -│(all | cassandra | celery | kerberos | mongo | pinot | statsd | trino)│ +│--integrationIntegration(s) to enable when running (can be more than one).                │ +│(all | cassandra | celery | kerberos | mongo | otel | pinot | statsd | trino)│ │--forward-credentials-fForward local credentials to container when running.│ │--db-reset-dReset DB when entering the container.│ │--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index 3f2ac3e9a47e2..d5713bbe3fb82 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -222,8 +222,8 @@ │--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ │--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ │--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).         │ -│(all | cassandra | celery | kerberos | mongo | pinot | statsd | trino)│ +│--integrationIntegration(s) to enable when running (can be more than one).                │ +│(all | cassandra | celery | kerberos | mongo | otel | pinot | statsd | trino)│ │--forward-credentials-fForward local credentials to container when running.│ │--db-reset-dReset DB when entering the container.│ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ diff --git a/images/breeze/output_start-airflow.svg b/images/breeze/output_start-airflow.svg index 77713ad9e6e50..f77a3c196c0c9 100644 --- a/images/breeze/output_start-airflow.svg +++ b/images/breeze/output_start-airflow.svg @@ -245,8 +245,8 @@ │--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ │--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ │--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).         │ -│(all | cassandra | celery | kerberos | mongo | pinot | statsd | trino)│ +│--integrationIntegration(s) to enable when running (can be more than one).                │ +│(all | cassandra | celery | kerberos | mongo | otel | pinot | statsd | trino)│ │--forward-credentials-fForward local credentials to container when running.│ │--db-reset-dReset DB when entering the container.│ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ diff --git a/images/breeze/output_testing_integration-tests.svg b/images/breeze/output_testing_integration-tests.svg index 351264fa3e5e6..f3a81ae1b25c0 100644 --- a/images/breeze/output_testing_integration-tests.svg +++ b/images/breeze/output_testing_integration-tests.svg @@ -159,8 +159,8 @@ Run the specified integratio tests. ╭─ Basic flag for integration tests command ───────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).         │ -│(all | cassandra | celery | kerberos | mongo | pinot | statsd | trino)│ +│--integrationIntegration(s) to enable when running (can be more than one).                │ +│(all | cassandra | celery | kerberos | mongo | otel | pinot | statsd | trino)│ │--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ │(INTEGER RANGE)                                                                  │ │[default: 60; x>=0]                                                              │ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index 4514c5c8bc3e5..910691fe423cd 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -215,8 +215,8 @@ │--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ │--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ │--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).         │ -│(all | cassandra | celery | kerberos | mongo | pinot | statsd | trino)│ +│--integrationIntegration(s) to enable when running (can be more than one).                │ +│(all | cassandra | celery | kerberos | mongo | otel | pinot | statsd | trino)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ │--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ diff --git a/scripts/ci/docker-compose/integration-otel.yml b/scripts/ci/docker-compose/integration-otel.yml new file mode 100644 index 0000000000000..96e214c18069e --- /dev/null +++ b/scripts/ci/docker-compose/integration-otel.yml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +version: "3.7" +services: + otel-collector: + image: otel/opentelemetry-collector:0.70.0 + container_name: "breeze-opentelemetry-collector" + ports: + - "1888:1888" # pprof extension + - "28888:8888" # Prometheus metrics exposed by the collector + - "28889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP http receiver + - "55679:55679" # zpages extension + + prometheus: + image: prom/prometheus + container_name: "breeze-prometheus" + user: "0" + ports: + - "29090:9090" + volumes: + - ./prometheus-config.yml:/etc/prometheus/prometheus.yml + - ./prometheus/volume:/prometheus + + grafana: + image: grafana/grafana:8.2.4 + container_name: "breeze-grafana" + environment: + GF_AUTH_ANONYMOUS_ENABLED: true + GF_AUTH_ANONYMOUS_ORG_NAME: "Main Org." + GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" + GF_PATHS_PROVISIONING: /grafana/provisioning + ports: + - "23000:3000" + volumes: + - ./grafana/volume/data:/grafana + - ./grafana/volume/datasources:/grafana/datasources + - ./grafana/volume/dashboards:/grafana/dashboards + - ./grafana/volume/provisioning:/grafana/provisioning + + airflow: + # Environment Variables will need to be set in order to configure + # Breeze to emit the metrics. See: integration-statsd.yml + depends_on: + - otel-collector + - prometheus + - grafana diff --git a/scripts/ci/docker-compose/integration-statsd.yml b/scripts/ci/docker-compose/integration-statsd.yml index 317eb45b9d285..415e473efd9f4 100644 --- a/scripts/ci/docker-compose/integration-statsd.yml +++ b/scripts/ci/docker-compose/integration-statsd.yml @@ -32,7 +32,7 @@ services: ports: - "29090:9090" volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./prometheus-config.yml:/etc/prometheus/prometheus.yml - ./prometheus/volume:/prometheus grafana: diff --git a/scripts/ci/docker-compose/otel-collector-config.yml b/scripts/ci/docker-compose/otel-collector-config.yml new file mode 100644 index 0000000000000..ce84b7798ce28 --- /dev/null +++ b/scripts/ci/docker-compose/otel-collector-config.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +extensions: + memory_ballast: + size_mib: 512 + zpages: + endpoint: 0.0.0.0:55679 + +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + memory_limiter: + limit_mib: 1536 + spike_limit_mib: 512 + check_interval: 5s + +exporters: + logging: + logLevel: debug + prometheus: + metrics: + endpoint: "http://localhost:29090" + +service: + extensions: [memory_ballast, zpages] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [logging] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [logging, prometheus] diff --git a/scripts/ci/docker-compose/prometheus.yml b/scripts/ci/docker-compose/prometheus-config.yml similarity index 86% rename from scripts/ci/docker-compose/prometheus.yml rename to scripts/ci/docker-compose/prometheus-config.yml index 1760f1f26e961..09e55a20c8e93 100644 --- a/scripts/ci/docker-compose/prometheus.yml +++ b/scripts/ci/docker-compose/prometheus-config.yml @@ -16,7 +16,7 @@ # under the License. --- # Configuration for the Prometheus targets. -# Used by integration-statsd.yml +# Used by integration-statsd.yml and integration-otel.yml global: scrape_interval: 30s evaluation_interval: 30s @@ -30,6 +30,9 @@ scrape_configs: - job_name: 'statsd-exporter' static_configs: - targets: ['breeze-statsd-exporter:9102'] - tls_config: insecure_skip_verify: true + + - job_name: 'opentelemetry-collector' + static_configs: + - targets: ['breeze-opentelemetry-collector:8888']