From b37f5c8f613ea7a790a3f3e11e5072591fb62b19 Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Tue, 11 Feb 2025 13:01:41 +0530 Subject: [PATCH 1/3] fix: wait for healthy service --- ...er-compose-distributed-test-with-kafka.yaml | 15 ++++++++++----- docker-compose-distributed-test.yaml | 18 ++++++++++++------ docker-compose-test-with-kafka.yaml | 12 ++++++++---- docker-compose-test.yaml | 6 ++++-- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/docker-compose-distributed-test-with-kafka.yaml b/docker-compose-distributed-test-with-kafka.yaml index ac042d523..18443e067 100644 --- a/docker-compose-distributed-test-with-kafka.yaml +++ b/docker-compose-distributed-test-with-kafka.yaml @@ -11,8 +11,10 @@ services: volumes: - ./parseable-ingest-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro depends_on: - - parseable-ingest-one - - parseable-ingest-two + parseable-ingest-one: + condition: service_healthy + parseable-ingest-two: + condition: service_healthy networks: - parseable-internal healthcheck: @@ -210,9 +212,12 @@ services: networks: - parseable-internal depends_on: - - parseable-query - - parseable-ingest-haproxy - - minio + parseable-query: + condition: service_healthy + parseable-ingest-haproxy: + condition: service_healthy + minio: + condition: service_healthy deploy: restart_policy: condition: on-failure diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index 6f3fe33d1..20e2d4327 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -53,7 +53,8 @@ services: timeout: 20s retries: 5 depends_on: - - minio + minio: + condition: service_healthy deploy: restart_policy: condition: on-failure @@ -89,8 +90,10 @@ services: timeout: 20s retries: 5 depends_on: - - parseable-query - - minio + parseable-query: + condition: service_healthy + minio: + condition: service_healthy deploy: restart_policy: condition: on-failure @@ -121,9 +124,12 @@ services: networks: - parseable-internal depends_on: - - parseable-query - - parseable-ingest-one - - minio + parseable-query: + condition: service_healthy + parseable-ingest-one: + condition: service_healthy + minio: + condition: service_healthy deploy: restart_policy: condition: on-failure diff --git a/docker-compose-test-with-kafka.yaml b/docker-compose-test-with-kafka.yaml index 39f3ebc74..faa68a038 100644 --- a/docker-compose-test-with-kafka.yaml +++ b/docker-compose-test-with-kafka.yaml @@ -49,8 +49,10 @@ services: - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092 # additional settings like security, tuning, etc. depends_on: - - minio - - kafka-0 + minio: + condition: service_healthy + kafka-0: + condition: service_healthy healthcheck: test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] interval: 15s @@ -82,7 +84,8 @@ services: "parseable" ] depends_on: - - parseable + parseable: + condition: service_healthy networks: - parseable-internal deploy: @@ -153,7 +156,8 @@ services: - LOG_RATE=5000 - TOTAL_LOGS=500_000 depends_on: - - kafka-0 + kafka-0: + condition: service_healthy networks: - parseable-internal deploy: diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml index cb5e4de88..f03ca1d01 100644 --- a/docker-compose-test.yaml +++ b/docker-compose-test.yaml @@ -44,7 +44,8 @@ services: - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy depends_on: - - minio + minio: + condition: service_healthy healthcheck: test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ] interval: 15s @@ -76,7 +77,8 @@ services: "parseable" ] depends_on: - - parseable + parseable: + condition: service_healthy networks: - parseable-internal deploy: From c02fe9d7c424456ed752cb1a7f91c74fa52f6afc Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Tue, 11 Feb 2025 16:38:16 +0530 Subject: [PATCH 2/3] fix: docker liveness check requires curl --- Dockerfile.debug | 2 ++ docker-compose-distributed-test-with-kafka.yaml | 2 +- docker-compose-distributed-test.yaml | 4 ++-- docker-compose-test-with-kafka.yaml | 2 +- docker-compose-test.yaml | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile.debug b/Dockerfile.debug index 1d21291f0..c58220408 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -34,6 +34,8 @@ RUN cargo build # final stage FROM docker.io/debian:bookworm-slim +RUN apt update && apt install -y curl + WORKDIR /parseable # Copy the static binary into the final image diff --git a/docker-compose-distributed-test-with-kafka.yaml b/docker-compose-distributed-test-with-kafka.yaml index 18443e067..03d6bff81 100644 --- a/docker-compose-distributed-test-with-kafka.yaml +++ b/docker-compose-distributed-test-with-kafka.yaml @@ -57,7 +57,7 @@ services: parseable-query: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.debug platform: linux/amd64 command: [ "parseable", "s3-store" ] ports: diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index 20e2d4327..b875340af 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -28,7 +28,7 @@ services: parseable-query: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.debug platform: linux/amd64 command: [ "parseable", "s3-store" ] ports: @@ -64,7 +64,7 @@ services: parseable-ingest-one: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.debug platform: linux/amd64 command: [ "parseable", "s3-store", ] ports: diff --git a/docker-compose-test-with-kafka.yaml b/docker-compose-test-with-kafka.yaml index faa68a038..0b0010124 100644 --- a/docker-compose-test-with-kafka.yaml +++ b/docker-compose-test-with-kafka.yaml @@ -149,7 +149,7 @@ services: kafka-log-generator: build: context: ./scripts - dockerfile: Dockerfile + dockerfile: Dockerfile.debug environment: - KAFKA_BROKERS=kafka-0:9092 - KAFKA_TOPIC=test-logs-stream diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml index f03ca1d01..8f1e9ced2 100644 --- a/docker-compose-test.yaml +++ b/docker-compose-test.yaml @@ -27,7 +27,7 @@ services: parseable: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.debug platform: linux/amd64 command: [ "parseable", "s3-store", ] ports: From 5e8ac09b298a662a3dbc8c6ca79e36bcaa5b175a Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Tue, 11 Feb 2025 16:51:32 +0530 Subject: [PATCH 3/3] log: less --- docker-compose-distributed-test-with-kafka.yaml | 1 + docker-compose-distributed-test.yaml | 2 ++ docker-compose-test.yaml | 1 + 3 files changed, 4 insertions(+) diff --git a/docker-compose-distributed-test-with-kafka.yaml b/docker-compose-distributed-test-with-kafka.yaml index 03d6bff81..d7500e2ad 100644 --- a/docker-compose-distributed-test-with-kafka.yaml +++ b/docker-compose-distributed-test-with-kafka.yaml @@ -74,6 +74,7 @@ services: - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=query + - RUST_LOG=warn networks: - parseable-internal healthcheck: diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml index b875340af..b1e4fef72 100644 --- a/docker-compose-distributed-test.yaml +++ b/docker-compose-distributed-test.yaml @@ -45,6 +45,7 @@ services: - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=query + - RUST_LOG=warn networks: - parseable-internal healthcheck: @@ -82,6 +83,7 @@ services: - P_PARQUET_COMPRESSION_ALGO=snappy - P_MODE=ingest - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000 + - RUST_LOG=warn networks: - parseable-internal healthcheck: diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml index 8f1e9ced2..6aaf915da 100644 --- a/docker-compose-test.yaml +++ b/docker-compose-test.yaml @@ -43,6 +43,7 @@ services: - P_PASSWORD=parseableadmin - P_CHECK_UPDATE=false - P_PARQUET_COMPRESSION_ALGO=snappy + - RUST_LOG=warn depends_on: minio: condition: service_healthy