diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 936a7e7b8a7..7b00254aed7 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -65,38 +65,37 @@ jobs: --health-retries 5 kafka-broker: - image: confluentinc/cp-kafka:7.2.1 + image: confluentinc/confluent-local:${CP_VERSION:-7.4.11} ports: - "9092:9092" - "9101:9101" env: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://localhost:9092 + # Mode KRaft (Single Node) + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@localhost:9093' + KAFKA_LOG4J_LOGGERS: "org.apache.kafka.image.loader.MetadataLoader=WARN" + + # Listeners + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT' + KAFKA_LISTENERS: 'EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093' + KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_INTER_BROKER_LISTENER_NAME: 'EXTERNAL' + + # Configuration simplifiée KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_JMX_PORT: 9101 - KAFKA_JMX_HOSTNAME: localhost - KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - options: >- - --health-cmd "cub kafka-ready -b localhost:9092 1 5" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + + # ID du Cluster (Nécessaire pour KRaft) + CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' - zookeeper: - image: confluentinc/cp-zookeeper:7.2.1 - ports: - - "2181:2181" - env: KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 + options: >- - --health-cmd "cub zk-ready localhost:2181 5" + --health-cmd "cub kafka-ready -b localhost:9092 1 5" --health-interval 10s --health-timeout 5s --health-retries 5 diff --git a/docker-compose.yml b/docker-compose.yml index 149e44890f0..62ef0f5ffca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,7 @@ # Notes on image versions: # - For the key services such as postgres and pulsar we are trying to run # against the oldest supported version -# - For the zookeeper and kafka we are trying to use the oldest supported -# version that has arm64 images +# - For kafka we use the oldest version that supports KRaft. # - For everything else we are trying to run against the latest version. # # To run against the latest image versions update .env file. See .env.example @@ -74,67 +73,59 @@ services: # The oldest version with arm64 docker images. EOL May 2 2025 image: apachepulsar/pulsar:${PULSAR_VERSION:-3.0.0} container_name: pulsar-broker - command: bin/pulsar standalone + command: bin/pulsar standalone --no-functions-worker ports: - "${MAP_HOST_PULSAR:-127.0.0.1}:6650:6650" - "${MAP_HOST_PULSAR:-127.0.0.1}:8081:8080" environment: PULSAR_MEM: "-Xms384M -Xmx384M" + # Disable functions worker to save memory/time + PULSAR_PREFIX_functionsWorkerEnabled: "false" profiles: - all - pulsar kafka-broker: - # The oldest supported version with arm64 docker images. EOL October 27, 2023 - image: confluentinc/cp-kafka:${CP_VERSION:-7.0.9} + image: confluentinc/confluent-local:${CP_VERSION:-7.4.11} container_name: kafka-broker - depends_on: - - zookeeper ports: - "${MAP_HOST_KAFKA:-127.0.0.1}:9092:9092" - "${MAP_HOST_KAFKA:-127.0.0.1}:9101:9101" - - "${MAP_HOST_KAFKA:-127.0.0.1}:29092:29092" profiles: - all - kafka - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT_HOST://localhost:9092,PLAINTEXT://kafka-broker:29092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_JMX_PORT: 9101 - KAFKA_JMX_HOSTNAME: localhost - KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - healthcheck: - test: ["CMD", "cub", "kafka-ready", "-b", "localhost:9092", "1", "5"] - start_period: 5s - interval: 5s - timeout: 10s - retries: 100 - zookeeper: - # The oldest supported version with arm64 images. EOL October 27, 2023 - image: confluentinc/cp-zookeeper:${CP_VERSION:-7.0.9} - container_name: zookeeper - ports: - - "${MAP_HOST_ZOOKEEPER:-127.0.0.1}:2181:2181" - profiles: - - all - - kafka environment: - KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 + # Mode KRaft (Single Node) + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@localhost:9093' + KAFKA_LOG4J_LOGGERS: "org.apache.kafka.image.loader.MetadataLoader=WARN" + + # Listeners + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT' + KAFKA_LISTENERS: 'EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093' + KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_INTER_BROKER_LISTENER_NAME: 'EXTERNAL' + + # Configuration simplifiée + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + + # ID du Cluster (Nécessaire pour KRaft) + CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' + + KAFKA_HEAP_OPTS: -Xms256M -Xmx256M healthcheck: - test: ["CMD", "cub", "zk-ready", "localhost:2181", "5"] - start_period: 5s - interval: 5s - timeout: 10s - retries: 100 + # test: ["CMD-SHELL", "nc -z localhost 9092 || exit 1"] + test: ["CMD", "cub", "kafka-ready", "-b", "localhost:9092", "1", "5"] + start_period: 5s + interval: 5s + timeout: 10s + retries: 100 azurite: image: mcr.microsoft.com/azure-storage/azurite:${AZURITE_VERSION:-3.24.0} diff --git a/quickwit/quickwit-indexing/src/source/mod.rs b/quickwit/quickwit-indexing/src/source/mod.rs index 20e3effda87..ec74a6c30de 100644 --- a/quickwit/quickwit-indexing/src/source/mod.rs +++ b/quickwit/quickwit-indexing/src/source/mod.rs @@ -740,7 +740,11 @@ mod tests { #[cfg(all( test, - any(feature = "sqs-localstack-tests", feature = "kafka-broker-tests") + any( + feature = "sqs-localstack-tests", + feature = "kafka-broker-tests", + feature = "pulsar-broker-tests" + ) ))] mod test_setup_helper {