From b61dbbac177cb20c1b7c5183446714b329607d44 Mon Sep 17 00:00:00 2001 From: ebrahimmfadae Date: Tue, 12 Jul 2022 14:27:14 +0430 Subject: [PATCH 1/6] elastic: Add elastic stack services to docker-compose.yml --- docker-compose.yml | 17 +++++++++++++++++ filebeat.yml | 21 +++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 filebeat.yml diff --git a/docker-compose.yml b/docker-compose.yml index ba0bf4dbb..7623b1d10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -481,6 +481,22 @@ services: deploy: restart_policy: condition: on-failure + elasticsearch: + image: elasticsearch:8.3.2 + environment: + - ES_JAVA_OPTS=-Xms1g -Xmx1g + - discovery.type=single-node + volumes: + - elasticsearch_data:/usr/share/elasticsearch/data + kibana: + image: kibana:8.3.2 + filebeat: + image: filebeat:8.3.2 + user: root + volumes: + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/lib/docker:/var/lib/docker:ro + - /var/run/docker.sock:/var/run/docker.sock volumes: zookeeper-data: zookeeper-log: @@ -499,6 +515,7 @@ volumes: referral-data: storage-data: admin-data: + elasticsearch_data: networks: default: driver: bridge diff --git a/filebeat.yml b/filebeat.yml new file mode 100644 index 000000000..e7f4522dc --- /dev/null +++ b/filebeat.yml @@ -0,0 +1,21 @@ +filebeat.inputs: + - type: container + paths: + - '/var/lib/docker/containers/*/*.log' + +processors: + - add_docker_metadata: + host: "unix:///var/run/docker.sock" + + - decode_json_fields: + fields: [ "message" ] + target: "json" + overwrite_keys: true + +output.elasticsearch: + hosts: [ "elasticsearch:9200" ] + indices: + - index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" + +logging.json: true +logging.metrics.enabled: false \ No newline at end of file From 97dd875eea292983a4a96faf0ffdb2d4fa75ec05 Mon Sep 17 00:00:00 2001 From: ebrahimmfadae Date: Tue, 19 Jul 2022 09:30:09 +0430 Subject: [PATCH 2/6] elastic: Create immutable filebeat container --- docker-compose.build.yml | 3 +++ docker-compose.override.yml | 2 ++ docker-compose.yml | 8 ++++---- docker-images/filebeat/Dockerfile | 2 ++ filebeat.yml => docker-images/filebeat/filebeat.yml | 0 5 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 docker-images/filebeat/Dockerfile rename filebeat.yml => docker-images/filebeat/filebeat.yml (100%) diff --git a/docker-compose.build.yml b/docker-compose.build.yml index fd0601794..a850eac1b 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -45,3 +45,6 @@ services: admin: image: ghcr.io/opexdev/admin:$TAG build: admin/admin-app + filebeat: + image: ghcr.io/opexdev/filebeat:$TAG + build: docker-images/filebeat diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 15de102ab..42d161abf 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -41,3 +41,5 @@ services: build: captcha/captcha-app admin: build: admin/admin-app + filebeat: + build: docker-images/filebeat diff --git a/docker-compose.yml b/docker-compose.yml index 7623b1d10..b37d33e70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -482,19 +482,19 @@ services: restart_policy: condition: on-failure elasticsearch: - image: elasticsearch:8.3.2 + image: docker.elastic.co/elasticsearch:8.3.2 environment: - ES_JAVA_OPTS=-Xms1g -Xmx1g - discovery.type=single-node + - xpack.security.enabled=false volumes: - elasticsearch_data:/usr/share/elasticsearch/data kibana: - image: kibana:8.3.2 + image: docker.elastic.co/kibana:8.3.2 filebeat: - image: filebeat:8.3.2 + image: opex-filebeat user: root volumes: - - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - /var/lib/docker:/var/lib/docker:ro - /var/run/docker.sock:/var/run/docker.sock volumes: diff --git a/docker-images/filebeat/Dockerfile b/docker-images/filebeat/Dockerfile new file mode 100644 index 000000000..be9da7bae --- /dev/null +++ b/docker-images/filebeat/Dockerfile @@ -0,0 +1,2 @@ +FROM docker.elastic.co/filebeat:8.3.2 +COPY filebeat.yml /usr/share/filebeat/filebeat.yml diff --git a/filebeat.yml b/docker-images/filebeat/filebeat.yml similarity index 100% rename from filebeat.yml rename to docker-images/filebeat/filebeat.yml From 564614d481c2db5479c25bda02978c0ed1710961 Mon Sep 17 00:00:00 2001 From: ebrahimmfadae Date: Wed, 3 Aug 2022 22:15:07 +0430 Subject: [PATCH 3/6] elastic: Fix ssl connection issue --- docker-compose.yml | 15 ++++++++++++--- docker-images/filebeat/Dockerfile | 2 +- docker-images/filebeat/filebeat.yml | 7 +++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b37d33e70..bb022c3be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -482,21 +482,30 @@ services: restart_policy: condition: on-failure elasticsearch: - image: docker.elastic.co/elasticsearch:8.3.2 + image: elastic/elasticsearch:8.3.2 environment: - ES_JAVA_OPTS=-Xms1g -Xmx1g - discovery.type=single-node - - xpack.security.enabled=false + - ingest.geoip.downloader.enabled=false volumes: - elasticsearch_data:/usr/share/elasticsearch/data + networks: + - default kibana: - image: docker.elastic.co/kibana:8.3.2 + image: elastic/kibana:8.3.2 + networks: + - default filebeat: image: opex-filebeat + build: docker-images/filebeat user: root + environment: + - FILEBEAT_API_KEY=$FILEBEAT_API_KEY volumes: - /var/lib/docker:/var/lib/docker:ro - /var/run/docker.sock:/var/run/docker.sock + networks: + - default volumes: zookeeper-data: zookeeper-log: diff --git a/docker-images/filebeat/Dockerfile b/docker-images/filebeat/Dockerfile index be9da7bae..17c6916d4 100644 --- a/docker-images/filebeat/Dockerfile +++ b/docker-images/filebeat/Dockerfile @@ -1,2 +1,2 @@ -FROM docker.elastic.co/filebeat:8.3.2 +FROM elastic/filebeat:8.3.2 COPY filebeat.yml /usr/share/filebeat/filebeat.yml diff --git a/docker-images/filebeat/filebeat.yml b/docker-images/filebeat/filebeat.yml index e7f4522dc..cb86e7279 100644 --- a/docker-images/filebeat/filebeat.yml +++ b/docker-images/filebeat/filebeat.yml @@ -1,5 +1,5 @@ filebeat.inputs: - - type: container + - type: filestream paths: - '/var/lib/docker/containers/*/*.log' @@ -14,8 +14,11 @@ processors: output.elasticsearch: hosts: [ "elasticsearch:9200" ] + protocol: https + api_key: ${FILEBEAT_API_KEY} + ssl.verification_mode: "none" indices: - index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" logging.json: true -logging.metrics.enabled: false \ No newline at end of file +logging.metrics.enabled: false From d7c8b5493032cd499beb55f331c0557067489df3 Mon Sep 17 00:00:00 2001 From: ebrahimmfadae Date: Sat, 6 Aug 2022 17:37:06 +0430 Subject: [PATCH 4/6] elastic: Update filebeat config --- docker-images/filebeat/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-images/filebeat/filebeat.yml b/docker-images/filebeat/filebeat.yml index cb86e7279..12b767dae 100644 --- a/docker-images/filebeat/filebeat.yml +++ b/docker-images/filebeat/filebeat.yml @@ -16,7 +16,7 @@ output.elasticsearch: hosts: [ "elasticsearch:9200" ] protocol: https api_key: ${FILEBEAT_API_KEY} - ssl.verification_mode: "none" + ssl.verification_mode: "certificate" indices: - index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" From 9e4e078810a6cd1d5ba31bd1bd2d13768d7a75e5 Mon Sep 17 00:00:00 2001 From: ebrahimmfadae Date: Sat, 6 Aug 2022 17:48:49 +0430 Subject: [PATCH 5/6] Revert "elastic: Update filebeat config" This reverts commit d7c8b5493032cd499beb55f331c0557067489df3. --- docker-images/filebeat/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-images/filebeat/filebeat.yml b/docker-images/filebeat/filebeat.yml index 12b767dae..cb86e7279 100644 --- a/docker-images/filebeat/filebeat.yml +++ b/docker-images/filebeat/filebeat.yml @@ -16,7 +16,7 @@ output.elasticsearch: hosts: [ "elasticsearch:9200" ] protocol: https api_key: ${FILEBEAT_API_KEY} - ssl.verification_mode: "certificate" + ssl.verification_mode: "none" indices: - index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" From 6823fcc6a2cd1d12e0b8eab66f55872abc990763 Mon Sep 17 00:00:00 2001 From: ebrahimmfadae Date: Sat, 6 Aug 2022 17:49:18 +0430 Subject: [PATCH 6/6] elastic: Update docker-compose configs --- docker-compose.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bb022c3be..fd96279eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -487,17 +487,21 @@ services: - ES_JAVA_OPTS=-Xms1g -Xmx1g - discovery.type=single-node - ingest.geoip.downloader.enabled=false + - node.name=elasticsearch + - network.publish_host=elasticsearch volumes: - elasticsearch_data:/usr/share/elasticsearch/data networks: - default kibana: image: elastic/kibana:8.3.2 + environment: + - SERVER_PUBLICBASEURL=$KIBANA_PUBLIC_URL + - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate networks: - default filebeat: - image: opex-filebeat - build: docker-images/filebeat + image: ghcr.io/opexdev/opex-filebeat user: root environment: - FILEBEAT_API_KEY=$FILEBEAT_API_KEY