From 1f02f2ef4021117422bda1aaae570fd683d0215d Mon Sep 17 00:00:00 2001 From: Todd VanGundy Date: Sun, 30 Mar 2025 17:06:27 -0400 Subject: [PATCH 01/21] QuickWit-local-file --- README.md | 41 +++++++++++++++++ .../observability/base/kustomization.yaml | 2 + kustomize/observability/base/namespace.yaml | 18 ++++++++ .../observability/quickwit/helm-release.yaml | 21 +++++++++ .../quickwit/helm-repository.yaml | 10 ++++ .../observability/quickwit/kustomization.yaml | 5 ++ .../quickwit/local-file/kustomization.yaml | 5 ++ .../local-file/patches/helm-release.yaml | 46 +++++++++++++++++++ .../observability/quickwit/minio/bucket.yaml | 21 +++++++++ .../quickwit/minio/configmap.yaml | 10 ++++ .../quickwit/minio/kustomization.yaml | 7 +++ .../quickwit/minio/patches/helm-release.yaml | 41 +++++++++++++++++ .../observability/quickwit/namespace.yaml | 9 ++++ 13 files changed, 236 insertions(+) create mode 100644 kustomize/observability/base/kustomization.yaml create mode 100644 kustomize/observability/base/namespace.yaml create mode 100644 kustomize/observability/quickwit/helm-release.yaml create mode 100644 kustomize/observability/quickwit/helm-repository.yaml create mode 100644 kustomize/observability/quickwit/kustomization.yaml create mode 100644 kustomize/observability/quickwit/local-file/kustomization.yaml create mode 100644 kustomize/observability/quickwit/local-file/patches/helm-release.yaml create mode 100644 kustomize/observability/quickwit/minio/bucket.yaml create mode 100644 kustomize/observability/quickwit/minio/configmap.yaml create mode 100644 kustomize/observability/quickwit/minio/kustomization.yaml create mode 100644 kustomize/observability/quickwit/minio/patches/helm-release.yaml create mode 100644 kustomize/observability/quickwit/namespace.yaml diff --git a/README.md b/README.md index b9680fd1..d973371e 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,44 @@ Core configurations used as the basis for most blueprints ![CI Workflow](https://github.com/your-repo/core/actions/workflows/ci.yaml/badge.svg) + +# Blueprint.yaml + +## Quickwit + +``` +- name: observability-base + path: observability/base +- name: quickwit + path: observability/quickwit + dependsOn: + - observability-base + - pki-base + components: + - local-file + ``` + + ## Metrics Server + ``` + - name: metrics-server-resources + path: telemetry/resources + components: + - metrics-server + ``` + + ## FluentBit + ``` + - name: fluentbit + path: telemetry/base + components: + - fluentbit +- name: fluentbit-resources + path: telemetry/resources + components: + - fluentbit +- name: metrics-server-resources + path: telemetry/resources + components: + - metrics-server + ``` + \ No newline at end of file diff --git a/kustomize/observability/base/kustomization.yaml b/kustomize/observability/base/kustomization.yaml new file mode 100644 index 00000000..736967b1 --- /dev/null +++ b/kustomize/observability/base/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - namespace.yaml diff --git a/kustomize/observability/base/namespace.yaml b/kustomize/observability/base/namespace.yaml new file mode 100644 index 00000000..2a2180aa --- /dev/null +++ b/kustomize/observability/base/namespace.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: system-observability + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/warn: baseline + use-custom-ca: "true" + +# apiVersion: v1 +# kind: Namespace +# metadata: +# name: system-observability +# labels: +# pod-security.kubernetes.io/enforce: privileged +# pod-security.kubernetes.io/audit: privileged +# pod-security.kubernetes.io/warn: privileged diff --git a/kustomize/observability/quickwit/helm-release.yaml b/kustomize/observability/quickwit/helm-release.yaml new file mode 100644 index 00000000..e5edaf0b --- /dev/null +++ b/kustomize/observability/quickwit/helm-release.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: quickwit + namespace: system-observability +spec: + interval: 5m + timeout: 10m + chart: + spec: + chart: quickwit + # renovate: datasource=helm depName=quickwit package=quickwit helmRepo=https://helm.quickwit.io + version: 0.7.7 + sourceRef: + kind: HelmRepository + name: quickwit + namespace: system-gitops + values: + searcher: + replicaCount: 1 diff --git a/kustomize/observability/quickwit/helm-repository.yaml b/kustomize/observability/quickwit/helm-repository.yaml new file mode 100644 index 00000000..dd81eac1 --- /dev/null +++ b/kustomize/observability/quickwit/helm-repository.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: HelmRepository +metadata: + name: quickwit + namespace: system-gitops +spec: + interval: 10m + timeout: 3m + url: https://helm.quickwit.io diff --git a/kustomize/observability/quickwit/kustomization.yaml b/kustomize/observability/quickwit/kustomization.yaml new file mode 100644 index 00000000..e768af5d --- /dev/null +++ b/kustomize/observability/quickwit/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - helm-repository.yaml + - helm-release.yaml diff --git a/kustomize/observability/quickwit/local-file/kustomization.yaml b/kustomize/observability/quickwit/local-file/kustomization.yaml new file mode 100644 index 00000000..4f646e87 --- /dev/null +++ b/kustomize/observability/quickwit/local-file/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - path: patches/helm-release.yaml + # - path: patches/namespace.yaml diff --git a/kustomize/observability/quickwit/local-file/patches/helm-release.yaml b/kustomize/observability/quickwit/local-file/patches/helm-release.yaml new file mode 100644 index 00000000..ad044a72 --- /dev/null +++ b/kustomize/observability/quickwit/local-file/patches/helm-release.yaml @@ -0,0 +1,46 @@ +--- +# SECURITY NOTE: This approach uses a hostPath volume to store the indexes on the host machine. +# It also requires elevated privileges. Do not use in production. For a more secure setup, +# integrate with backing services such as Postgres, Minio, or S3. +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: quickwit + namespace: system-observability +spec: + values: + environment: + QW_METASTORE_URI: file:///quickwit/qwdata/indexes + config: + default_index_root_uri: file:///quickwit/qwdata/indexes + podSecurityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: true + securityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: true + searcher: + replicaCount: 1 + extraVolumes: + - name: quickwit-indexes + hostPath: + path: /var/lib/quickwit/indexes + type: DirectoryOrCreate + extraVolumeMounts: + - name: quickwit-indexes + mountPath: /quickwit/qwdata/indexes + readOnly: true + indexer: + extraVolumes: + - name: quickwit-indexes + hostPath: + path: /var/lib/quickwit/indexes + type: DirectoryOrCreate + extraVolumeMounts: + - name: quickwit-indexes + mountPath: /quickwit/qwdata/indexes + readOnly: false diff --git a/kustomize/observability/quickwit/minio/bucket.yaml b/kustomize/observability/quickwit/minio/bucket.yaml new file mode 100644 index 00000000..1bb3eb11 --- /dev/null +++ b/kustomize/observability/quickwit/minio/bucket.yaml @@ -0,0 +1,21 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: minio-quickwit-bucket + namespace: system-object-store +spec: + interval: 5m + timeout: 5m + chart: + spec: + chart: charts/minio-bucket + sourceRef: + kind: GitRepository + name: core + namespace: system-gitops + values: + bucket: quickwit + secretNamespace: system-observability + minioEndpoint: "https://minio.system-object-store.svc.cluster.local:443" + podLabels: + use-custom-ca: "true" diff --git a/kustomize/observability/quickwit/minio/configmap.yaml b/kustomize/observability/quickwit/minio/configmap.yaml new file mode 100644 index 00000000..2f8895ef --- /dev/null +++ b/kustomize/observability/quickwit/minio/configmap.yaml @@ -0,0 +1,10 @@ +# This does nothing important until https://github.com/quickwit-oss/quickwit/issues/5199 is resolved +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: minio-sts-env + namespace: system-observability +data: + AWS_WEB_IDENTITY_TOKEN_FILE: "/var/run/secrets/kubernetes.io/serviceaccount/token" + AWS_S3_ENDPOINT: "https://minio.system-object-store.svc.cluster.local" diff --git a/kustomize/observability/quickwit/minio/kustomization.yaml b/kustomize/observability/quickwit/minio/kustomization.yaml new file mode 100644 index 00000000..69c2a6ba --- /dev/null +++ b/kustomize/observability/quickwit/minio/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - configmap.yaml + - bucket.yaml +patches: + - path: patches/helm-release.yaml diff --git a/kustomize/observability/quickwit/minio/patches/helm-release.yaml b/kustomize/observability/quickwit/minio/patches/helm-release.yaml new file mode 100644 index 00000000..c0b983ce --- /dev/null +++ b/kustomize/observability/quickwit/minio/patches/helm-release.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: quickwit + namespace: system-observability +spec: + dependsOn: + - name: minio-quickwit-bucket + namespace: system-object-store + values: + additionalLabels: + use-custom-ca: "true" + config: + default_index_root_uri: s3://quickwit/indexes + storage: + s3: + endpoint: https://minio.system-object-store.svc.cluster.local + flavor: minio + region: "us-east-1" + valuesFrom: + - kind: Secret + name: minio-quickwit-keys + valuesKey: access_key + targetPath: config.storage.s3.access_key_id + - kind: Secret + name: minio-quickwit-keys + valuesKey: secret_key + targetPath: config.storage.s3.secret_access_key + + # NOTE: Reinstate this after resolution of https://github.com/quickwit-oss/quickwit/issues/5199 + # indexer: + # extraEnvFrom: + # - configMapRef: + # name: minio-sts-env + # searcher: + # extraEnvFrom: + # - configMapRef: + # name: minio-sts-env + # serviceAccount: + # name: quickwit diff --git a/kustomize/observability/quickwit/namespace.yaml b/kustomize/observability/quickwit/namespace.yaml new file mode 100644 index 00000000..5544d5b6 --- /dev/null +++ b/kustomize/observability/quickwit/namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: system-observability + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/warn: baseline + use-custom-ca: "true" From 0f6fc06106e604f2616a917b515bfc055255eee8 Mon Sep 17 00:00:00 2001 From: Todd VanGundy Date: Mon, 31 Mar 2025 08:37:18 -0400 Subject: [PATCH 02/21] minio support --- .../blueprint.original.yaml | 85 +++++++++++ .../blueprint.quickwit-local.yaml | 92 ++++++++++++ .../blueprint.quickwit-minio.yaml | 132 ++++++++++++++++++ 3 files changed, 309 insertions(+) create mode 100644 contexts/local-docker-desktop/blueprint.original.yaml create mode 100644 contexts/local-docker-desktop/blueprint.quickwit-local.yaml create mode 100644 contexts/local-docker-desktop/blueprint.quickwit-minio.yaml diff --git a/contexts/local-docker-desktop/blueprint.original.yaml b/contexts/local-docker-desktop/blueprint.original.yaml new file mode 100644 index 00000000..ec4ca744 --- /dev/null +++ b/contexts/local-docker-desktop/blueprint.original.yaml @@ -0,0 +1,85 @@ +kind: Blueprint +apiVersion: blueprints.windsorcli.dev/v1alpha1 +metadata: + name: local + description: This blueprint outlines resources in the local context +repository: + url: http://git.test/git/core + ref: + branch: main + secretName: flux-system +sources: [] +terraform: +- path: cluster/talos +- path: gitops/flux +kustomize: +- name: policy-base + path: policy/base + components: + - kyverno +- name: policy-resources + path: policy/resources + dependsOn: + - policy-base +- name: csi + path: csi + dependsOn: + - policy-resources + force: true + components: + - openebs + - openebs/dynamic-localpv +- name: ingress-base + path: ingress/base + dependsOn: + - pki-resources + force: true + components: + - nginx + - nginx/nodeport + - nginx/coredns + - nginx/flux-webhook + - nginx/web +- name: pki-base + path: pki/base + dependsOn: + - policy-resources + force: true + components: + - cert-manager + - trust-manager +- name: pki-resources + path: pki/resources + dependsOn: + - pki-base + force: true + components: + - private-issuer/ca + - public-issuer/selfsigned +- name: dns + path: dns + dependsOn: + - ingress-base + - pki-base + force: true + components: + - coredns + - coredns/etcd + - external-dns + - external-dns/localhost + - external-dns/coredns + - external-dns/ingress +- name: gitops + path: gitops/flux + dependsOn: + - ingress-base + force: true + components: + - webhook +- name: demo + path: demo/bookinfo + dependsOn: + - ingress-base + force: true + components: + - ingress diff --git a/contexts/local-docker-desktop/blueprint.quickwit-local.yaml b/contexts/local-docker-desktop/blueprint.quickwit-local.yaml new file mode 100644 index 00000000..bfb721e4 --- /dev/null +++ b/contexts/local-docker-desktop/blueprint.quickwit-local.yaml @@ -0,0 +1,92 @@ +kind: Blueprint +apiVersion: blueprints.windsorcli.dev/v1alpha1 +metadata: + name: local + description: This blueprint outlines resources in the local context +repository: + url: http://git.test/git/core + ref: + branch: main + secretName: flux-system +sources: [] +terraform: +- path: cluster/talos +- path: gitops/flux +kustomize: +- name: policy-base + path: policy/base + components: + - kyverno +- name: policy-resources + path: policy/resources + dependsOn: + - policy-base +- name: csi + path: csi + dependsOn: + - policy-resources + force: true + components: + - openebs + - openebs/dynamic-localpv +- name: ingress-base + path: ingress/base + dependsOn: + - pki-resources + force: true + components: + - nginx + - nginx/nodeport + - nginx/coredns + - nginx/flux-webhook + - nginx/web +- name: pki-base + path: pki/base + dependsOn: + - policy-resources + force: true + components: + - cert-manager + - trust-manager +- name: pki-resources + path: pki/resources + dependsOn: + - pki-base + force: true + components: + - private-issuer/ca + - public-issuer/selfsigned +- name: dns + path: dns + dependsOn: + - ingress-base + - pki-base + force: true + components: + - coredns + - coredns/etcd + - external-dns + - external-dns/localhost + - external-dns/coredns + - external-dns/ingress +- name: gitops + path: gitops/flux + dependsOn: + - ingress-base + force: true + components: + - webhook +- name: demo + path: demo/bookinfo + dependsOn: + - ingress-base + force: true + components: + - ingress +- name: quickwit + path: observability/quickwit + dependsOn: + - observability-base + - pki-base + components: + - local-file diff --git a/contexts/local-docker-desktop/blueprint.quickwit-minio.yaml b/contexts/local-docker-desktop/blueprint.quickwit-minio.yaml new file mode 100644 index 00000000..86e36e6c --- /dev/null +++ b/contexts/local-docker-desktop/blueprint.quickwit-minio.yaml @@ -0,0 +1,132 @@ +kind: Blueprint +apiVersion: blueprints.windsorcli.dev/v1alpha1 +metadata: + name: local + description: This blueprint outlines resources in the local context +repository: + url: http://git.test/git/telemetry-test + ref: + branch: main + secretName: flux-system +sources: +- name: core + url: github.com/windsorcli/core + ref: + tag: v0.2.0 +terraform: +- source: core + path: cluster/talos +- source: core + path: gitops/flux +kustomize: +- name: policy-base + path: policy/base + source: core + components: + - kyverno +- name: policy-resources + path: policy/resources + source: core + dependsOn: + - policy-base +- name: csi + path: csi + source: core + dependsOn: + - policy-resources + force: true + components: + - openebs + - openebs/dynamic-localpv +- name: ingress-base + path: ingress/base + source: core + dependsOn: + - pki-resources + force: true + components: + - nginx + - nginx/nodeport + - nginx/coredns + - nginx/flux-webhook + - nginx/web +- name: pki-base + path: pki/base + source: core + dependsOn: + - policy-resources + force: true + components: + - cert-manager + - trust-manager +- name: pki-resources + path: pki/resources + source: core + dependsOn: + - pki-base + force: true + components: + - private-issuer/ca + - public-issuer/selfsigned +- name: dns + path: dns + source: core + dependsOn: + - ingress-base + - pki-base + force: true + components: + - coredns + - coredns/etcd + - external-dns + - external-dns/localhost + - external-dns/coredns + - external-dns/ingress +- name: gitops + path: gitops/flux + source: core + dependsOn: + - ingress-base + force: true + components: + - webhook +- name: demo + path: demo/bookinfo + source: core + dependsOn: + - ingress-base + force: true + components: + - ingress +- name: fluentbit + path: telemetry/base + components: + - fluentbit +- name: fluentbit-resources + path: telemetry/resources + components: + - fluentbit +- name: metrics-server-resources + path: telemetry/resources + components: + - metrics-server +- name: object-store-base + path: object-store/base + components: + - minio +- name: object-store-resources + path: object-store/resources + dependsOn: + - object-store-base + components: + - minio +- name: observability-base + path: observability/base +- name: quickwit + path: observability/quickwit + dependsOn: + - observability-base + - pki-base + - object-store-resources + components: + - minio From 962b649ea29d098e774784ae47c08268df7ebac3 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Tue, 8 Jul 2025 00:08:52 +0200 Subject: [PATCH 03/21] quickwit changes --- contexts/local/blueprint.quickwit-local.yaml | 52 ++++++++++++------- contexts/local/blueprint.yaml | 19 +++++++ .../quickwit}/kustomization.yaml | 1 - .../quickwit/patches/helm-release.yaml | 12 +++++ .../observability/quickwit/helm-release.yaml | 2 +- .../observability/quickwit/namespace.yaml | 9 ---- .../quickwit/pvc/kustomization.yaml | 6 +++ .../patches/helm-release.yaml | 10 ++-- .../observability/quickwit/pvc/volume.yaml | 11 ++++ terraform/gitops/flux/.terraform.lock.hcl | 27 +++++----- 10 files changed, 100 insertions(+), 49 deletions(-) rename kustomize/observability/{quickwit/local-file => grafana/quickwit}/kustomization.yaml (75%) create mode 100644 kustomize/observability/grafana/quickwit/patches/helm-release.yaml delete mode 100644 kustomize/observability/quickwit/namespace.yaml create mode 100644 kustomize/observability/quickwit/pvc/kustomization.yaml rename kustomize/observability/quickwit/{local-file => pvc}/patches/helm-release.yaml (85%) create mode 100644 kustomize/observability/quickwit/pvc/volume.yaml diff --git a/contexts/local/blueprint.quickwit-local.yaml b/contexts/local/blueprint.quickwit-local.yaml index bfb721e4..0206858d 100644 --- a/contexts/local/blueprint.quickwit-local.yaml +++ b/contexts/local/blueprint.quickwit-local.yaml @@ -8,11 +8,29 @@ repository: ref: branch: main secretName: flux-system -sources: [] +sources: +- name: core + url: github.com/windsorcli/core + ref: + branch: main terraform: - path: cluster/talos - path: gitops/flux + destroy: false kustomize: +- name: telemetry-base + path: telemetry/base + components: + - prometheus + - prometheus/flux +- name: telemetry-resources + path: telemetry/resources + dependsOn: + - telemetry-base + components: + - metrics-server + - prometheus + - prometheus/flux - name: policy-base path: policy/base components: @@ -25,15 +43,13 @@ kustomize: path: csi dependsOn: - policy-resources - force: true components: - openebs - openebs/dynamic-localpv -- name: ingress-base - path: ingress/base +- name: ingress + path: ingress dependsOn: - pki-resources - force: true components: - nginx - nginx/nodeport @@ -44,7 +60,6 @@ kustomize: path: pki/base dependsOn: - policy-resources - force: true components: - cert-manager - trust-manager @@ -52,16 +67,13 @@ kustomize: path: pki/resources dependsOn: - pki-base - force: true components: - private-issuer/ca - public-issuer/selfsigned - name: dns path: dns dependsOn: - - ingress-base - pki-base - force: true components: - coredns - coredns/etcd @@ -72,21 +84,25 @@ kustomize: - name: gitops path: gitops/flux dependsOn: - - ingress-base - force: true + - ingress components: - webhook -- name: demo - path: demo/bookinfo +- name: observability + path: observability dependsOn: - - ingress-base - force: true + - csi + - ingress components: - - ingress + - grafana + - grafana/ingress + - grafana/prometheus + - grafana/node + - grafana/kubernetes + - grafana/flux + - grafana/quickwit - name: quickwit path: observability/quickwit dependsOn: - - observability-base - pki-base components: - - local-file + - pvc diff --git a/contexts/local/blueprint.yaml b/contexts/local/blueprint.yaml index 415c588c..0206858d 100644 --- a/contexts/local/blueprint.yaml +++ b/contexts/local/blueprint.yaml @@ -87,3 +87,22 @@ kustomize: - ingress components: - webhook +- name: observability + path: observability + dependsOn: + - csi + - ingress + components: + - grafana + - grafana/ingress + - grafana/prometheus + - grafana/node + - grafana/kubernetes + - grafana/flux + - grafana/quickwit +- name: quickwit + path: observability/quickwit + dependsOn: + - pki-base + components: + - pvc diff --git a/kustomize/observability/quickwit/local-file/kustomization.yaml b/kustomize/observability/grafana/quickwit/kustomization.yaml similarity index 75% rename from kustomize/observability/quickwit/local-file/kustomization.yaml rename to kustomize/observability/grafana/quickwit/kustomization.yaml index 4f646e87..8138d116 100644 --- a/kustomize/observability/quickwit/local-file/kustomization.yaml +++ b/kustomize/observability/grafana/quickwit/kustomization.yaml @@ -2,4 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component patches: - path: patches/helm-release.yaml - # - path: patches/namespace.yaml diff --git a/kustomize/observability/grafana/quickwit/patches/helm-release.yaml b/kustomize/observability/grafana/quickwit/patches/helm-release.yaml new file mode 100644 index 00000000..c11d67be --- /dev/null +++ b/kustomize/observability/grafana/quickwit/patches/helm-release.yaml @@ -0,0 +1,12 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: grafana + namespace: system-observability +spec: + dependsOn: + - name: quickwit + namespace: system-observability + values: + plugins: + - quickwit-quickwit-datasource \ No newline at end of file diff --git a/kustomize/observability/quickwit/helm-release.yaml b/kustomize/observability/quickwit/helm-release.yaml index e5edaf0b..c175a095 100644 --- a/kustomize/observability/quickwit/helm-release.yaml +++ b/kustomize/observability/quickwit/helm-release.yaml @@ -11,7 +11,7 @@ spec: spec: chart: quickwit # renovate: datasource=helm depName=quickwit package=quickwit helmRepo=https://helm.quickwit.io - version: 0.7.7 + version: 0.7.17 sourceRef: kind: HelmRepository name: quickwit diff --git a/kustomize/observability/quickwit/namespace.yaml b/kustomize/observability/quickwit/namespace.yaml deleted file mode 100644 index 5544d5b6..00000000 --- a/kustomize/observability/quickwit/namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: system-observability - labels: - pod-security.kubernetes.io/enforce: baseline - pod-security.kubernetes.io/audit: baseline - pod-security.kubernetes.io/warn: baseline - use-custom-ca: "true" diff --git a/kustomize/observability/quickwit/pvc/kustomization.yaml b/kustomize/observability/quickwit/pvc/kustomization.yaml new file mode 100644 index 00000000..907131bb --- /dev/null +++ b/kustomize/observability/quickwit/pvc/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - volume.yaml +patches: + - path: patches/helm-release.yaml diff --git a/kustomize/observability/quickwit/local-file/patches/helm-release.yaml b/kustomize/observability/quickwit/pvc/patches/helm-release.yaml similarity index 85% rename from kustomize/observability/quickwit/local-file/patches/helm-release.yaml rename to kustomize/observability/quickwit/pvc/patches/helm-release.yaml index ad044a72..21faa5fe 100644 --- a/kustomize/observability/quickwit/local-file/patches/helm-release.yaml +++ b/kustomize/observability/quickwit/pvc/patches/helm-release.yaml @@ -27,9 +27,8 @@ spec: replicaCount: 1 extraVolumes: - name: quickwit-indexes - hostPath: - path: /var/lib/quickwit/indexes - type: DirectoryOrCreate + persistentVolumeClaim: + claimName: quickwit-indexes extraVolumeMounts: - name: quickwit-indexes mountPath: /quickwit/qwdata/indexes @@ -37,9 +36,8 @@ spec: indexer: extraVolumes: - name: quickwit-indexes - hostPath: - path: /var/lib/quickwit/indexes - type: DirectoryOrCreate + persistentVolumeClaim: + claimName: quickwit-indexes extraVolumeMounts: - name: quickwit-indexes mountPath: /quickwit/qwdata/indexes diff --git a/kustomize/observability/quickwit/pvc/volume.yaml b/kustomize/observability/quickwit/pvc/volume.yaml new file mode 100644 index 00000000..ad5d1204 --- /dev/null +++ b/kustomize/observability/quickwit/pvc/volume.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: quickwit-indexes + namespace: system-observability +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/terraform/gitops/flux/.terraform.lock.hcl b/terraform/gitops/flux/.terraform.lock.hcl index 68cb7881..5157854d 100644 --- a/terraform/gitops/flux/.terraform.lock.hcl +++ b/terraform/gitops/flux/.terraform.lock.hcl @@ -2,21 +2,20 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/helm" { - version = "2.17.0" + version = "3.0.2" hashes = [ - "h1:If79Gw54AMearm13Sk9RmWuDesCQQMUtmlJXXqISxfU=", - "h1:kQMkcPVvHOguOqnxoEU2sm1ND9vCHiT8TvZ2x6v/Rsw=", - "zh:06fb4e9932f0afc1904d2279e6e99353c2ddac0d765305ce90519af410706bd4", - "zh:104eccfc781fc868da3c7fec4385ad14ed183eb985c96331a1a937ac79c2d1a7", - "zh:129345c82359837bb3f0070ce4891ec232697052f7d5ccf61d43d818912cf5f3", - "zh:3956187ec239f4045975b35e8c30741f701aa494c386aaa04ebabffe7749f81c", - "zh:66a9686d92a6b3ec43de3ca3fde60ef3d89fb76259ed3313ca4eb9bb8c13b7dd", - "zh:88644260090aa621e7e8083585c468c8dd5e09a3c01a432fb05da5c4623af940", - "zh:a248f650d174a883b32c5b94f9e725f4057e623b00f171936dcdcc840fad0b3e", - "zh:aa498c1f1ab93be5c8fbf6d48af51dc6ef0f10b2ea88d67bcb9f02d1d80d3930", - "zh:bf01e0f2ec2468c53596e027d376532a2d30feb72b0b5b810334d043109ae32f", - "zh:c46fa84cc8388e5ca87eb575a534ebcf68819c5a5724142998b487cb11246654", - "zh:d0c0f15ffc115c0965cbfe5c81f18c2e114113e7a1e6829f6bfd879ce5744fbb", + "h1:tOye2RnjFNXH236AsqGaIWtz4j6PZrpPuJhOSBt0KxU=", + "zh:2778de76c7dfb2e85c75fe6de3c11172a25551ed499bfb9e9f940a5be81167b0", + "zh:3b4c436a41e4fbae5f152852a9bd5c97db4460af384e26977477a40adf036690", + "zh:617a372f5bb2288f3faf5fd4c878a68bf08541cf418a3dbb8a19bc41ad4a0bf2", + "zh:84de431479548c96cb61c495278e320f361e80ab4f8835a5425ece24a9b6d310", + "zh:8b4cf5f81d10214e5e1857d96cff60a382a22b9caded7f5d7a92e5537fc166c1", + "zh:baeb26a00ffbcf3d507cdd940b2a2887eee723af5d3319a53eec69048d5e341e", + "zh:ca05a8814e9bf5fbffcd642df3a8d9fae9549776c7057ceae6d6f56471bae80f", + "zh:ca4bf3f94dedb5c5b1a73568f2dad7daf0ef3f85e688bc8bc2d0e915ec148366", + "zh:d331f2129fd3165c4bda875c84a65555b22eb007801522b9e017d065ac69b67e", + "zh:e583b2b478dde67da28e605ab4ef6521c2e390299b471d7d8ef05a0b608dcdad", + "zh:f238b86611647c108c073d265f8891a2738d3158c247468ae0ff5b1a3ac4122a", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } From 559e5f8d81a6be0dfc12ec8bc8026c0ed9b7e110 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Wed, 9 Jul 2025 10:37:07 +0200 Subject: [PATCH 04/21] quickwit changes --- contexts/local/blueprint.yaml | 7 ++++--- .../resources/fluentbit/quickwit/clusteroutput.yaml | 13 +++++++++++++ .../resources/fluentbit/quickwit/kustomization.yaml | 4 ++++ 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml create mode 100644 kustomize/telemetry/resources/fluentbit/quickwit/kustomization.yaml diff --git a/contexts/local/blueprint.yaml b/contexts/local/blueprint.yaml index a1c42406..40e9c71f 100644 --- a/contexts/local/blueprint.yaml +++ b/contexts/local/blueprint.yaml @@ -35,7 +35,7 @@ kustomize: - prometheus/flux - fluentbit - fluentbit/containerd - - fluentbit/fluentd + - fluentbit/quickwit - fluentbit/kubernetes - fluentbit/systemd - name: policy-base @@ -100,6 +100,7 @@ kustomize: - csi - ingress components: + - quickwit - grafana - grafana/ingress - grafana/prometheus @@ -107,9 +108,9 @@ kustomize: - grafana/kubernetes - grafana/flux - grafana/quickwit -- name: quickwit +- name: quickwit-backend path: observability/quickwit dependsOn: - - pki-base + - observability components: - pvc diff --git a/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml b/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml new file mode 100644 index 00000000..e2c266ff --- /dev/null +++ b/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterOutput +metadata: + labels: + fluentbit.fluent.io/component: logging + fluentbit.fluent.io/enabled: "true" + name: quickwit +spec: + forward: + host: quickwit-indexer.system-observability.svc.cluster.local + port: 7280 + matchRegex: (?:kube|service)\.(.*) diff --git a/kustomize/telemetry/resources/fluentbit/quickwit/kustomization.yaml b/kustomize/telemetry/resources/fluentbit/quickwit/kustomization.yaml new file mode 100644 index 00000000..051eb969 --- /dev/null +++ b/kustomize/telemetry/resources/fluentbit/quickwit/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - clusteroutput.yaml From 39e0b07234b95492588c4aa45a5195cbc63ad99a Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Wed, 9 Jul 2025 12:39:03 +0200 Subject: [PATCH 05/21] Adds index and cluster output for fluentbit --- .../observability/grafana/quickwit/index.yaml | 37 +++++++++++++++++++ .../grafana/quickwit/kustomization.yaml | 2 + .../fluentbit/quickwit/clusteroutput.yaml | 4 +- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 kustomize/observability/grafana/quickwit/index.yaml diff --git a/kustomize/observability/grafana/quickwit/index.yaml b/kustomize/observability/grafana/quickwit/index.yaml new file mode 100644 index 00000000..4bc0fe2c --- /dev/null +++ b/kustomize/observability/grafana/quickwit/index.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: create-quickwit-index + namespace: system-observability +spec: + template: + spec: + containers: + - name: create-index + image: curlimages/curl:8.14.1 + command: + - /bin/sh + - -c + - | + cat < /tmp/index.yaml + version: 0.7 + + index_id: fluentbit-logs + + doc_mapping: + mode: dynamic + field_mappings: + - name: timestamp + type: datetime + input_formats: + - unix_timestamp + output_format: unix_timestamp_secs + fast: true + timestamp_field: timestamp + + indexing_settings: + commit_timeout_secs: 10 + EOF + + curl -X POST http://quickwit-indexer.system-observability.svc.cluster.local:7280/api/v1/indexes -H "Content-Type: application/yaml" --data-binary @/tmp/index.yaml + restartPolicy: OnFailure \ No newline at end of file diff --git a/kustomize/observability/grafana/quickwit/kustomization.yaml b/kustomize/observability/grafana/quickwit/kustomization.yaml index 8138d116..48836433 100644 --- a/kustomize/observability/grafana/quickwit/kustomization.yaml +++ b/kustomize/observability/grafana/quickwit/kustomization.yaml @@ -1,4 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component +resources: + - index.yaml patches: - path: patches/helm-release.yaml diff --git a/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml b/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml index e2c266ff..de1cfdff 100644 --- a/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml +++ b/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml @@ -7,7 +7,9 @@ metadata: fluentbit.fluent.io/enabled: "true" name: quickwit spec: - forward: + http: host: quickwit-indexer.system-observability.svc.cluster.local port: 7280 + uri: /api/v1/fluentbit-logs/ingest + format: json matchRegex: (?:kube|service)\.(.*) From 97ceeb2f08e3ea512f436db11b28b2027c7f9a0f Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Thu, 10 Jul 2025 10:42:49 +0200 Subject: [PATCH 06/21] adds fluentd as aggregator --- contexts/local/blueprint.yaml | 4 +- kustomize/observability/fluentd/fluentd.yaml | 51 +++++++++++++++++++ .../observability/fluentd/helm-release.yaml | 27 ++++++++++ .../observability/fluentd/kustomization.yaml | 4 ++ .../fluentd/quickwit/clusterfilter.yaml | 22 ++++++++ .../fluentd/quickwit/clusteroutput.yaml | 11 ++++ .../fluentd/quickwit/kustomization.yaml | 5 ++ .../fluentd/stdout/clusteroutput.yaml | 9 ++++ .../fluentd/stdout}/kustomization.yaml | 0 .../grafana/quickwit/kustomization.yaml | 11 +++- .../grafana/quickwit/patches/patch.json | 15 ++++++ .../fluentbit/quickwit/clusteroutput.yaml | 15 ------ 12 files changed, 156 insertions(+), 18 deletions(-) create mode 100644 kustomize/observability/fluentd/fluentd.yaml create mode 100644 kustomize/observability/fluentd/helm-release.yaml create mode 100644 kustomize/observability/fluentd/kustomization.yaml create mode 100644 kustomize/observability/fluentd/quickwit/clusterfilter.yaml create mode 100644 kustomize/observability/fluentd/quickwit/clusteroutput.yaml create mode 100644 kustomize/observability/fluentd/quickwit/kustomization.yaml create mode 100644 kustomize/observability/fluentd/stdout/clusteroutput.yaml rename kustomize/{telemetry/resources/fluentbit/quickwit => observability/fluentd/stdout}/kustomization.yaml (100%) create mode 100644 kustomize/observability/grafana/quickwit/patches/patch.json delete mode 100644 kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml diff --git a/contexts/local/blueprint.yaml b/contexts/local/blueprint.yaml index 40e9c71f..3e1228b7 100644 --- a/contexts/local/blueprint.yaml +++ b/contexts/local/blueprint.yaml @@ -35,7 +35,7 @@ kustomize: - prometheus/flux - fluentbit - fluentbit/containerd - - fluentbit/quickwit + - fluentbit/fluentd - fluentbit/kubernetes - fluentbit/systemd - name: policy-base @@ -101,6 +101,8 @@ kustomize: - ingress components: - quickwit + - fluentd + - fluentd/quickwit - grafana - grafana/ingress - grafana/prometheus diff --git a/kustomize/observability/fluentd/fluentd.yaml b/kustomize/observability/fluentd/fluentd.yaml new file mode 100644 index 00000000..41aea384 --- /dev/null +++ b/kustomize/observability/fluentd/fluentd.yaml @@ -0,0 +1,51 @@ +apiVersion: fluentd.fluent.io/v1alpha1 +kind: Fluentd +metadata: + labels: + app.kubernetes.io/name: fluentd + name: fluentd + namespace: system-observability +spec: + fluentdCfgSelector: + matchLabels: + config.fluentd.fluent.io/enabled: "true" + globalInputs: + - forward: + bind: 0.0.0.0 + port: 24224 + image: ghcr.io/fluent/fluent-operator/fluentd:v1.17.0 + logLevel: info + mode: collector + positionDB: {} + replicas: 1 + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 128Mi + service: {} +status: + messages: all matched cfgs is valid + state: active +--- +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterFluentdConfig +metadata: + labels: + config.fluentd.fluent.io/enabled: "true" + name: fluentd-config +spec: + clusterFilterSelector: + matchLabels: + filter.fluentd.fluent.io/enabled: "true" + clusterOutputSelector: + matchLabels: + output.fluentd.fluent.io/enabled: "true" + watchedNamespaces: + - kube-system + - default +status: + messages: Generate fluentd configs successfully + state: valid diff --git a/kustomize/observability/fluentd/helm-release.yaml b/kustomize/observability/fluentd/helm-release.yaml new file mode 100644 index 00000000..d95ec49c --- /dev/null +++ b/kustomize/observability/fluentd/helm-release.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: fluent-operator-config + namespace: system-observability +spec: + interval: 5m + timeout: 5m + chart: + spec: + chart: charts/fluent-operator + # chart: fluent-operator + # renovate: datasource=helm depName=fluent-operator package=fluent-operator helmRepo=https://fluent.github.io/helm-charts + # version: 3.2.0 + sourceRef: + kind: GitRepository + name: fluent-operator + namespace: system-gitops + values: + containerRuntime: containerd + fluentd: + enable: true + crdsEnable: false + fluentbit: + enable: true + crdsEnable: false diff --git a/kustomize/observability/fluentd/kustomization.yaml b/kustomize/observability/fluentd/kustomization.yaml new file mode 100644 index 00000000..3aa9a8c5 --- /dev/null +++ b/kustomize/observability/fluentd/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - fluentd.yaml diff --git a/kustomize/observability/fluentd/quickwit/clusterfilter.yaml b/kustomize/observability/fluentd/quickwit/clusterfilter.yaml new file mode 100644 index 00000000..51b2a269 --- /dev/null +++ b/kustomize/observability/fluentd/quickwit/clusterfilter.yaml @@ -0,0 +1,22 @@ +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterFilter +metadata: + name: otel + labels: + filter.fluentd.fluent.io/enabled: "true" +spec: + filters: + - recordTransformer: + enableRuby: true + renewRecord: false + removeKeys: "logtag,time,log,kubernetes" + records: + - key: timestamp_nanos + value: | + $${Time.parse(record["time"]).to_i * 1_000_000_000 + Time.parse(record["time"]).nsec} + - key: body + value: | + $${{"message" => record["log"]}} + - key: resource_attributes + value: | + $${{"pod_name" => record["kubernetes"]["pod_name"], "namespace_name" => record["kubernetes"]["namespace_name"], "container_name" => record["kubernetes"]["container_name"], "container_image_id" => record["kubernetes"]["docker_id"], "container_image" => record["kubernetes"]["container_image"]}} diff --git a/kustomize/observability/fluentd/quickwit/clusteroutput.yaml b/kustomize/observability/fluentd/quickwit/clusteroutput.yaml new file mode 100644 index 00000000..772bdd3c --- /dev/null +++ b/kustomize/observability/fluentd/quickwit/clusteroutput.yaml @@ -0,0 +1,11 @@ +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterOutput +metadata: + name: quickwit + labels: + output.fluentd.fluent.io/enabled: "true" +spec: + outputs: + - http: + endpoint: http://quickwit-indexer.system-observability.svc.cluster.local:7280/api/v1/otel-logs-v0_7/ingest + contentType: application/json diff --git a/kustomize/observability/fluentd/quickwit/kustomization.yaml b/kustomize/observability/fluentd/quickwit/kustomization.yaml new file mode 100644 index 00000000..68328af8 --- /dev/null +++ b/kustomize/observability/fluentd/quickwit/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - clusterfilter.yaml + - clusteroutput.yaml diff --git a/kustomize/observability/fluentd/stdout/clusteroutput.yaml b/kustomize/observability/fluentd/stdout/clusteroutput.yaml new file mode 100644 index 00000000..61a4a912 --- /dev/null +++ b/kustomize/observability/fluentd/stdout/clusteroutput.yaml @@ -0,0 +1,9 @@ +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterOutput +metadata: + name: stdout + labels: + output.fluentd.fluent.io/enabled: "true" +spec: + outputs: + - stdout: {} diff --git a/kustomize/telemetry/resources/fluentbit/quickwit/kustomization.yaml b/kustomize/observability/fluentd/stdout/kustomization.yaml similarity index 100% rename from kustomize/telemetry/resources/fluentbit/quickwit/kustomization.yaml rename to kustomize/observability/fluentd/stdout/kustomization.yaml diff --git a/kustomize/observability/grafana/quickwit/kustomization.yaml b/kustomize/observability/grafana/quickwit/kustomization.yaml index 48836433..925b2cd2 100644 --- a/kustomize/observability/grafana/quickwit/kustomization.yaml +++ b/kustomize/observability/grafana/quickwit/kustomization.yaml @@ -1,6 +1,13 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component -resources: - - index.yaml +# resources: +# - index.yaml patches: - path: patches/helm-release.yaml + - target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: grafana + namespace: system-observability + path: patches/patch.json \ No newline at end of file diff --git a/kustomize/observability/grafana/quickwit/patches/patch.json b/kustomize/observability/grafana/quickwit/patches/patch.json new file mode 100644 index 00000000..6ff9ac5e --- /dev/null +++ b/kustomize/observability/grafana/quickwit/patches/patch.json @@ -0,0 +1,15 @@ +[ + { + "op": "add", + "path": "/spec/values/datasources/datasources.yaml/datasources/-", + "value": { + "name": "Quickwit Logs", + "uid": "quickwit-logs", + "type": "quickwit-quickwit-datasource", + "url": "http://quickwit-searcher.system-observability.svc.cluster.local:7280/api/v1", + "jsonData": { + "index": "otel-logs-v0_7" + } + } + } +] diff --git a/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml b/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml deleted file mode 100644 index de1cfdff..00000000 --- a/kustomize/telemetry/resources/fluentbit/quickwit/clusteroutput.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: fluentbit.fluent.io/v1alpha2 -kind: ClusterOutput -metadata: - labels: - fluentbit.fluent.io/component: logging - fluentbit.fluent.io/enabled: "true" - name: quickwit -spec: - http: - host: quickwit-indexer.system-observability.svc.cluster.local - port: 7280 - uri: /api/v1/fluentbit-logs/ingest - format: json - matchRegex: (?:kube|service)\.(.*) From e83f386432e7549b3831c7a1e26ce486b1d1b45c Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 14 Jul 2025 10:39:00 +0200 Subject: [PATCH 07/21] fix blueprint --- contexts/local/blueprint.quickwit-local.yaml | 10 +++ contexts/local/blueprint.quickwit-minio.yaml | 89 ++++++++++---------- contexts/local/blueprint.yaml | 6 -- 3 files changed, 53 insertions(+), 52 deletions(-) diff --git a/contexts/local/blueprint.quickwit-local.yaml b/contexts/local/blueprint.quickwit-local.yaml index 0206858d..31cb5083 100644 --- a/contexts/local/blueprint.quickwit-local.yaml +++ b/contexts/local/blueprint.quickwit-local.yaml @@ -23,6 +23,8 @@ kustomize: components: - prometheus - prometheus/flux + - fluentbit + - fluentbit/prometheus - name: telemetry-resources path: telemetry/resources dependsOn: @@ -31,6 +33,11 @@ kustomize: - metrics-server - prometheus - prometheus/flux + - fluentbit + - fluentbit/containerd + - fluentbit/fluentd + - fluentbit/kubernetes + - fluentbit/systemd - name: policy-base path: policy/base components: @@ -93,6 +100,9 @@ kustomize: - csi - ingress components: + - quickwit + - fluentd + - fluentd/quickwit - grafana - grafana/ingress - grafana/prometheus diff --git a/contexts/local/blueprint.quickwit-minio.yaml b/contexts/local/blueprint.quickwit-minio.yaml index 86e36e6c..e4746214 100644 --- a/contexts/local/blueprint.quickwit-minio.yaml +++ b/contexts/local/blueprint.quickwit-minio.yaml @@ -4,7 +4,7 @@ metadata: name: local description: This blueprint outlines resources in the local context repository: - url: http://git.test/git/telemetry-test + url: http://git.test/git/core ref: branch: main secretName: flux-system @@ -12,38 +12,51 @@ sources: - name: core url: github.com/windsorcli/core ref: - tag: v0.2.0 + branch: main terraform: -- source: core - path: cluster/talos -- source: core - path: gitops/flux +- path: cluster/talos +- path: gitops/flux + destroy: false kustomize: +- name: telemetry-base + path: telemetry/base + components: + - prometheus + - prometheus/flux + - fluentbit + - fluentbit/prometheus +- name: telemetry-resources + path: telemetry/resources + dependsOn: + - telemetry-base + components: + - metrics-server + - prometheus + - prometheus/flux + - fluentbit + - fluentbit/containerd + - fluentbit/fluentd + - fluentbit/kubernetes + - fluentbit/systemd - name: policy-base path: policy/base - source: core components: - kyverno - name: policy-resources path: policy/resources - source: core dependsOn: - policy-base - name: csi path: csi - source: core dependsOn: - policy-resources - force: true components: - openebs - openebs/dynamic-localpv -- name: ingress-base - path: ingress/base - source: core +- name: ingress + path: ingress dependsOn: - pki-resources - force: true components: - nginx - nginx/nodeport @@ -52,29 +65,22 @@ kustomize: - nginx/web - name: pki-base path: pki/base - source: core dependsOn: - policy-resources - force: true components: - cert-manager - trust-manager - name: pki-resources path: pki/resources - source: core dependsOn: - pki-base - force: true components: - private-issuer/ca - public-issuer/selfsigned - name: dns path: dns - source: core dependsOn: - - ingress-base - pki-base - force: true components: - coredns - coredns/etcd @@ -84,32 +90,26 @@ kustomize: - external-dns/ingress - name: gitops path: gitops/flux - source: core dependsOn: - - ingress-base - force: true + - ingress components: - webhook -- name: demo - path: demo/bookinfo - source: core +- name: observability + path: observability dependsOn: - - ingress-base - force: true + - csi + - ingress components: - - ingress -- name: fluentbit - path: telemetry/base - components: - - fluentbit -- name: fluentbit-resources - path: telemetry/resources - components: - - fluentbit -- name: metrics-server-resources - path: telemetry/resources - components: - - metrics-server + - quickwit + - fluentd + - fluentd/quickwit + - grafana + - grafana/ingress + - grafana/prometheus + - grafana/node + - grafana/kubernetes + - grafana/flux + - grafana/quickwit - name: object-store-base path: object-store/base components: @@ -120,13 +120,10 @@ kustomize: - object-store-base components: - minio -- name: observability-base - path: observability/base - name: quickwit path: observability/quickwit dependsOn: - - observability-base - - pki-base + - observability - object-store-resources components: - minio diff --git a/contexts/local/blueprint.yaml b/contexts/local/blueprint.yaml index 3e1228b7..d8a65bd7 100644 --- a/contexts/local/blueprint.yaml +++ b/contexts/local/blueprint.yaml @@ -110,9 +110,3 @@ kustomize: - grafana/kubernetes - grafana/flux - grafana/quickwit -- name: quickwit-backend - path: observability/quickwit - dependsOn: - - observability - components: - - pvc From 65b9338cf787feaf228f60aa85df9ad312859530 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 14 Jul 2025 10:40:27 +0200 Subject: [PATCH 08/21] fixes --- .../observability/grafana/quickwit/index.yaml | 37 ------------------- .../grafana/quickwit/kustomization.yaml | 4 +- .../quickwit/patches/helm-release.yaml | 2 +- 3 files changed, 2 insertions(+), 41 deletions(-) delete mode 100644 kustomize/observability/grafana/quickwit/index.yaml diff --git a/kustomize/observability/grafana/quickwit/index.yaml b/kustomize/observability/grafana/quickwit/index.yaml deleted file mode 100644 index 4bc0fe2c..00000000 --- a/kustomize/observability/grafana/quickwit/index.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: create-quickwit-index - namespace: system-observability -spec: - template: - spec: - containers: - - name: create-index - image: curlimages/curl:8.14.1 - command: - - /bin/sh - - -c - - | - cat < /tmp/index.yaml - version: 0.7 - - index_id: fluentbit-logs - - doc_mapping: - mode: dynamic - field_mappings: - - name: timestamp - type: datetime - input_formats: - - unix_timestamp - output_format: unix_timestamp_secs - fast: true - timestamp_field: timestamp - - indexing_settings: - commit_timeout_secs: 10 - EOF - - curl -X POST http://quickwit-indexer.system-observability.svc.cluster.local:7280/api/v1/indexes -H "Content-Type: application/yaml" --data-binary @/tmp/index.yaml - restartPolicy: OnFailure \ No newline at end of file diff --git a/kustomize/observability/grafana/quickwit/kustomization.yaml b/kustomize/observability/grafana/quickwit/kustomization.yaml index 925b2cd2..23eec4af 100644 --- a/kustomize/observability/grafana/quickwit/kustomization.yaml +++ b/kustomize/observability/grafana/quickwit/kustomization.yaml @@ -1,7 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component -# resources: -# - index.yaml patches: - path: patches/helm-release.yaml - target: @@ -10,4 +8,4 @@ patches: kind: HelmRelease name: grafana namespace: system-observability - path: patches/patch.json \ No newline at end of file + path: patches/patch.json diff --git a/kustomize/observability/grafana/quickwit/patches/helm-release.yaml b/kustomize/observability/grafana/quickwit/patches/helm-release.yaml index c11d67be..b269cc98 100644 --- a/kustomize/observability/grafana/quickwit/patches/helm-release.yaml +++ b/kustomize/observability/grafana/quickwit/patches/helm-release.yaml @@ -9,4 +9,4 @@ spec: namespace: system-observability values: plugins: - - quickwit-quickwit-datasource \ No newline at end of file + - quickwit-quickwit-datasource From 26bffe98dded46af70976c26626ef6ba35bfc9e1 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Wed, 23 Jul 2025 23:09:25 +0200 Subject: [PATCH 09/21] test --- contexts/local/blueprint.quickwit-local.yaml | 7 +------ contexts/local/blueprint.quickwit-minio.yaml | 9 ++------- contexts/local/blueprint.yaml | 1 + .../observability/fluentd/quickwit/kustomization.yaml | 2 ++ .../fluentd/quickwit/patches/helm-release.yaml | 9 +++++++++ 5 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 kustomize/observability/fluentd/quickwit/patches/helm-release.yaml diff --git a/contexts/local/blueprint.quickwit-local.yaml b/contexts/local/blueprint.quickwit-local.yaml index 31cb5083..372ed8ea 100644 --- a/contexts/local/blueprint.quickwit-local.yaml +++ b/contexts/local/blueprint.quickwit-local.yaml @@ -101,6 +101,7 @@ kustomize: - ingress components: - quickwit + - quickwit/pvc - fluentd - fluentd/quickwit - grafana @@ -110,9 +111,3 @@ kustomize: - grafana/kubernetes - grafana/flux - grafana/quickwit -- name: quickwit - path: observability/quickwit - dependsOn: - - pki-base - components: - - pvc diff --git a/contexts/local/blueprint.quickwit-minio.yaml b/contexts/local/blueprint.quickwit-minio.yaml index e4746214..1ae12373 100644 --- a/contexts/local/blueprint.quickwit-minio.yaml +++ b/contexts/local/blueprint.quickwit-minio.yaml @@ -99,8 +99,10 @@ kustomize: dependsOn: - csi - ingress + - object-store-resources components: - quickwit + - quickwit/minio - fluentd - fluentd/quickwit - grafana @@ -120,10 +122,3 @@ kustomize: - object-store-base components: - minio -- name: quickwit - path: observability/quickwit - dependsOn: - - observability - - object-store-resources - components: - - minio diff --git a/contexts/local/blueprint.yaml b/contexts/local/blueprint.yaml index d8a65bd7..372ed8ea 100644 --- a/contexts/local/blueprint.yaml +++ b/contexts/local/blueprint.yaml @@ -101,6 +101,7 @@ kustomize: - ingress components: - quickwit + - quickwit/pvc - fluentd - fluentd/quickwit - grafana diff --git a/kustomize/observability/fluentd/quickwit/kustomization.yaml b/kustomize/observability/fluentd/quickwit/kustomization.yaml index 68328af8..3b06a027 100644 --- a/kustomize/observability/fluentd/quickwit/kustomization.yaml +++ b/kustomize/observability/fluentd/quickwit/kustomization.yaml @@ -3,3 +3,5 @@ kind: Component resources: - clusterfilter.yaml - clusteroutput.yaml +patches: + - path: patches/helm-release.yaml diff --git a/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml b/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml new file mode 100644 index 00000000..00e0a991 --- /dev/null +++ b/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: fluent-operator-config + namespace: system-observability +spec: + dependOn: + - name: quickwit From c9041adbb963c552c6d5bb8a0653173c7ef9fb94 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Wed, 23 Jul 2025 23:20:53 +0200 Subject: [PATCH 10/21] fixes --- contexts/local/blueprint.quickwit-local.yaml | 1 + contexts/local/blueprint.quickwit-minio.yaml | 1 + contexts/local/blueprint.yaml | 1 + .../observability/fluentd/quickwit/patches/helm-release.yaml | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/contexts/local/blueprint.quickwit-local.yaml b/contexts/local/blueprint.quickwit-local.yaml index 372ed8ea..d53f6fb3 100644 --- a/contexts/local/blueprint.quickwit-local.yaml +++ b/contexts/local/blueprint.quickwit-local.yaml @@ -104,6 +104,7 @@ kustomize: - quickwit/pvc - fluentd - fluentd/quickwit + - fluentd/stdout - grafana - grafana/ingress - grafana/prometheus diff --git a/contexts/local/blueprint.quickwit-minio.yaml b/contexts/local/blueprint.quickwit-minio.yaml index 1ae12373..9d1740ef 100644 --- a/contexts/local/blueprint.quickwit-minio.yaml +++ b/contexts/local/blueprint.quickwit-minio.yaml @@ -105,6 +105,7 @@ kustomize: - quickwit/minio - fluentd - fluentd/quickwit + - fluentd/stdout - grafana - grafana/ingress - grafana/prometheus diff --git a/contexts/local/blueprint.yaml b/contexts/local/blueprint.yaml index 372ed8ea..d53f6fb3 100644 --- a/contexts/local/blueprint.yaml +++ b/contexts/local/blueprint.yaml @@ -104,6 +104,7 @@ kustomize: - quickwit/pvc - fluentd - fluentd/quickwit + - fluentd/stdout - grafana - grafana/ingress - grafana/prometheus diff --git a/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml b/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml index 00e0a991..004f3a97 100644 --- a/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml +++ b/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml @@ -2,7 +2,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: - name: fluent-operator-config + name: fluent-operator namespace: system-observability spec: dependOn: From 86c00d6f468f2b656876735d389301bac5beb3a7 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Wed, 23 Jul 2025 23:22:38 +0200 Subject: [PATCH 11/21] test --- kustomize/observability/fluentd/quickwit/kustomization.yaml | 2 -- .../observability/fluentd/quickwit/patches/helm-release.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/kustomize/observability/fluentd/quickwit/kustomization.yaml b/kustomize/observability/fluentd/quickwit/kustomization.yaml index 3b06a027..68328af8 100644 --- a/kustomize/observability/fluentd/quickwit/kustomization.yaml +++ b/kustomize/observability/fluentd/quickwit/kustomization.yaml @@ -3,5 +3,3 @@ kind: Component resources: - clusterfilter.yaml - clusteroutput.yaml -patches: - - path: patches/helm-release.yaml diff --git a/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml b/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml index 004f3a97..00e0a991 100644 --- a/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml +++ b/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml @@ -2,7 +2,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: - name: fluent-operator + name: fluent-operator-config namespace: system-observability spec: dependOn: From 733b151392bb4c3a9fbd558eace6fa1c4d9764e8 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Wed, 23 Jul 2025 23:28:35 +0200 Subject: [PATCH 12/21] cleaning up --- .../fluentd/quickwit/patches/helm-release.yaml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 kustomize/observability/fluentd/quickwit/patches/helm-release.yaml diff --git a/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml b/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml deleted file mode 100644 index 00e0a991..00000000 --- a/kustomize/observability/fluentd/quickwit/patches/helm-release.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: fluent-operator-config - namespace: system-observability -spec: - dependOn: - - name: quickwit From cfb5940e69322da623cb6f4a4f8f625e1a9fb1c0 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Thu, 24 Jul 2025 16:10:54 +0200 Subject: [PATCH 13/21] remove context --- contexts/local/blueprint.original.yaml | 85 ------------- contexts/local/blueprint.quickwit-local.yaml | 114 ----------------- contexts/local/blueprint.quickwit-minio.yaml | 125 ------------------- contexts/local/blueprint.yaml | 114 ----------------- 4 files changed, 438 deletions(-) delete mode 100644 contexts/local/blueprint.original.yaml delete mode 100644 contexts/local/blueprint.quickwit-local.yaml delete mode 100644 contexts/local/blueprint.quickwit-minio.yaml delete mode 100644 contexts/local/blueprint.yaml diff --git a/contexts/local/blueprint.original.yaml b/contexts/local/blueprint.original.yaml deleted file mode 100644 index ec4ca744..00000000 --- a/contexts/local/blueprint.original.yaml +++ /dev/null @@ -1,85 +0,0 @@ -kind: Blueprint -apiVersion: blueprints.windsorcli.dev/v1alpha1 -metadata: - name: local - description: This blueprint outlines resources in the local context -repository: - url: http://git.test/git/core - ref: - branch: main - secretName: flux-system -sources: [] -terraform: -- path: cluster/talos -- path: gitops/flux -kustomize: -- name: policy-base - path: policy/base - components: - - kyverno -- name: policy-resources - path: policy/resources - dependsOn: - - policy-base -- name: csi - path: csi - dependsOn: - - policy-resources - force: true - components: - - openebs - - openebs/dynamic-localpv -- name: ingress-base - path: ingress/base - dependsOn: - - pki-resources - force: true - components: - - nginx - - nginx/nodeport - - nginx/coredns - - nginx/flux-webhook - - nginx/web -- name: pki-base - path: pki/base - dependsOn: - - policy-resources - force: true - components: - - cert-manager - - trust-manager -- name: pki-resources - path: pki/resources - dependsOn: - - pki-base - force: true - components: - - private-issuer/ca - - public-issuer/selfsigned -- name: dns - path: dns - dependsOn: - - ingress-base - - pki-base - force: true - components: - - coredns - - coredns/etcd - - external-dns - - external-dns/localhost - - external-dns/coredns - - external-dns/ingress -- name: gitops - path: gitops/flux - dependsOn: - - ingress-base - force: true - components: - - webhook -- name: demo - path: demo/bookinfo - dependsOn: - - ingress-base - force: true - components: - - ingress diff --git a/contexts/local/blueprint.quickwit-local.yaml b/contexts/local/blueprint.quickwit-local.yaml deleted file mode 100644 index d53f6fb3..00000000 --- a/contexts/local/blueprint.quickwit-local.yaml +++ /dev/null @@ -1,114 +0,0 @@ -kind: Blueprint -apiVersion: blueprints.windsorcli.dev/v1alpha1 -metadata: - name: local - description: This blueprint outlines resources in the local context -repository: - url: http://git.test/git/core - ref: - branch: main - secretName: flux-system -sources: -- name: core - url: github.com/windsorcli/core - ref: - branch: main -terraform: -- path: cluster/talos -- path: gitops/flux - destroy: false -kustomize: -- name: telemetry-base - path: telemetry/base - components: - - prometheus - - prometheus/flux - - fluentbit - - fluentbit/prometheus -- name: telemetry-resources - path: telemetry/resources - dependsOn: - - telemetry-base - components: - - metrics-server - - prometheus - - prometheus/flux - - fluentbit - - fluentbit/containerd - - fluentbit/fluentd - - fluentbit/kubernetes - - fluentbit/systemd -- name: policy-base - path: policy/base - components: - - kyverno -- name: policy-resources - path: policy/resources - dependsOn: - - policy-base -- name: csi - path: csi - dependsOn: - - policy-resources - components: - - openebs - - openebs/dynamic-localpv -- name: ingress - path: ingress - dependsOn: - - pki-resources - components: - - nginx - - nginx/nodeport - - nginx/coredns - - nginx/flux-webhook - - nginx/web -- name: pki-base - path: pki/base - dependsOn: - - policy-resources - components: - - cert-manager - - trust-manager -- name: pki-resources - path: pki/resources - dependsOn: - - pki-base - components: - - private-issuer/ca - - public-issuer/selfsigned -- name: dns - path: dns - dependsOn: - - pki-base - components: - - coredns - - coredns/etcd - - external-dns - - external-dns/localhost - - external-dns/coredns - - external-dns/ingress -- name: gitops - path: gitops/flux - dependsOn: - - ingress - components: - - webhook -- name: observability - path: observability - dependsOn: - - csi - - ingress - components: - - quickwit - - quickwit/pvc - - fluentd - - fluentd/quickwit - - fluentd/stdout - - grafana - - grafana/ingress - - grafana/prometheus - - grafana/node - - grafana/kubernetes - - grafana/flux - - grafana/quickwit diff --git a/contexts/local/blueprint.quickwit-minio.yaml b/contexts/local/blueprint.quickwit-minio.yaml deleted file mode 100644 index 9d1740ef..00000000 --- a/contexts/local/blueprint.quickwit-minio.yaml +++ /dev/null @@ -1,125 +0,0 @@ -kind: Blueprint -apiVersion: blueprints.windsorcli.dev/v1alpha1 -metadata: - name: local - description: This blueprint outlines resources in the local context -repository: - url: http://git.test/git/core - ref: - branch: main - secretName: flux-system -sources: -- name: core - url: github.com/windsorcli/core - ref: - branch: main -terraform: -- path: cluster/talos -- path: gitops/flux - destroy: false -kustomize: -- name: telemetry-base - path: telemetry/base - components: - - prometheus - - prometheus/flux - - fluentbit - - fluentbit/prometheus -- name: telemetry-resources - path: telemetry/resources - dependsOn: - - telemetry-base - components: - - metrics-server - - prometheus - - prometheus/flux - - fluentbit - - fluentbit/containerd - - fluentbit/fluentd - - fluentbit/kubernetes - - fluentbit/systemd -- name: policy-base - path: policy/base - components: - - kyverno -- name: policy-resources - path: policy/resources - dependsOn: - - policy-base -- name: csi - path: csi - dependsOn: - - policy-resources - components: - - openebs - - openebs/dynamic-localpv -- name: ingress - path: ingress - dependsOn: - - pki-resources - components: - - nginx - - nginx/nodeport - - nginx/coredns - - nginx/flux-webhook - - nginx/web -- name: pki-base - path: pki/base - dependsOn: - - policy-resources - components: - - cert-manager - - trust-manager -- name: pki-resources - path: pki/resources - dependsOn: - - pki-base - components: - - private-issuer/ca - - public-issuer/selfsigned -- name: dns - path: dns - dependsOn: - - pki-base - components: - - coredns - - coredns/etcd - - external-dns - - external-dns/localhost - - external-dns/coredns - - external-dns/ingress -- name: gitops - path: gitops/flux - dependsOn: - - ingress - components: - - webhook -- name: observability - path: observability - dependsOn: - - csi - - ingress - - object-store-resources - components: - - quickwit - - quickwit/minio - - fluentd - - fluentd/quickwit - - fluentd/stdout - - grafana - - grafana/ingress - - grafana/prometheus - - grafana/node - - grafana/kubernetes - - grafana/flux - - grafana/quickwit -- name: object-store-base - path: object-store/base - components: - - minio -- name: object-store-resources - path: object-store/resources - dependsOn: - - object-store-base - components: - - minio diff --git a/contexts/local/blueprint.yaml b/contexts/local/blueprint.yaml deleted file mode 100644 index d53f6fb3..00000000 --- a/contexts/local/blueprint.yaml +++ /dev/null @@ -1,114 +0,0 @@ -kind: Blueprint -apiVersion: blueprints.windsorcli.dev/v1alpha1 -metadata: - name: local - description: This blueprint outlines resources in the local context -repository: - url: http://git.test/git/core - ref: - branch: main - secretName: flux-system -sources: -- name: core - url: github.com/windsorcli/core - ref: - branch: main -terraform: -- path: cluster/talos -- path: gitops/flux - destroy: false -kustomize: -- name: telemetry-base - path: telemetry/base - components: - - prometheus - - prometheus/flux - - fluentbit - - fluentbit/prometheus -- name: telemetry-resources - path: telemetry/resources - dependsOn: - - telemetry-base - components: - - metrics-server - - prometheus - - prometheus/flux - - fluentbit - - fluentbit/containerd - - fluentbit/fluentd - - fluentbit/kubernetes - - fluentbit/systemd -- name: policy-base - path: policy/base - components: - - kyverno -- name: policy-resources - path: policy/resources - dependsOn: - - policy-base -- name: csi - path: csi - dependsOn: - - policy-resources - components: - - openebs - - openebs/dynamic-localpv -- name: ingress - path: ingress - dependsOn: - - pki-resources - components: - - nginx - - nginx/nodeport - - nginx/coredns - - nginx/flux-webhook - - nginx/web -- name: pki-base - path: pki/base - dependsOn: - - policy-resources - components: - - cert-manager - - trust-manager -- name: pki-resources - path: pki/resources - dependsOn: - - pki-base - components: - - private-issuer/ca - - public-issuer/selfsigned -- name: dns - path: dns - dependsOn: - - pki-base - components: - - coredns - - coredns/etcd - - external-dns - - external-dns/localhost - - external-dns/coredns - - external-dns/ingress -- name: gitops - path: gitops/flux - dependsOn: - - ingress - components: - - webhook -- name: observability - path: observability - dependsOn: - - csi - - ingress - components: - - quickwit - - quickwit/pvc - - fluentd - - fluentd/quickwit - - fluentd/stdout - - grafana - - grafana/ingress - - grafana/prometheus - - grafana/node - - grafana/kubernetes - - grafana/flux - - grafana/quickwit From 24721dc7255a6efaea4814a19fd2bd16a6182015 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Thu, 24 Jul 2025 16:12:34 +0200 Subject: [PATCH 14/21] quickwit: remove minio --- .../observability/quickwit/minio/bucket.yaml | 21 ---------- .../quickwit/minio/configmap.yaml | 10 ----- .../quickwit/minio/kustomization.yaml | 7 ---- .../quickwit/minio/patches/helm-release.yaml | 41 ------------------- 4 files changed, 79 deletions(-) delete mode 100644 kustomize/observability/quickwit/minio/bucket.yaml delete mode 100644 kustomize/observability/quickwit/minio/configmap.yaml delete mode 100644 kustomize/observability/quickwit/minio/kustomization.yaml delete mode 100644 kustomize/observability/quickwit/minio/patches/helm-release.yaml diff --git a/kustomize/observability/quickwit/minio/bucket.yaml b/kustomize/observability/quickwit/minio/bucket.yaml deleted file mode 100644 index 1bb3eb11..00000000 --- a/kustomize/observability/quickwit/minio/bucket.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: minio-quickwit-bucket - namespace: system-object-store -spec: - interval: 5m - timeout: 5m - chart: - spec: - chart: charts/minio-bucket - sourceRef: - kind: GitRepository - name: core - namespace: system-gitops - values: - bucket: quickwit - secretNamespace: system-observability - minioEndpoint: "https://minio.system-object-store.svc.cluster.local:443" - podLabels: - use-custom-ca: "true" diff --git a/kustomize/observability/quickwit/minio/configmap.yaml b/kustomize/observability/quickwit/minio/configmap.yaml deleted file mode 100644 index 2f8895ef..00000000 --- a/kustomize/observability/quickwit/minio/configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# This does nothing important until https://github.com/quickwit-oss/quickwit/issues/5199 is resolved ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio-sts-env - namespace: system-observability -data: - AWS_WEB_IDENTITY_TOKEN_FILE: "/var/run/secrets/kubernetes.io/serviceaccount/token" - AWS_S3_ENDPOINT: "https://minio.system-object-store.svc.cluster.local" diff --git a/kustomize/observability/quickwit/minio/kustomization.yaml b/kustomize/observability/quickwit/minio/kustomization.yaml deleted file mode 100644 index 69c2a6ba..00000000 --- a/kustomize/observability/quickwit/minio/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1alpha1 -kind: Component -resources: - - configmap.yaml - - bucket.yaml -patches: - - path: patches/helm-release.yaml diff --git a/kustomize/observability/quickwit/minio/patches/helm-release.yaml b/kustomize/observability/quickwit/minio/patches/helm-release.yaml deleted file mode 100644 index c0b983ce..00000000 --- a/kustomize/observability/quickwit/minio/patches/helm-release.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: quickwit - namespace: system-observability -spec: - dependsOn: - - name: minio-quickwit-bucket - namespace: system-object-store - values: - additionalLabels: - use-custom-ca: "true" - config: - default_index_root_uri: s3://quickwit/indexes - storage: - s3: - endpoint: https://minio.system-object-store.svc.cluster.local - flavor: minio - region: "us-east-1" - valuesFrom: - - kind: Secret - name: minio-quickwit-keys - valuesKey: access_key - targetPath: config.storage.s3.access_key_id - - kind: Secret - name: minio-quickwit-keys - valuesKey: secret_key - targetPath: config.storage.s3.secret_access_key - - # NOTE: Reinstate this after resolution of https://github.com/quickwit-oss/quickwit/issues/5199 - # indexer: - # extraEnvFrom: - # - configMapRef: - # name: minio-sts-env - # searcher: - # extraEnvFrom: - # - configMapRef: - # name: minio-sts-env - # serviceAccount: - # name: quickwit From 89388e90a58ef1c69b2053d6f3fabfa506073aa9 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Thu, 24 Jul 2025 16:19:58 +0200 Subject: [PATCH 15/21] fix blueprint --- contexts/_template/blueprint.jsonnet | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/contexts/_template/blueprint.jsonnet b/contexts/_template/blueprint.jsonnet index d5007aad..435682c7 100644 --- a/contexts/_template/blueprint.jsonnet +++ b/contexts/_template/blueprint.jsonnet @@ -193,7 +193,12 @@ local kustomizeConfigs = { ], components: [ "prometheus", - "prometheus/flux" + "prometheus/flux", + "fluentbit", + "fluentbit/containerd", + "fluentbit/fluentd", + "fluentbit/kubernetes", + "fluentbit/systemd" ], }, { @@ -261,12 +266,18 @@ local kustomizeConfigs = { "ingress" ], components: [ + "quickwit", + "quickwit/pvc", + "fluentd", + "fluentd/quickwit", + "fluentd/stdout", "grafana", "grafana/ingress", "grafana/prometheus", "grafana/node", "grafana/kubernetes", - "grafana/flux" + "grafana/flux", + "grafana/quickwit" ], } ], From aa6d1f0f7e2080fe35307108658a39310d718bd6 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Thu, 24 Jul 2025 15:11:49 -0400 Subject: [PATCH 16/21] Ignore all contexts other than _template --- contexts/.gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contexts/.gitignore b/contexts/.gitignore index 4d93ab0a..50943667 100644 --- a/contexts/.gitignore +++ b/contexts/.gitignore @@ -1 +1,4 @@ +# Ignore all directories in contexts/ +*/ +# But allow _template directory !/_template/ From 1ba56720aa1ebe4c881d6f614bf5708cb8a63540 Mon Sep 17 00:00:00 2001 From: rmvangun <85766511+rmvangun@users.noreply.github.com> Date: Thu, 24 Jul 2025 18:07:06 -0400 Subject: [PATCH 17/21] fix(coredns): Prevent loop when forwarding to self-hosted dns (#706) --- kustomize/dns/coredns/etcd/patches/helm-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/dns/coredns/etcd/patches/helm-release.yaml b/kustomize/dns/coredns/etcd/patches/helm-release.yaml index 0f15d949..63000cc5 100644 --- a/kustomize/dns/coredns/etcd/patches/helm-release.yaml +++ b/kustomize/dns/coredns/etcd/patches/helm-release.yaml @@ -25,7 +25,7 @@ spec: tls /etc/coredns/tls/tls.crt /etc/coredns/tls/tls.key /etc/coredns/tls/ca.crt fallthrough - name: forward - parameters: . /etc/resolv.conf + parameters: . 1.1.1.1 8.8.8.8 - name: loop - name: reload - name: prometheus From a4996d969552491299ef03d1a0aa2a84d40c93ac Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Thu, 24 Jul 2025 18:24:36 -0400 Subject: [PATCH 18/21] Cleanup --- kustomize/observability/base/kustomization.yaml | 2 -- kustomize/observability/base/namespace.yaml | 9 --------- 2 files changed, 11 deletions(-) delete mode 100644 kustomize/observability/base/kustomization.yaml delete mode 100644 kustomize/observability/base/namespace.yaml diff --git a/kustomize/observability/base/kustomization.yaml b/kustomize/observability/base/kustomization.yaml deleted file mode 100644 index 736967b1..00000000 --- a/kustomize/observability/base/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - namespace.yaml diff --git a/kustomize/observability/base/namespace.yaml b/kustomize/observability/base/namespace.yaml deleted file mode 100644 index 5544d5b6..00000000 --- a/kustomize/observability/base/namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: system-observability - labels: - pod-security.kubernetes.io/enforce: baseline - pod-security.kubernetes.io/audit: baseline - pod-security.kubernetes.io/warn: baseline - use-custom-ca: "true" From f2afcdbac0d013a2cedf9f353dd1073ccfaa584f Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Fri, 25 Jul 2025 09:05:53 -0400 Subject: [PATCH 19/21] Accept all incoming logs --- kustomize/observability/fluentd/fluentd.yaml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/kustomize/observability/fluentd/fluentd.yaml b/kustomize/observability/fluentd/fluentd.yaml index 41aea384..ec8e2373 100644 --- a/kustomize/observability/fluentd/fluentd.yaml +++ b/kustomize/observability/fluentd/fluentd.yaml @@ -13,10 +13,8 @@ spec: - forward: bind: 0.0.0.0 port: 24224 - image: ghcr.io/fluent/fluent-operator/fluentd:v1.17.0 - logLevel: info - mode: collector - positionDB: {} + # renovate: datasource=docker depName=ghcr.io/fluent/fluent-operator/fluentd package=ghcr.io/fluent/fluent-operator/fluentd + image: ghcr.io/fluent/fluent-operator/fluentd:v1.17.1 replicas: 1 resources: limits: @@ -25,10 +23,6 @@ spec: requests: cpu: 100m memory: 128Mi - service: {} -status: - messages: all matched cfgs is valid - state: active --- apiVersion: fluentd.fluent.io/v1alpha1 kind: ClusterFluentdConfig @@ -43,9 +37,3 @@ spec: clusterOutputSelector: matchLabels: output.fluentd.fluent.io/enabled: "true" - watchedNamespaces: - - kube-system - - default -status: - messages: Generate fluentd configs successfully - state: valid From 66a52a81d9f9887fcec506d4472f136af5c43fca Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Fri, 25 Jul 2025 09:20:33 -0400 Subject: [PATCH 20/21] Create sub-component folders --- contexts/_template/blueprint.jsonnet | 9 ++++++--- .../{quickwit => filters/otel}/clusterfilter.yaml | 0 .../{quickwit => filters/otel}/kustomization.yaml | 1 - .../fluentd/{ => outputs}/quickwit/clusteroutput.yaml | 0 .../{stdout => outputs/quickwit}/kustomization.yaml | 0 .../fluentd/{ => outputs}/stdout/clusteroutput.yaml | 0 .../fluentd/outputs/stdout/kustomization.yaml | 4 ++++ 7 files changed, 10 insertions(+), 4 deletions(-) rename kustomize/observability/fluentd/{quickwit => filters/otel}/clusterfilter.yaml (100%) rename kustomize/observability/fluentd/{quickwit => filters/otel}/kustomization.yaml (80%) rename kustomize/observability/fluentd/{ => outputs}/quickwit/clusteroutput.yaml (100%) rename kustomize/observability/fluentd/{stdout => outputs/quickwit}/kustomization.yaml (100%) rename kustomize/observability/fluentd/{ => outputs}/stdout/clusteroutput.yaml (100%) create mode 100644 kustomize/observability/fluentd/outputs/stdout/kustomization.yaml diff --git a/contexts/_template/blueprint.jsonnet b/contexts/_template/blueprint.jsonnet index c25b3e02..c0d876ce 100644 --- a/contexts/_template/blueprint.jsonnet +++ b/contexts/_template/blueprint.jsonnet @@ -175,7 +175,8 @@ local kustomizeConfigs = { ], components: [ "fluentd", - "fluentd/quickwit", + "fluentd/filters/otel", + "fluentd/outputs/quickwit", "quickwit", "quickwit/pvc", "grafana", @@ -281,7 +282,8 @@ local kustomizeConfigs = { ], components: [ "fluentd", - "fluentd/quickwit", + "fluentd/filters/otel", + "fluentd/outputs/quickwit", "quickwit", "quickwit/pvc", "grafana", @@ -450,7 +452,8 @@ local kustomizeConfigs = { ], components: [ "fluentd", - "fluentd/quickwit", + "fluentd/filters/otel", + "fluentd/outputs/quickwit", "quickwit", "quickwit/pvc", "grafana", diff --git a/kustomize/observability/fluentd/quickwit/clusterfilter.yaml b/kustomize/observability/fluentd/filters/otel/clusterfilter.yaml similarity index 100% rename from kustomize/observability/fluentd/quickwit/clusterfilter.yaml rename to kustomize/observability/fluentd/filters/otel/clusterfilter.yaml diff --git a/kustomize/observability/fluentd/quickwit/kustomization.yaml b/kustomize/observability/fluentd/filters/otel/kustomization.yaml similarity index 80% rename from kustomize/observability/fluentd/quickwit/kustomization.yaml rename to kustomize/observability/fluentd/filters/otel/kustomization.yaml index 68328af8..d70f1fb4 100644 --- a/kustomize/observability/fluentd/quickwit/kustomization.yaml +++ b/kustomize/observability/fluentd/filters/otel/kustomization.yaml @@ -2,4 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: - clusterfilter.yaml - - clusteroutput.yaml diff --git a/kustomize/observability/fluentd/quickwit/clusteroutput.yaml b/kustomize/observability/fluentd/outputs/quickwit/clusteroutput.yaml similarity index 100% rename from kustomize/observability/fluentd/quickwit/clusteroutput.yaml rename to kustomize/observability/fluentd/outputs/quickwit/clusteroutput.yaml diff --git a/kustomize/observability/fluentd/stdout/kustomization.yaml b/kustomize/observability/fluentd/outputs/quickwit/kustomization.yaml similarity index 100% rename from kustomize/observability/fluentd/stdout/kustomization.yaml rename to kustomize/observability/fluentd/outputs/quickwit/kustomization.yaml diff --git a/kustomize/observability/fluentd/stdout/clusteroutput.yaml b/kustomize/observability/fluentd/outputs/stdout/clusteroutput.yaml similarity index 100% rename from kustomize/observability/fluentd/stdout/clusteroutput.yaml rename to kustomize/observability/fluentd/outputs/stdout/clusteroutput.yaml diff --git a/kustomize/observability/fluentd/outputs/stdout/kustomization.yaml b/kustomize/observability/fluentd/outputs/stdout/kustomization.yaml new file mode 100644 index 00000000..051eb969 --- /dev/null +++ b/kustomize/observability/fluentd/outputs/stdout/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - clusteroutput.yaml From 52cc3197f0018bb454dafcc03068bb300234b3ab Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Fri, 25 Jul 2025 09:24:53 -0400 Subject: [PATCH 21/21] Remove README --- README.md | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/README.md b/README.md index 0f7e4778..5111fa70 100644 --- a/README.md +++ b/README.md @@ -2,46 +2,6 @@ Core Terraform modules and Kubernetes configurations used by [Windsor CLI](https://github.com/windsorcli/cli) to provision and manage infrastructure across multiple cloud providers. ![CI Workflow](https://github.com/your-repo/core/actions/workflows/ci.yaml/badge.svg) - -# Blueprint.yaml - -## Quickwit - -``` -- name: observability-base - path: observability/base -- name: quickwit - path: observability/quickwit - dependsOn: - - observability-base - - pki-base - components: - - local-file - ``` - - ## Metrics Server - ``` - - name: metrics-server-resources - path: telemetry/resources - components: - - metrics-server - ``` - - ## FluentBit - ``` - - name: fluentbit - path: telemetry/base - components: - - fluentbit -- name: fluentbit-resources - path: telemetry/resources - components: - - fluentbit -- name: metrics-server-resources - path: telemetry/resources - components: - - metrics-server - ``` ## Contributing