feat(observability): Add quickwit support with local storage

README.md

@@ -2,7 +2,7 @@
 Core Terraform modules and Kubernetes configurations used by [Windsor CLI](https://github.com/windsorcli/cli) to provision and manage infrastructure across multiple cloud providers.
 
 ![CI Workflow](https://github.com/your-repo/core/actions/workflows/ci.yaml/badge.svg)
-
+  
 ## Contributing
 
 This project uses several tools to maintain code quality and consistency:

contexts/.gitignore

@@ -1 +1,4 @@
+# Ignore all directories in contexts/
+*/
+# But allow _template directory
 !/_template/

contexts/_template/blueprint.jsonnet

@@ -175,12 +175,17 @@ local kustomizeConfigs = {
       ],
       components: [
         "fluentd",
+        "fluentd/filters/otel",
+        "fluentd/outputs/quickwit",
+        "quickwit",
+        "quickwit/pvc",
         "grafana",
         "grafana/ingress",
         "grafana/prometheus",
         "grafana/node",
         "grafana/kubernetes",
-        "grafana/flux"
+        "grafana/flux",
+        "grafana/quickwit"
       ],
     }
   ],
@@ -277,12 +282,17 @@ local kustomizeConfigs = {
       ],
       components: [
         "fluentd",
+        "fluentd/filters/otel",
+        "fluentd/outputs/quickwit",
+        "quickwit",
+        "quickwit/pvc",
         "grafana",
         "grafana/ingress",
         "grafana/prometheus",
         "grafana/node",
         "grafana/kubernetes",
-        "grafana/flux"
+        "grafana/flux",
+        "grafana/quickwit"
       ],
     }
   ],
@@ -442,12 +452,17 @@ local kustomizeConfigs = {
       ],
       components: [
         "fluentd",
+        "fluentd/filters/otel",
+        "fluentd/outputs/quickwit",
+        "quickwit",
+        "quickwit/pvc",
         "grafana",
         "grafana/ingress",
         "grafana/prometheus",
         "grafana/node",
         "grafana/kubernetes",
-        "grafana/flux"
+        "grafana/flux",
+        "grafana/quickwit"
       ],
     }
   ]

kustomize/observability/fluentd/filters/otel/clusterfilter.yaml

@@ -0,0 +1,22 @@
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: ClusterFilter
+metadata:
+  name: otel
+  labels:
+    filter.fluentd.fluent.io/enabled: "true"
+spec:
+  filters:
+    - recordTransformer:
+        enableRuby: true
+        renewRecord: false
+        removeKeys: "logtag,time,log,kubernetes"
+        records:
+          - key: timestamp_nanos
+            value: |
+              $${Time.parse(record["time"]).to_i * 1_000_000_000 + Time.parse(record["time"]).nsec}
+          - key: body
+            value: |
+              $${{"message" => record["log"]}}
+          - key: resource_attributes
+            value: |
+              $${{"pod_name" => record["kubernetes"]["pod_name"], "namespace_name" => record["kubernetes"]["namespace_name"], "container_name" => record["kubernetes"]["container_name"], "container_image_id" => record["kubernetes"]["docker_id"], "container_image" => record["kubernetes"]["container_image"]}}

kustomize/observability/fluentd/filters/otel/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+resources:
+  - clusterfilter.yaml

kustomize/observability/fluentd/fluentd.yaml

@@ -13,10 +13,8 @@ spec:
   - forward:
       bind: 0.0.0.0
       port: 24224
-  image: ghcr.io/fluent/fluent-operator/fluentd:v1.17.0
-  logLevel: info
-  mode: collector
-  positionDB: {}
+  # renovate: datasource=docker depName=ghcr.io/fluent/fluent-operator/fluentd package=ghcr.io/fluent/fluent-operator/fluentd
+  image: ghcr.io/fluent/fluent-operator/fluentd:v1.17.1
   replicas: 1
   resources:
     limits:
@@ -25,10 +23,6 @@ spec:
     requests:
       cpu: 100m
       memory: 128Mi
-  service: {}
-status:
-  messages: all matched cfgs is valid
-  state: active
 ---
 apiVersion: fluentd.fluent.io/v1alpha1
 kind: ClusterFluentdConfig
@@ -43,9 +37,3 @@ spec:
   clusterOutputSelector:
     matchLabels:
       output.fluentd.fluent.io/enabled: "true"
-  watchedNamespaces:
-  - kube-system
-  - default
-status:
-  messages: Generate fluentd configs successfully
-  state: valid

kustomize/observability/fluentd/helm-release.yaml

@@ -12,7 +12,7 @@ spec:
       chart: charts/fluent-operator
       # chart: fluent-operator
       # renovate: datasource=helm depName=fluent-operator package=fluent-operator helmRepo=https://fluent.github.io/helm-charts
-      # version: 3.4.2
+      version: 3.4.2
       sourceRef:
         kind: GitRepository
         name: fluent-operator

kustomize/observability/fluentd/outputs/quickwit/clusteroutput.yaml

@@ -0,0 +1,11 @@
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: ClusterOutput
+metadata:
+  name: quickwit
+  labels:
+    output.fluentd.fluent.io/enabled: "true"
+spec:
+  outputs:
+    - http:
+        endpoint: http://quickwit-indexer.system-observability.svc.cluster.local:7280/api/v1/otel-logs-v0_7/ingest
+        contentType: application/json

kustomize/observability/fluentd/outputs/stdout/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+resources:
+  - clusteroutput.yaml

kustomize/observability/grafana/quickwit/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+patches:
+  - path: patches/helm-release.yaml
+  - target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: grafana
+      namespace: system-observability
+    path: patches/patch.json

kustomize/observability/grafana/quickwit/patches/helm-release.yaml

@@ -0,0 +1,12 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: grafana
+  namespace: system-observability
+spec:
+  dependsOn:
+    - name: quickwit
+      namespace: system-observability
+  values:
+    plugins:
+      - quickwit-quickwit-datasource

kustomize/observability/grafana/quickwit/patches/patch.json

@@ -0,0 +1,15 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/values/datasources/datasources.yaml/datasources/-",
+    "value": {
+      "name": "Quickwit Logs",
+      "uid": "quickwit-logs",
+      "type": "quickwit-quickwit-datasource",
+      "url": "http://quickwit-searcher.system-observability.svc.cluster.local:7280/api/v1",
+      "jsonData": {
+        "index": "otel-logs-v0_7"
+      }
+    }
+  }
+]

kustomize/observability/quickwit/helm-release.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: quickwit
+  namespace: system-observability
+spec:
+  interval: 5m
+  timeout: 10m
+  chart:
+    spec:
+      chart: quickwit
+      # renovate: datasource=helm depName=quickwit package=quickwit helmRepo=https://helm.quickwit.io
+      version: 0.7.17
+      sourceRef: 
+        kind: HelmRepository
+        name: quickwit
+        namespace: system-gitops
+  values:
+    searcher:
+      replicaCount: 1

kustomize/observability/quickwit/helm-repository.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: quickwit
+  namespace: system-gitops
+spec:
+  interval: 10m
+  timeout: 3m
+  url: https://helm.quickwit.io

kustomize/observability/quickwit/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+resources:
+  - helm-repository.yaml
+  - helm-release.yaml

kustomize/observability/quickwit/pvc/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+resources:
+  - volume.yaml
+patches:
+  - path: patches/helm-release.yaml

kustomize/observability/quickwit/pvc/patches/helm-release.yaml

@@ -0,0 +1,44 @@
+---
+# SECURITY NOTE: This approach uses a hostPath volume to store the indexes on the host machine.
+# It also requires elevated privileges. Do not use in production. For a more secure setup,
+# integrate with backing services such as Postgres, Minio, or S3.
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: quickwit
+  namespace: system-observability
+spec:
+  values:
+    environment:
+      QW_METASTORE_URI: file:///quickwit/qwdata/indexes
+    config:
+      default_index_root_uri: file:///quickwit/qwdata/indexes
+    podSecurityContext:
+      runAsNonRoot: false
+      runAsUser: 0
+      runAsGroup: 0
+      allowPrivilegeEscalation: true
+    securityContext:
+      runAsNonRoot: false
+      runAsUser: 0
+      runAsGroup: 0
+      allowPrivilegeEscalation: true
+    searcher:
+      replicaCount: 1
+      extraVolumes:
+        - name: quickwit-indexes
+          persistentVolumeClaim:
+            claimName: quickwit-indexes
+      extraVolumeMounts:
+        - name: quickwit-indexes
+          mountPath: /quickwit/qwdata/indexes
+          readOnly: true
+    indexer:
+      extraVolumes:
+        - name: quickwit-indexes
+          persistentVolumeClaim:
+            claimName: quickwit-indexes
+      extraVolumeMounts:
+        - name: quickwit-indexes
+          mountPath: /quickwit/qwdata/indexes
+          readOnly: false

kustomize/observability/quickwit/pvc/volume.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: quickwit-indexes
+  namespace: system-observability
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi