diff --git a/README.md b/README.md index 8d0c5d51..5111fa70 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Core Terraform modules and Kubernetes configurations used by [Windsor CLI](https://github.com/windsorcli/cli) to provision and manage infrastructure across multiple cloud providers. ![CI Workflow](https://github.com/your-repo/core/actions/workflows/ci.yaml/badge.svg) - + ## Contributing This project uses several tools to maintain code quality and consistency: diff --git a/contexts/.gitignore b/contexts/.gitignore index 4d93ab0a..50943667 100644 --- a/contexts/.gitignore +++ b/contexts/.gitignore @@ -1 +1,4 @@ +# Ignore all directories in contexts/ +*/ +# But allow _template directory !/_template/ diff --git a/contexts/_template/blueprint.jsonnet b/contexts/_template/blueprint.jsonnet index 54cde2ee..c0d876ce 100644 --- a/contexts/_template/blueprint.jsonnet +++ b/contexts/_template/blueprint.jsonnet @@ -175,12 +175,17 @@ local kustomizeConfigs = { ], components: [ "fluentd", + "fluentd/filters/otel", + "fluentd/outputs/quickwit", + "quickwit", + "quickwit/pvc", "grafana", "grafana/ingress", "grafana/prometheus", "grafana/node", "grafana/kubernetes", - "grafana/flux" + "grafana/flux", + "grafana/quickwit" ], } ], @@ -277,12 +282,17 @@ local kustomizeConfigs = { ], components: [ "fluentd", + "fluentd/filters/otel", + "fluentd/outputs/quickwit", + "quickwit", + "quickwit/pvc", "grafana", "grafana/ingress", "grafana/prometheus", "grafana/node", "grafana/kubernetes", - "grafana/flux" + "grafana/flux", + "grafana/quickwit" ], } ], @@ -442,12 +452,17 @@ local kustomizeConfigs = { ], components: [ "fluentd", + "fluentd/filters/otel", + "fluentd/outputs/quickwit", + "quickwit", + "quickwit/pvc", "grafana", "grafana/ingress", "grafana/prometheus", "grafana/node", "grafana/kubernetes", - "grafana/flux" + "grafana/flux", + "grafana/quickwit" ], } ] diff --git a/kustomize/observability/base/kustomization.yaml b/kustomize/observability/base/kustomization.yaml deleted file mode 100644 index 736967b1..00000000 --- a/kustomize/observability/base/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - namespace.yaml diff --git a/kustomize/observability/base/namespace.yaml b/kustomize/observability/base/namespace.yaml deleted file mode 100644 index 5544d5b6..00000000 --- a/kustomize/observability/base/namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: system-observability - labels: - pod-security.kubernetes.io/enforce: baseline - pod-security.kubernetes.io/audit: baseline - pod-security.kubernetes.io/warn: baseline - use-custom-ca: "true" diff --git a/kustomize/observability/fluentd/filters/otel/clusterfilter.yaml b/kustomize/observability/fluentd/filters/otel/clusterfilter.yaml new file mode 100644 index 00000000..51b2a269 --- /dev/null +++ b/kustomize/observability/fluentd/filters/otel/clusterfilter.yaml @@ -0,0 +1,22 @@ +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterFilter +metadata: + name: otel + labels: + filter.fluentd.fluent.io/enabled: "true" +spec: + filters: + - recordTransformer: + enableRuby: true + renewRecord: false + removeKeys: "logtag,time,log,kubernetes" + records: + - key: timestamp_nanos + value: | + $${Time.parse(record["time"]).to_i * 1_000_000_000 + Time.parse(record["time"]).nsec} + - key: body + value: | + $${{"message" => record["log"]}} + - key: resource_attributes + value: | + $${{"pod_name" => record["kubernetes"]["pod_name"], "namespace_name" => record["kubernetes"]["namespace_name"], "container_name" => record["kubernetes"]["container_name"], "container_image_id" => record["kubernetes"]["docker_id"], "container_image" => record["kubernetes"]["container_image"]}} diff --git a/kustomize/observability/fluentd/filters/otel/kustomization.yaml b/kustomize/observability/fluentd/filters/otel/kustomization.yaml new file mode 100644 index 00000000..d70f1fb4 --- /dev/null +++ b/kustomize/observability/fluentd/filters/otel/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - clusterfilter.yaml diff --git a/kustomize/observability/fluentd/fluentd.yaml b/kustomize/observability/fluentd/fluentd.yaml index 41aea384..ec8e2373 100644 --- a/kustomize/observability/fluentd/fluentd.yaml +++ b/kustomize/observability/fluentd/fluentd.yaml @@ -13,10 +13,8 @@ spec: - forward: bind: 0.0.0.0 port: 24224 - image: ghcr.io/fluent/fluent-operator/fluentd:v1.17.0 - logLevel: info - mode: collector - positionDB: {} + # renovate: datasource=docker depName=ghcr.io/fluent/fluent-operator/fluentd package=ghcr.io/fluent/fluent-operator/fluentd + image: ghcr.io/fluent/fluent-operator/fluentd:v1.17.1 replicas: 1 resources: limits: @@ -25,10 +23,6 @@ spec: requests: cpu: 100m memory: 128Mi - service: {} -status: - messages: all matched cfgs is valid - state: active --- apiVersion: fluentd.fluent.io/v1alpha1 kind: ClusterFluentdConfig @@ -43,9 +37,3 @@ spec: clusterOutputSelector: matchLabels: output.fluentd.fluent.io/enabled: "true" - watchedNamespaces: - - kube-system - - default -status: - messages: Generate fluentd configs successfully - state: valid diff --git a/kustomize/observability/fluentd/helm-release.yaml b/kustomize/observability/fluentd/helm-release.yaml index 67289895..fd37b734 100644 --- a/kustomize/observability/fluentd/helm-release.yaml +++ b/kustomize/observability/fluentd/helm-release.yaml @@ -12,7 +12,7 @@ spec: chart: charts/fluent-operator # chart: fluent-operator # renovate: datasource=helm depName=fluent-operator package=fluent-operator helmRepo=https://fluent.github.io/helm-charts - # version: 3.4.2 + version: 3.4.2 sourceRef: kind: GitRepository name: fluent-operator diff --git a/kustomize/observability/fluentd/outputs/quickwit/clusteroutput.yaml b/kustomize/observability/fluentd/outputs/quickwit/clusteroutput.yaml new file mode 100644 index 00000000..772bdd3c --- /dev/null +++ b/kustomize/observability/fluentd/outputs/quickwit/clusteroutput.yaml @@ -0,0 +1,11 @@ +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterOutput +metadata: + name: quickwit + labels: + output.fluentd.fluent.io/enabled: "true" +spec: + outputs: + - http: + endpoint: http://quickwit-indexer.system-observability.svc.cluster.local:7280/api/v1/otel-logs-v0_7/ingest + contentType: application/json diff --git a/kustomize/observability/fluentd/stdout/kustomization.yaml b/kustomize/observability/fluentd/outputs/quickwit/kustomization.yaml similarity index 100% rename from kustomize/observability/fluentd/stdout/kustomization.yaml rename to kustomize/observability/fluentd/outputs/quickwit/kustomization.yaml diff --git a/kustomize/observability/fluentd/stdout/clusteroutput.yaml b/kustomize/observability/fluentd/outputs/stdout/clusteroutput.yaml similarity index 100% rename from kustomize/observability/fluentd/stdout/clusteroutput.yaml rename to kustomize/observability/fluentd/outputs/stdout/clusteroutput.yaml diff --git a/kustomize/observability/fluentd/outputs/stdout/kustomization.yaml b/kustomize/observability/fluentd/outputs/stdout/kustomization.yaml new file mode 100644 index 00000000..051eb969 --- /dev/null +++ b/kustomize/observability/fluentd/outputs/stdout/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - clusteroutput.yaml diff --git a/kustomize/observability/grafana/quickwit/kustomization.yaml b/kustomize/observability/grafana/quickwit/kustomization.yaml new file mode 100644 index 00000000..23eec4af --- /dev/null +++ b/kustomize/observability/grafana/quickwit/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - path: patches/helm-release.yaml + - target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: grafana + namespace: system-observability + path: patches/patch.json diff --git a/kustomize/observability/grafana/quickwit/patches/helm-release.yaml b/kustomize/observability/grafana/quickwit/patches/helm-release.yaml new file mode 100644 index 00000000..b269cc98 --- /dev/null +++ b/kustomize/observability/grafana/quickwit/patches/helm-release.yaml @@ -0,0 +1,12 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: grafana + namespace: system-observability +spec: + dependsOn: + - name: quickwit + namespace: system-observability + values: + plugins: + - quickwit-quickwit-datasource diff --git a/kustomize/observability/grafana/quickwit/patches/patch.json b/kustomize/observability/grafana/quickwit/patches/patch.json new file mode 100644 index 00000000..6ff9ac5e --- /dev/null +++ b/kustomize/observability/grafana/quickwit/patches/patch.json @@ -0,0 +1,15 @@ +[ + { + "op": "add", + "path": "/spec/values/datasources/datasources.yaml/datasources/-", + "value": { + "name": "Quickwit Logs", + "uid": "quickwit-logs", + "type": "quickwit-quickwit-datasource", + "url": "http://quickwit-searcher.system-observability.svc.cluster.local:7280/api/v1", + "jsonData": { + "index": "otel-logs-v0_7" + } + } + } +] diff --git a/kustomize/observability/quickwit/helm-release.yaml b/kustomize/observability/quickwit/helm-release.yaml new file mode 100644 index 00000000..c175a095 --- /dev/null +++ b/kustomize/observability/quickwit/helm-release.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: quickwit + namespace: system-observability +spec: + interval: 5m + timeout: 10m + chart: + spec: + chart: quickwit + # renovate: datasource=helm depName=quickwit package=quickwit helmRepo=https://helm.quickwit.io + version: 0.7.17 + sourceRef: + kind: HelmRepository + name: quickwit + namespace: system-gitops + values: + searcher: + replicaCount: 1 diff --git a/kustomize/observability/quickwit/helm-repository.yaml b/kustomize/observability/quickwit/helm-repository.yaml new file mode 100644 index 00000000..dd81eac1 --- /dev/null +++ b/kustomize/observability/quickwit/helm-repository.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: HelmRepository +metadata: + name: quickwit + namespace: system-gitops +spec: + interval: 10m + timeout: 3m + url: https://helm.quickwit.io diff --git a/kustomize/observability/quickwit/kustomization.yaml b/kustomize/observability/quickwit/kustomization.yaml new file mode 100644 index 00000000..e768af5d --- /dev/null +++ b/kustomize/observability/quickwit/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - helm-repository.yaml + - helm-release.yaml diff --git a/kustomize/observability/quickwit/pvc/kustomization.yaml b/kustomize/observability/quickwit/pvc/kustomization.yaml new file mode 100644 index 00000000..907131bb --- /dev/null +++ b/kustomize/observability/quickwit/pvc/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - volume.yaml +patches: + - path: patches/helm-release.yaml diff --git a/kustomize/observability/quickwit/pvc/patches/helm-release.yaml b/kustomize/observability/quickwit/pvc/patches/helm-release.yaml new file mode 100644 index 00000000..21faa5fe --- /dev/null +++ b/kustomize/observability/quickwit/pvc/patches/helm-release.yaml @@ -0,0 +1,44 @@ +--- +# SECURITY NOTE: This approach uses a hostPath volume to store the indexes on the host machine. +# It also requires elevated privileges. Do not use in production. For a more secure setup, +# integrate with backing services such as Postgres, Minio, or S3. +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: quickwit + namespace: system-observability +spec: + values: + environment: + QW_METASTORE_URI: file:///quickwit/qwdata/indexes + config: + default_index_root_uri: file:///quickwit/qwdata/indexes + podSecurityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: true + securityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: true + searcher: + replicaCount: 1 + extraVolumes: + - name: quickwit-indexes + persistentVolumeClaim: + claimName: quickwit-indexes + extraVolumeMounts: + - name: quickwit-indexes + mountPath: /quickwit/qwdata/indexes + readOnly: true + indexer: + extraVolumes: + - name: quickwit-indexes + persistentVolumeClaim: + claimName: quickwit-indexes + extraVolumeMounts: + - name: quickwit-indexes + mountPath: /quickwit/qwdata/indexes + readOnly: false diff --git a/kustomize/observability/quickwit/pvc/volume.yaml b/kustomize/observability/quickwit/pvc/volume.yaml new file mode 100644 index 00000000..ad5d1204 --- /dev/null +++ b/kustomize/observability/quickwit/pvc/volume.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: quickwit-indexes + namespace: system-observability +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi