diff --git a/.changelog/90.txt b/.changelog/90.txt new file mode 100644 index 00000000..9c5a7880 --- /dev/null +++ b/.changelog/90.txt @@ -0,0 +1,4 @@ +```release-note:feature +Add envoy_hcp_metrics_bind_socket_dir flag to configure a directory where a unix socket is created. +This enables Envoy metrics collection, which will be forwarded to a HCP metrics collector. +``` \ No newline at end of file diff --git a/internal/bootstrap/bootstrap_config.go b/internal/bootstrap/bootstrap_config.go index 746a2a62..f9c8ccda 100644 --- a/internal/bootstrap/bootstrap_config.go +++ b/internal/bootstrap/bootstrap_config.go @@ -8,6 +8,7 @@ import ( "net" "net/url" "os" + "path" "strings" "text/template" ) @@ -48,6 +49,11 @@ type BootstrapConfig struct { // stats_config.stats_tags can be made by overriding envoy_stats_config_json. StatsTags []string `mapstructure:"envoy_stats_tags"` + // HCPMetricsBindSocketDir is a string that configures the directory for a + // unix socket where Envoy will forward metrics. These metrics get pushed to + // the HCP Metrics collector to show service mesh metrics on HCP. + HCPMetricsBindSocketDir string `mapstructure:"envoy_hcp_metrics_bind_socket_dir"` + // PrometheusBindAddr configures an : on which the Envoy will listen // and expose a single /metrics HTTP endpoint for Prometheus to scrape. It // does this by proxying that URL to the internal admin server's prometheus @@ -237,6 +243,11 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs, omitDeprecatedTa args.StatsFlushInterval = c.StatsFlushInterval } + // Setup HCP Metrics if needed. This MUST happen after the Static*JSON is set above + if c.HCPMetricsBindSocketDir != "" { + appendHCPMetricsConfig(args, c.HCPMetricsBindSocketDir) + } + return nil } @@ -270,7 +281,7 @@ func (c *BootstrapConfig) generateStatsSinks(args *BootstrapTplArgs) error { } if len(stats_sinks) > 0 { - args.StatsSinksJSON = "[\n" + strings.Join(stats_sinks, ",\n") + "\n]" + args.StatsSinksJSON = strings.Join(stats_sinks, ",\n") } return nil } @@ -795,6 +806,56 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd return nil } +// appendHCPMetricsConfig generates config to enable a socket at path: /_.sock +// or /.sock, if namespace is empty. +func appendHCPMetricsConfig(args *BootstrapTplArgs, hcpMetricsBindSocketDir string) { + sock := fmt.Sprintf("%s_%s.sock",args.Namespace,args.ProxyID) + path := path.Join(hcpMetricsBindSocketDir, sock) + + if args.StatsSinksJSON != "" { + args.StatsSinksJSON += ",\n" + } + args.StatsSinksJSON += `{ + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + }` + + if args.StaticClustersJSON != "" { + args.StaticClustersJSON += ",\n" + } + args.StaticClustersJSON += fmt.Sprintf(`{ + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "%s" + } + } + } + } + ] + } + ] + } + }`, path) +} + func containsSelfAdminCluster(clustersJSON string) (bool, error) { clusterNames := []struct { Name string diff --git a/internal/bootstrap/bootstrap_config_test.go b/internal/bootstrap/bootstrap_config_test.go index 19cb0fac..c486fad4 100644 --- a/internal/bootstrap/bootstrap_config_test.go +++ b/internal/bootstrap/bootstrap_config_test.go @@ -91,6 +91,19 @@ const ( ] } }` + +expectedStatsdSink = `{ + "name": "envoy.stat_sinks.statsd", + "typedConfig": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9125 + } + } + } +}` expectedPromListener = `{ "name": "envoy_prometheus_metrics_listener", "address": { @@ -514,6 +527,43 @@ const ( } ] }` + +expectedHCPMetricsCluster = `{ + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + } + } + } + } + ] + } + ] + } + }` + + expectedHCPMetricsStatsSink = `{ + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + }` ) func TestBootstrapConfig_ConfigureArgs(t *testing.T) { @@ -558,13 +608,64 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.custom_exciting_sink", "config": { "foo": "bar" } - }]`, + }`, + }, + }, + { + name: "hcp-metrics-sink", + baseArgs: BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + Namespace: "default", }, + input: BootstrapConfig{ + HCPMetricsBindSocketDir: "/tmp/consul/hcp-metrics", + }, + wantArgs: BootstrapTplArgs{ + ProxyID: "web-sidecar-proxy", + Namespace: "default", + StatsConfigJSON: defaultStatsConfigJSON, + StatsSinksJSON: `{ + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + }`, + StaticClustersJSON: `{ + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/default_web-sidecar-proxy.sock" + } + } + } + } + ] + } + ] + } + }`, + }, + wantErr: false, }, { name: "simple-statsd-sink", @@ -573,7 +674,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", @@ -584,7 +685,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -601,7 +702,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", @@ -618,7 +719,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { "config": { "foo": "bar" } - }]`, + }`, }, wantErr: false, }, @@ -630,18 +731,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { env: []string{"MY_STATSD_URL=udp://127.0.0.1:9125"}, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ - "name": "envoy.stat_sinks.statsd", - "typedConfig": { - "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", - "address": { - "socket_address": { - "address": "127.0.0.1", - "port_value": 9125 - } - } - } - }]`, + StatsSinksJSON: expectedStatsdSink, }, wantErr: false, }, @@ -653,7 +743,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { env: []string{"HOST_IP=127.0.0.1"}, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", @@ -664,7 +754,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -686,7 +776,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.dog_statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", @@ -697,7 +787,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -708,7 +798,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { }, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.dog_statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", @@ -718,7 +808,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -731,7 +821,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { env: []string{"MY_STATSD_URL=udp://127.0.0.1:9125"}, wantArgs: BootstrapTplArgs{ StatsConfigJSON: defaultStatsConfigJSON, - StatsSinksJSON: `[{ + StatsSinksJSON: `{ "name": "envoy.stat_sinks.dog_statsd", "typedConfig": { "@type": "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", @@ -742,7 +832,7 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) { } } } - }]`, + }`, }, wantErr: false, }, @@ -1540,3 +1630,72 @@ func TestConsulTagSpecifiers(t *testing.T) { }) } } + + +func TestAppendHCPMetrics(t *testing.T){ + tests := map[string]struct{ + inputArgs *BootstrapTplArgs + bindSocketDir string + wantArgs *BootstrapTplArgs + }{ + "dir-without-trailing-slash":{ + inputArgs: &BootstrapTplArgs{ + Namespace: "default", + ProxyID: "web-sidecar-proxy", + }, + bindSocketDir: "/tmp/consul/hcp-metrics", + wantArgs: &BootstrapTplArgs{ + Namespace: "default", + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedHCPMetricsStatsSink, + StaticClustersJSON: expectedHCPMetricsCluster, + }, + }, + "dir-with-trailing-slash":{ + inputArgs: &BootstrapTplArgs{ + Namespace: "default", + ProxyID: "web-sidecar-proxy", + }, + bindSocketDir: "/tmp/consul/hcp-metrics", + wantArgs: &BootstrapTplArgs{ + Namespace: "default", + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedHCPMetricsStatsSink, + StaticClustersJSON: expectedHCPMetricsCluster, + }, + }, + "append-clusters-and-stats-sink": { + inputArgs: &BootstrapTplArgs{ + Namespace: "default", + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedStatsdSink, + StaticClustersJSON: expectedSelfAdminCluster, + }, + bindSocketDir: "/tmp/consul/hcp-metrics", + wantArgs: &BootstrapTplArgs{ + Namespace: "default", + ProxyID: "web-sidecar-proxy", + StatsSinksJSON: expectedStatsdSink + ",\n" + expectedHCPMetricsStatsSink , + StaticClustersJSON: expectedSelfAdminCluster + ",\n" + expectedHCPMetricsCluster, + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + appendHCPMetricsConfig(tt.inputArgs,tt.bindSocketDir) + + // Some of our JSON strings are comma separated objects to be + // insertedinto an array which is not valid JSON on it's own so wrap + // them all in an array. For simple values this is still valid JSON + // too. + wantStatsSink := "[" + tt.wantArgs.StatsSinksJSON+ "]" + gotStatsSink := "[" + tt.inputArgs.StatsSinksJSON + "]" + require.JSONEq(t, wantStatsSink, gotStatsSink,"field StatsSinksJSON should be equivalent JSON") + + wantClusters := "[" + tt.wantArgs.StaticClustersJSON+ "]" + gotClusters := "[" + tt.inputArgs.StaticClustersJSON + "]" + require.JSONEq(t, wantClusters, gotClusters, "field StaticClustersJSON should be equivalent JSON") + }) + } +} \ No newline at end of file diff --git a/internal/bootstrap/bootstrap_tpl.go b/internal/bootstrap/bootstrap_tpl.go index 02ced12d..553a5b24 100644 --- a/internal/bootstrap/bootstrap_tpl.go +++ b/internal/bootstrap/bootstrap_tpl.go @@ -263,7 +263,9 @@ const bootstrapTemplate = `{ {{- end }} }, {{- if .StatsSinksJSON }} - "stats_sinks": {{ .StatsSinksJSON }}, + "stats_sinks": [ + {{ .StatsSinksJSON }} + ], {{- end }} {{- if .StatsConfigJSON }} "stats_config": {{ .StatsConfigJSON }}, diff --git a/pkg/consuldp/bootstrap_test.go b/pkg/consuldp/bootstrap_test.go index 9d1bc413..b74bcba7 100644 --- a/pkg/consuldp/bootstrap_test.go +++ b/pkg/consuldp/bootstrap_test.go @@ -109,6 +109,31 @@ func TestBootstrapConfig(t *testing.T) { }), }, }, + "hcp-metrics": { + cfg: &Config{ + Service: &ServiceConfig{ + ServiceID: "web-proxy", + NodeName: nodeName, + Namespace: "default", + }, + Envoy: &EnvoyConfig{ + AdminBindAddress: "127.0.0.1", + AdminBindPort: 19000, + }, + Telemetry: &TelemetryConfig{ + UseCentralConfig: true, + }, + XDSServer: &XDSServer{BindAddress: "127.0.0.1", BindPort: xdsBindPort}, + }, + rsp: &pbdataplane.GetEnvoyBootstrapParamsResponse{ + Service: "web", + Namespace: "default", + NodeName: nodeName, + Config: makeStruct(map[string]any{ + "envoy_hcp_metrics_bind_socket_dir": "/tmp/consul/hcp-metrics", + }), + }, + }, "custom-prometheus-scrape-path": { &Config{ Service: &ServiceConfig{ @@ -192,6 +217,7 @@ func TestBootstrapConfig(t *testing.T) { GetEnvoyBootstrapParams(mock.Anything, &pbdataplane.GetEnvoyBootstrapParamsRequest{ NodeSpec: &pbdataplane.GetEnvoyBootstrapParamsRequest_NodeName{NodeName: tc.cfg.Service.NodeName}, ServiceId: tc.cfg.Service.ServiceID, + Namespace: tc.cfg.Service.Namespace, }).Call. Return(tc.rsp, nil) diff --git a/pkg/consuldp/testdata/TestBootstrapConfig/hcp-metrics.golden b/pkg/consuldp/testdata/TestBootstrapConfig/hcp-metrics.golden new file mode 100644 index 00000000..360512fa --- /dev/null +++ b/pkg/consuldp/testdata/TestBootstrapConfig/hcp-metrics.golden @@ -0,0 +1,207 @@ +{ + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + }, + "node": { + "cluster": "web", + "id": "web-proxy", + "metadata": { + "node_name": "agentless-node", + "namespace": "default", + "partition": "default" + } + }, + "layered_runtime": { + "layers": [ + { + "name": "base", + "static_layer": { + "re2.max_program_size.error_level": 1048576 + } + } + ] + }, + "static_resources": { + "clusters": [ + { + "name": "consul-dataplane", + "ignore_health_on_host_removal": false, + "connect_timeout": "1s", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "consul-dataplane", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 1234 + } + } + } + } + ] + } + ] + } + }, + { + "name": "hcp_metrics_collector", + "type": "STATIC", + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "hcp_metrics_collector", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/tmp/consul/hcp-metrics/default_web-proxy.sock" + } + } + } + } + ] + } + ] + } + } + ] + }, + "stats_sinks": [ + { + "name": "envoy.stat_sinks.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": { + "cluster_name": "hcp_metrics_collector" + } + } + } + } + ], + "stats_config": { + "stats_tags": [ + { + "regex": "^cluster\\.(?:passthrough~)?((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.custom_hash" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service_subset" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.namespace" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:([^.]+)\\.)?[^.]+\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.partition" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.datacenter" + }, + { + "regex": "^cluster\\.([^.]+\\.(?:[^.]+\\.)?([^.]+)\\.external\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.peer" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.routing_type" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.destination.trust_domain" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.target" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.destination.full_target" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.(([^.]+)(?:\\.[^.]+)?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.service" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.datacenter" + }, + { + "regex": "^(?:tcp|http)\\.upstream_peered\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.peer" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.([^.]+(?:\\.([^.]+))?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.namespace" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.([^.]+))?\\.[^.]+\\.)", + "tag_name": "consul.upstream.partition" + }, + { + "tag_name": "local_cluster", + "fixed_value": "web" + }, + { + "tag_name": "consul.source.service", + "fixed_value": "web" + }, + { + "tag_name": "consul.source.namespace", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.partition", + "fixed_value": "default" + } + ], + "use_all_default_tags": true + }, + "dynamic_resources": { + "lds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "cds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "ads_config": { + "api_type": "DELTA_GRPC", + "transport_api_version": "V3", + "grpc_services": { + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "" + } + ], + "envoy_grpc": { + "cluster_name": "consul-dataplane" + } + } + } + } +}