diff --git a/CHANGELOG.md b/CHANGELOG.md index b43c5e97174..f027ce897ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ * [FEATURE] Update in dependency `weaveworks/common`. TLS config options added to the Server. #2535 * [FEATURE] Experimental: Added support for `/api/v1/metadata` Prometheus-based endpoint. #2549 * [FEATURE] Add ability to limit concurrent queries to Cassandra with `-cassandra.query-concurrency` flag. #2562 +* [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 419651a36c1..24c976ea5b5 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -646,6 +646,19 @@ The `querier_config` configures the Cortex querier. # instances form a ring and addresses are picked from the ring). # CLI flag: -experimental.querier.store-gateway-addresses [store_gateway_addresses: | default = ""] + +store_gateway_client: + # TLS cert path for the client + # CLI flag: -experimental.querier.store-gateway-client.tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the client + # CLI flag: -experimental.querier.store-gateway-client.tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the client + # CLI flag: -experimental.querier.store-gateway-client.tls-ca-path + [tls_ca_path: | default = ""] ``` ### `query_frontend_config` @@ -757,6 +770,19 @@ The `ruler_config` configures the Cortex ruler. # CLI flag: -ruler.external.url [external_url: | default = ] +ruler_client: + # TLS cert path for the client + # CLI flag: -ruler.client.tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the client + # CLI flag: -ruler.client.tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the client + # CLI flag: -ruler.client.tls-ca-path + [tls_ca_path: | default = ""] + # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval [evaluation_interval: | default = 1m] @@ -1964,6 +1990,18 @@ grpc_client_config: # Number of times to backoff and retry before failing. # CLI flag: -ingester.client.backoff-retries [max_retries: | default = 10] + + # TLS cert path for the client + # CLI flag: -ingester.client.tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the client + # CLI flag: -ingester.client.tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the client + # CLI flag: -ingester.client.tls-ca-path + [tls_ca_path: | default = ""] ``` ### `frontend_worker_config` @@ -2025,6 +2063,18 @@ grpc_client_config: # Number of times to backoff and retry before failing. # CLI flag: -querier.frontend-client.backoff-retries [max_retries: | default = 10] + + # TLS cert path for the client + # CLI flag: -querier.frontend-client.tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the client + # CLI flag: -querier.frontend-client.tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the client + # CLI flag: -querier.frontend-client.tls-ca-path + [tls_ca_path: | default = ""] ``` ### `etcd_config` @@ -2530,6 +2580,18 @@ The `configstore_config` configures the config database storing rules and alerts # Timeout for requests to Weave Cloud configs service. # CLI flag: -.configs.client-timeout [client_timeout: | default = 5s] + +# TLS cert path for the client +# CLI flag: -.configs.tls-cert-path +[tls_cert_path: | default = ""] + +# TLS key path for the client +# CLI flag: -.configs.tls-key-path +[tls_key_path: | default = ""] + +# TLS CA path for the client +# CLI flag: -.configs.tls-ca-path +[tls_ca_path: | default = ""] ``` ### `tsdb_config` diff --git a/docs/configuration/single-process-config-blocks-tls.yaml b/docs/configuration/single-process-config-blocks-tls.yaml new file mode 100644 index 00000000000..3a5b9214c31 --- /dev/null +++ b/docs/configuration/single-process-config-blocks-tls.yaml @@ -0,0 +1,100 @@ + +# Configuration for running Cortex in single-process mode. +# This should not be used in production. It is only for getting started +# and development. + +# Disable the requirement that every request to Cortex has a +# X-Scope-OrgID header. `fake` will be substituted in instead. +auth_enabled: false + +server: + http_listen_port: 9009 + + # Configure the server to allow messages up to 100MB. + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + grpc_tls_config: + cert_file: "server.crt" + key_file: "server.key" + client_auth_type: "RequireAndVerifyClientCert" + client_ca_file: "root.crt" + + +distributor: + shard_by_all_labels: true + pool: + health_check_ingesters: true + +ingester_client: + grpc_client_config: + # Configure the client to allow messages up to 100MB. + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + use_gzip_compression: true + tls_cert_path: "client.crt" + tls_key_path: "client.key" + tls_ca_path: "root.crt" + +ingester: + # Disable blocks transfers on ingesters shutdown or rollout. + max_transfer_retries: 0 + + lifecycler: + # The address to advertise for this ingester. Will be autodiscovered by + # looking up address on eth0 or en0; can be specified if this fails. + # address: 127.0.0.1 + + # We want to start immediately and flush on shutdown. + join_after: 0 + min_ready_duration: 0s + final_sleep: 0s + num_tokens: 512 + + # Use an in memory ring store, so we don't need to launch a Consul. + ring: + kvstore: + store: inmemory + replication_factor: 1 + +storage: + engine: tsdb + +tsdb: + dir: /tmp/cortex/tsdb + bucket_store: + sync_dir: /tmp/cortex/tsdb-sync + + # You can choose between local storage and Amazon S3, Google GCS and Azure storage. Each option requires additional configuration + # as shown below. All options can be configured via flags as well which might be handy for secret inputs. + backend: s3 # s3, gcs, azure or filesystem are valid options + s3: + bucket_name: cortex + endpoint: s3.dualstack.us-east-1.amazonaws.com + # Configure your S3 credentials below. + # secret_access_key: "TODO" + # access_key_id: "TODO" +# gcs: +# bucket_name: cortex +# service_account: # if empty or omitted Cortex will use your default service account as per Google's fallback logic +# azure: +# account_name: +# account_key: +# container_name: +# endpoint_suffix: +# max_retries: # Number of retries for recoverable errors (defaults to 20) +# filesystem: +# dir: ./data/tsdb + +compactor: + data_dir: /tmp/cortex/compactor + sharding_ring: + kvstore: + store: inmemory + +frontend_worker: + match_max_concurrent: true + grpc_client_config: + tls_cert_path: "client.crt" + tls_key_path: "client.key" + tls_ca_path: "root.crt" diff --git a/docs/production/tls.md b/docs/production/tls.md new file mode 100644 index 00000000000..6c1b859713e --- /dev/null +++ b/docs/production/tls.md @@ -0,0 +1,111 @@ +--- +title: "Securing communication between Cortex components with TLS" +linkTitle: "Securing communication between Cortex components with TLS" +weight: 5 +slug: tls +--- + +Cortex is a distributed system with significant traffic between its services. +To allow for secure communication, Cortex supports TLS between all its +components. This guide describes the process of setting up TLS. + +### Generation of certs to configure TLS + +The first step to securing inter-service communication in Cortex with TLS is +generating certificates. A Certifying Authority (CA) will be used for this +purpose which should be private to the organization, as any certificates signed +by this CA will have permissions to communicate with the cluster. + +We will use the following script to generate self signed certs for the cluster: + +``` +# Refer: github.com/cortexproject/cortex/integration/certs/genCerts.sh + +# keys +openssl genrsa -out root.key +openssl genrsa -out client.key +openssl genrsa -out server.key + +# root cert / certifying authority +openssl req -x509 -new -nodes -key root.key -subj "/C=US/ST=KY/O=Org/CN=root" -sha256 -days 100000 -out root.crt + +# csrs - certificate signing requests +openssl req -new -sha256 -key client.key -subj "/C=US/ST=KY/O=Org/CN=client" -out client.csr +openssl req -new -sha256 -key server.key -subj "/C=US/ST=KY/O=Org/CN=localhost" -out server.csr + +# certificates +openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days 100000 -sha256 +openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days 100000 -sha256 +``` + +Note that the above script generates certificates that are valid for 100000 days. +This can be changed by adjusting the `-days` option in the above commands. +It is recommended that the certs be replaced atleast once every 2 years. + +The above script generates keys `client.key, server.key` and certs +`client.crt, server.crt` for both the client and server. The CA cert is +generated as `root.crt`. + +### Load certs into the HTTP/GRPC server/client + +Every HTTP/GRPC link between Cortex components supports TLS configuration +through the following config parameters: + +#### Server flags + +``` + # Path to the TLS Cert for the HTTP Server + -server.http-tls-cert-path=/path/to/server.crt + + # Path to the TLS Key for the HTTP Server + -server.http-tls-key-path=/path/to/server.key + + # Type of Client Auth for the HTTP Server + -server.http-tls-client-auth="RequireAndVerifyClientCert" + + # Path to the Client CA Cert for the HTTP Server + -server.http-tls-ca-path="/path/to/root.crt" + + # Path to the TLS Cert for the GRPC Server + -server.grpc-tls-cert-path=/path/to/server.crt + + # Path to the TLS Key for the GRPC Server + -server.grpc-tls-key-path=/path/to/server.key + + # Type of Client Auth for the GRPC Server + -server.grpc-tls-client-auth="RequireAndVerifyClientCert" + + # Path to the Client CA Cert for the GRPC Server + -server.grpc-tls-ca-path=/path/to/root.crt +``` + +#### Client flags + +Client flags are component specific. + +For an HTTP client in the Alertmanager: +``` + # Path to the TLS Cert for the HTTP Client + -alertmanager.configs.tls-cert-path=/path/to/client.crt + + # Path to the TLS Key for the HTTP Client + -alertmanager.configs.tls-key-path=/path/to/client.key + + # Path to the TLS CA for the HTTP Client + -alertmanager.configs.tls-ca-path=/path/to/root.crt +``` + +For a GRPC client in the Querier: +``` + # Path to the TLS Cert for the GRPC Client + -querier.frontend-client.tls-cert-path=/path/to/client.crt + + # Path to the TLS Key for the GRPC Client + -querier.frontend-client.tls-key-path=/path/to/client.key + + # Path to the TLS CA for the GRPC Client + -querier.frontend-client.tls-ca-path=/path/to/root.crt +``` + +TLS can be configured in a similar fashion for other GRPC clients like the +ingester client. \ No newline at end of file diff --git a/integration/certs/genCerts.sh b/integration/certs/genCerts.sh new file mode 100644 index 00000000000..455f6398c27 --- /dev/null +++ b/integration/certs/genCerts.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copied from https://github.com/joe-elliott/cert-exporter/blob/5ce49ebf6bfcdcb178d31145ae2a460f3b348cf5/test/files/genCerts.sh +# Copyright [2020] [cert-exporter authors] +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +certFolder=$1 +days=$2 + +pushd $certFolder + +# keys +openssl genrsa -out root.key +openssl genrsa -out client.key +openssl genrsa -out server.key + +# root cert +openssl req -x509 -new -nodes -key root.key -subj "/C=US/ST=KY/O=Org/CN=root" -sha256 -days $days -out root.crt + +# csrs +openssl req -new -sha256 -key client.key -subj "/C=US/ST=KY/O=Org/CN=client" -out client.csr +openssl req -new -sha256 -key server.key -subj "/C=US/ST=KY/O=Org/CN=localhost" -out server.csr + +openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days $days -sha256 +openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days $days -sha256 + +popd diff --git a/integration/configs.go b/integration/configs.go index aab608fb7ec..162512aa82c 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -22,16 +22,21 @@ const ( cortexConfigFile = "config.yaml" cortexSchemaConfigFile = "schema.yaml" blocksStorageEngine = "tsdb" + clientCertFile = "certs/client.crt" + clientKeyFile = "certs/client.key" + caCertFile = "certs/root.crt" + serverCertFile = "certs/server.crt" + serverKeyFile = "certs/server.key" storeConfigTemplate = ` - from: {{.From}} store: {{.IndexStore}} schema: v9 index: prefix: cortex_ - period: 168h + period: 168h chunks: prefix: cortex_chunks_ - period: 168h + period: 168h ` cortexAlertmanagerUserConfigYaml = `route: @@ -120,7 +125,7 @@ storage: table_manager: poll_interval: 1m - retention_period: 168h + retention_period: 168h schema: {{.SchemaConfig}} diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index cac4e67adfe..cccf8470f76 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -4,6 +4,7 @@ package main import ( "fmt" + "os/exec" "sync" "testing" "time" @@ -19,6 +20,10 @@ import ( "github.com/cortexproject/cortex/integration/e2ecortex" ) +const ( + integrationHomeFolder = "integration/" +) + type queryFrontendSetup func(t *testing.T, s *e2e.Scenario) (configFile string, flags map[string]string) func TestQueryFrontendWithBlocksStorageViaFlags(t *testing.T) { @@ -78,6 +83,30 @@ func TestQueryFrontendWithChunksStorageViaConfigFile(t *testing.T) { }) } +func TestQueryFrontendTLSWithBlocksStorageViaFlags(t *testing.T) { + runQueryFrontendTest(t, func(t *testing.T, s *e2e.Scenario) (configFile string, flags map[string]string) { + minio := e2edb.NewMinio(9000, BlocksStorageFlags["-experimental.tsdb.s3.bucket-name"]) + require.NoError(t, s.StartAndWaitReady(minio)) + + // setup tls + cmd := exec.Command("bash", "certs/genCerts.sh", "certs", "1") + require.NoError(t, cmd.Run()) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientCertFile, clientCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientKeyFile, clientKeyFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+caCertFile, caCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverCertFile, serverCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverKeyFile, serverKeyFile)) + + return "", mergeFlags( + BlocksStorageFlags, + getServerTLSFlags(), + getClientTLSFlagsWithPrefix("ingester.client"), + getClientTLSFlagsWithPrefix("querier.frontend-client"), + getClientTLSFlagsWithPrefix("ingester.client"), + ) + }) +} + func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { const numUsers = 10 const numQueriesPerUser = 10 diff --git a/integration/util.go b/integration/util.go index 0b5fc39b7b2..893c516ec96 100644 --- a/integration/util.go +++ b/integration/util.go @@ -14,7 +14,7 @@ import ( ) var ( - // Expose some utilities form the framework so that we don't have to prefix them + // Expose some utilities from the framework so that we don't have to prefix them // with the package name in tests. mergeFlags = e2e.MergeFlags newDynamoClient = e2edb.NewDynamoClient @@ -51,3 +51,20 @@ func copyFileToSharedDir(s *e2e.Scenario, src, dst string) error { return writeFileToSharedDir(s, dst, content) } + +func getServerTLSFlags() map[string]string { + return map[string]string{ + "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), + "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), + "-server.grpc-tls-client-auth": "RequireAndVerifyClientCert", + "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, caCertFile), + } +} + +func getClientTLSFlagsWithPrefix(prefix string) map[string]string { + return map[string]string{ + "-" + prefix + ".tls-cert-path": filepath.Join(e2e.ContainerSharedDir, clientCertFile), + "-" + prefix + ".tls-key-path": filepath.Join(e2e.ContainerSharedDir, clientKeyFile), + "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, caCertFile), + } +} diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index a020bb54f02..05343c37c82 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -2,6 +2,7 @@ package client import ( "context" + "crypto/tls" "encoding/json" "flag" "fmt" @@ -9,27 +10,29 @@ import ( "net/url" "time" - "github.com/cortexproject/cortex/pkg/configs/userconfig" - "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/instrument" + "github.com/cortexproject/cortex/pkg/configs/userconfig" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" + tls_cfg "github.com/cortexproject/cortex/pkg/util/tls" ) // Config says where we can find the ruler userconfig. type Config struct { - ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` - ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. + ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` + ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. + TLS tls_cfg.ClientConfig `yaml:",inline"` } // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.Var(&cfg.ConfigsAPIURL, prefix+"configs.url", "URL of configs API server.") f.DurationVar(&cfg.ClientTimeout, prefix+"configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") + cfg.TLS.RegisterFlagsWithPrefix(prefix+"configs", f) } var configsRequestDuration = instrument.NewHistogramCollector(promauto.NewHistogramVec(prometheus.HistogramOpts{ @@ -51,16 +54,27 @@ type Client interface { // New creates a new ConfigClient. func New(cfg Config) (*ConfigDBClient, error) { - return &ConfigDBClient{ + client := &ConfigDBClient{ URL: cfg.ConfigsAPIURL.URL, Timeout: cfg.ClientTimeout, - }, nil + } + + tlsConfig, err := cfg.TLS.GetTLSConfig() + if err != nil { + return nil, err + } + + if tlsConfig != nil { + client.TLSConfig = tlsConfig + } + return client, nil } // ConfigDBClient allows retrieving recording and alerting rules from the configs server. type ConfigDBClient struct { - URL *url.URL - Timeout time.Duration + URL *url.URL + Timeout time.Duration + TLSConfig *tls.Config } // GetRules implements Client @@ -73,7 +87,7 @@ func (c ConfigDBClient) GetRules(ctx context.Context, since userconfig.ID) (map[ var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetRules", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error - response, err = doRequest(endpoint, c.Timeout, since) + response, err = doRequest(endpoint, c.Timeout, c.TLSConfig, since) return err }) if err != nil { @@ -99,19 +113,22 @@ func (c ConfigDBClient) GetAlerts(ctx context.Context, since userconfig.ID) (*Co var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetAlerts", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error - response, err = doRequest(endpoint, c.Timeout, since) + response, err = doRequest(endpoint, c.Timeout, c.TLSConfig, since) return err }) return response, err } -func doRequest(endpoint string, timeout time.Duration, since userconfig.ID) (*ConfigsResponse, error) { +func doRequest(endpoint string, timeout time.Duration, tlsConfig *tls.Config, since userconfig.ID) (*ConfigsResponse, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err } client := &http.Client{Timeout: timeout} + if tlsConfig != nil { + client.Transport = &http.Transport{TLSClientConfig: tlsConfig} + } resp, err := client.Do(req) if err != nil { diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go index 86b0427c7ee..7ada6b20d89 100644 --- a/pkg/configs/client/configs_test.go +++ b/pkg/configs/client/configs_test.go @@ -35,7 +35,7 @@ func TestDoRequest(t *testing.T) { })) defer server.Close() - resp, err := doRequest(server.URL, 1*time.Second, 0) + resp, err := doRequest(server.URL, 1*time.Second, nil, 0) assert.Nil(t, err) expected := ConfigsResponse{Configs: map[string]userconfig.View{ diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index c28269c11ba..8c5d3b54d30 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -34,9 +34,11 @@ type closableHealthAndIngesterClient struct { // MakeIngesterClient makes a new IngesterClient func MakeIngesterClient(addr string, cfg Config) (HealthAndIngesterClient, error) { - opts := []grpc.DialOption{grpc.WithInsecure()} - opts = append(opts, cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration))...) - conn, err := grpc.Dial(addr, opts...) + dialOpts, err := cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration)) + if err != nil { + return nil, err + } + conn, err := grpc.Dial(addr, dialOpts...) if err != nil { return nil, err } @@ -53,7 +55,7 @@ func (c *closableHealthAndIngesterClient) Close() error { // Config is the configuration struct for the ingester client type Config struct { - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` + GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config"` } // RegisterFlags registers configuration settings used by the ingester client config. diff --git a/pkg/querier/blocks_store_balanced_set.go b/pkg/querier/blocks_store_balanced_set.go index ff54044a1d5..4be29bb17a7 100644 --- a/pkg/querier/blocks_store_balanced_set.go +++ b/pkg/querier/blocks_store_balanced_set.go @@ -17,6 +17,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/tls" ) // BlocksStoreSet implementation used when the blocks are not sharded in the store-gateway @@ -29,7 +30,7 @@ type blocksStoreBalancedSet struct { dnsProvider *dns.Provider } -func newBlocksStoreBalancedSet(serviceAddresses []string, logger log.Logger, reg prometheus.Registerer) *blocksStoreBalancedSet { +func newBlocksStoreBalancedSet(serviceAddresses []string, tlsCfg tls.ClientConfig, logger log.Logger, reg prometheus.Registerer) *blocksStoreBalancedSet { const dnsResolveInterval = 10 * time.Second dnsProviderReg := extprom.WrapRegistererWithPrefix("cortex_storegateway_client_", reg) @@ -37,7 +38,7 @@ func newBlocksStoreBalancedSet(serviceAddresses []string, logger log.Logger, reg s := &blocksStoreBalancedSet{ serviceAddresses: serviceAddresses, dnsProvider: dns.NewProvider(logger, dnsProviderReg, dns.GolangResolverType), - clientsPool: newStoreGatewayClientPool(nil, logger, reg), + clientsPool: newStoreGatewayClientPool(nil, tlsCfg, logger, reg), } s.Service = services.NewTimerService(dnsResolveInterval, s.starting, s.resolve, nil) diff --git a/pkg/querier/blocks_store_balanced_set_test.go b/pkg/querier/blocks_store_balanced_set_test.go index 3afe3c42135..c27c4779cc8 100644 --- a/pkg/querier/blocks_store_balanced_set_test.go +++ b/pkg/querier/blocks_store_balanced_set_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/tls" ) func TestBlocksStoreBalancedSet_GetClientsFor(t *testing.T) { @@ -20,7 +21,7 @@ func TestBlocksStoreBalancedSet_GetClientsFor(t *testing.T) { ctx := context.Background() reg := prometheus.NewPedanticRegistry() - s := newBlocksStoreBalancedSet(serviceAddrs, log.NewNopLogger(), reg) + s := newBlocksStoreBalancedSet(serviceAddrs, tls.ClientConfig{}, log.NewNopLogger(), reg) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index 5aa1cb9abfd..27e8e380c17 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -126,7 +126,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa reg.MustRegister(storesRing) } - stores, err = newBlocksStoreReplicationSet(storesRing, logger, reg) + stores, err = newBlocksStoreReplicationSet(storesRing, querierCfg.StoreGatewayClient, logger, reg) if err != nil { return nil, errors.Wrap(err, "failed to create store set") } @@ -135,7 +135,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa return nil, errNoStoreGatewayAddress } - stores = newBlocksStoreBalancedSet(querierCfg.GetStoreGatewayAddresses(), logger, reg) + stores = newBlocksStoreBalancedSet(querierCfg.GetStoreGatewayAddresses(), querierCfg.StoreGatewayClient, logger, reg) } return NewBlocksStoreQueryable(stores, scanner, reg) diff --git a/pkg/querier/blocks_store_replicated_set.go b/pkg/querier/blocks_store_replicated_set.go index ecaa0a4350b..6e4e5723e8f 100644 --- a/pkg/querier/blocks_store_replicated_set.go +++ b/pkg/querier/blocks_store_replicated_set.go @@ -13,6 +13,7 @@ import ( cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/tls" ) // BlocksStoreSet implementation used when the blocks are sharded and replicated across @@ -28,10 +29,10 @@ type blocksStoreReplicationSet struct { subservicesWatcher *services.FailureWatcher } -func newBlocksStoreReplicationSet(storesRing *ring.Ring, logger log.Logger, reg prometheus.Registerer) (*blocksStoreReplicationSet, error) { +func newBlocksStoreReplicationSet(storesRing *ring.Ring, tlsCfg tls.ClientConfig, logger log.Logger, reg prometheus.Registerer) (*blocksStoreReplicationSet, error) { s := &blocksStoreReplicationSet{ storesRing: storesRing, - clientsPool: newStoreGatewayClientPool(client.NewRingServiceDiscovery(storesRing), logger, reg), + clientsPool: newStoreGatewayClientPool(client.NewRingServiceDiscovery(storesRing), tlsCfg, logger, reg), } var err error diff --git a/pkg/querier/blocks_store_replicated_set_test.go b/pkg/querier/blocks_store_replicated_set_test.go index a09f1f151e0..353d8f1bfd2 100644 --- a/pkg/querier/blocks_store_replicated_set_test.go +++ b/pkg/querier/blocks_store_replicated_set_test.go @@ -24,6 +24,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/test" + "github.com/cortexproject/cortex/pkg/util/tls" ) func Test_findSmallestInstanceSet(t *testing.T) { @@ -175,7 +176,7 @@ func TestBlocksStoreReplicationSet_GetClientsFor(t *testing.T) { require.NoError(t, err) reg := prometheus.NewPedanticRegistry() - s, err := newBlocksStoreReplicationSet(r, log.NewNopLogger(), reg) + s, err := newBlocksStoreReplicationSet(r, tls.ClientConfig{}, log.NewNopLogger(), reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck diff --git a/pkg/querier/frontend/worker.go b/pkg/querier/frontend/worker.go index 4de4fc8a233..7721b0eb054 100644 --- a/pkg/querier/frontend/worker.go +++ b/pkg/querier/frontend/worker.go @@ -27,7 +27,7 @@ type WorkerConfig struct { MatchMaxConcurrency bool `yaml:"match_max_concurrent"` DNSLookupDuration time.Duration `yaml:"dns_lookup_duration"` - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` + GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config"` } // RegisterFlags adds the flags required to config this to the given FlagSet. @@ -139,10 +139,10 @@ func (w *worker) watchDNSLoop(servCtx context.Context) error { } func (w *worker) connect(ctx context.Context, address string) (FrontendClient, error) { - opts := []grpc.DialOption{ - grpc.WithInsecure(), + opts, err := w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) + if err != nil { + return nil, err } - opts = append(opts, w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil)...) conn, err := grpc.DialContext(ctx, address, opts...) if err != nil { diff --git a/pkg/querier/frontend/worker_frontend_manager.go b/pkg/querier/frontend/worker_frontend_manager.go index 2af3ce883d3..f3ea40de4c7 100644 --- a/pkg/querier/frontend/worker_frontend_manager.go +++ b/pkg/querier/frontend/worker_frontend_manager.go @@ -27,7 +27,7 @@ var ( type frontendManager struct { server *server.Server client FrontendClient - clientCfg grpcclient.Config + clientCfg grpcclient.ConfigWithTLS log log.Logger @@ -37,7 +37,7 @@ type frontendManager struct { currentProcessors *atomic.Int32 } -func newFrontendManager(serverCtx context.Context, log log.Logger, server *server.Server, client FrontendClient, clientCfg grpcclient.Config) *frontendManager { +func newFrontendManager(serverCtx context.Context, log log.Logger, server *server.Server, client FrontendClient, clientCfg grpcclient.ConfigWithTLS) *frontendManager { f := &frontendManager{ log: log, client: client, @@ -133,8 +133,8 @@ func (f *frontendManager) process(ctx context.Context, c Frontend_ProcessClient) } // Ensure responses that are too big are not retried. - if len(response.Body) >= f.clientCfg.MaxSendMsgSize { - errMsg := fmt.Sprintf("response larger than the max (%d vs %d)", len(response.Body), f.clientCfg.MaxSendMsgSize) + if len(response.Body) >= f.clientCfg.GRPC.MaxSendMsgSize { + errMsg := fmt.Sprintf("response larger than the max (%d vs %d)", len(response.Body), f.clientCfg.GRPC.MaxSendMsgSize) response = &httpgrpc.HTTPResponse{ Code: http.StatusRequestEntityTooLarge, Body: []byte(errMsg), diff --git a/pkg/querier/frontend/worker_frontend_manager_test.go b/pkg/querier/frontend/worker_frontend_manager_test.go index 90131363502..d409832b97f 100644 --- a/pkg/querier/frontend/worker_frontend_manager_test.go +++ b/pkg/querier/frontend/worker_frontend_manager_test.go @@ -91,7 +91,7 @@ func TestConcurrency(t *testing.T) { } for _, tt := range tests { t.Run(fmt.Sprintf("Testing concurrency %v", tt.concurrency), func(t *testing.T) { - mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.Config{}) + mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.ConfigWithTLS{}) for _, c := range tt.concurrency { calls.Store(0) @@ -127,7 +127,7 @@ func TestRecvFailDoesntCancelProcess(t *testing.T) { failRecv: true, } - mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.Config{}) + mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.ConfigWithTLS{}) mgr.concurrentRequests(1) time.Sleep(50 * time.Millisecond) @@ -151,7 +151,7 @@ func TestServeCancelStopsProcess(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) - mgr := newFrontendManager(ctx, util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.Config{MaxSendMsgSize: 100000}) + mgr := newFrontendManager(ctx, util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.ConfigWithTLS{GRPC: grpcclient.Config{MaxSendMsgSize: 100000}}) mgr.concurrentRequests(1) time.Sleep(50 * time.Millisecond) diff --git a/pkg/querier/frontend/worker_test.go b/pkg/querier/frontend/worker_test.go index ba3ee655a26..01ac2e3eb15 100644 --- a/pkg/querier/frontend/worker_test.go +++ b/pkg/querier/frontend/worker_test.go @@ -83,7 +83,7 @@ func TestResetConcurrency(t *testing.T) { } for i := 0; i < tt.numManagers; i++ { - w.managers[strconv.Itoa(i)] = newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.Config{}) + w.managers[strconv.Itoa(i)] = newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.ConfigWithTLS{}) } w.resetConcurrency() diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 3e2b3373a79..22ac39f7545 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -23,6 +23,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier/lazyquery" "github.com/cortexproject/cortex/pkg/querier/series" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/tls" ) // Config contains the configuration require to create a querier @@ -51,7 +52,8 @@ type Config struct { ActiveQueryTrackerDir string `yaml:"active_query_tracker_dir"` // Blocks storage only. - StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayClient tls.ClientConfig `yaml:"store_gateway_client"` } var ( @@ -60,6 +62,7 @@ var ( // RegisterFlags adds the flags required to config this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.StoreGatewayClient.RegisterFlagsWithPrefix("experimental.querier.store-gateway-client", f) f.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.") f.DurationVar(&cfg.Timeout, "querier.timeout", 2*time.Minute, "The timeout for a query.") if f.Lookup("promql.lookback-delta") == nil { diff --git a/pkg/querier/store_gateway_client.go b/pkg/querier/store_gateway_client.go index 98d0ef0648b..75f3c3bfe03 100644 --- a/pkg/querier/store_gateway_client.go +++ b/pkg/querier/store_gateway_client.go @@ -13,9 +13,10 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/tls" ) -func newStoreGatewayClientFactory(cfg grpcclient.Config, reg prometheus.Registerer) client.PoolFactory { +func newStoreGatewayClientFactory(clientCfg grpcclient.Config, tlsCfg tls.ClientConfig, reg prometheus.Registerer) client.PoolFactory { requestDuration := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Namespace: "cortex", Name: "storegateway_client_request_duration_seconds", @@ -25,13 +26,16 @@ func newStoreGatewayClientFactory(cfg grpcclient.Config, reg prometheus.Register }, []string{"operation", "status_code"}) return func(addr string) (client.PoolClient, error) { - return dialStoreGatewayClient(cfg, addr, requestDuration) + return dialStoreGatewayClient(clientCfg, tlsCfg, addr, requestDuration) } } -func dialStoreGatewayClient(cfg grpcclient.Config, addr string, requestDuration *prometheus.HistogramVec) (*storeGatewayClient, error) { - opts := []grpc.DialOption{grpc.WithInsecure()} - opts = append(opts, cfg.DialOption(grpcclient.Instrument(requestDuration))...) +func dialStoreGatewayClient(clientCfg grpcclient.Config, tlsCfg tls.ClientConfig, addr string, requestDuration *prometheus.HistogramVec) (*storeGatewayClient, error) { + opts, err := tlsCfg.GetGRPCDialOptions() + if err != nil { + return nil, err + } + opts = append(opts, clientCfg.DialOption(grpcclient.Instrument(requestDuration))...) conn, err := grpc.Dial(addr, opts...) if err != nil { return nil, errors.Wrapf(err, "failed to dial store-gateway %s", addr) @@ -58,7 +62,7 @@ func (c *storeGatewayClient) String() string { return c.conn.Target() } -func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, logger log.Logger, reg prometheus.Registerer) *client.Pool { +func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, tlsCfg tls.ClientConfig, logger log.Logger, reg prometheus.Registerer) *client.Pool { // We prefer sane defaults instead of exposing further config options. clientCfg := grpcclient.Config{ MaxRecvMsgSize: 100 << 20, @@ -68,7 +72,6 @@ func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, logger log RateLimitBurst: 0, BackoffOnRatelimits: false, } - poolCfg := client.PoolConfig{ CheckInterval: time.Minute, HealthCheckEnabled: true, @@ -82,5 +85,5 @@ func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, logger log ConstLabels: map[string]string{"client": "querier"}, }) - return client.NewPool("store-gateway", poolCfg, discovery, newStoreGatewayClientFactory(clientCfg, reg), clientsCount, logger) + return client.NewPool("store-gateway", poolCfg, discovery, newStoreGatewayClientFactory(clientCfg, tlsCfg, reg), clientsCount, logger) } diff --git a/pkg/querier/store_gateway_client_test.go b/pkg/querier/store_gateway_client_test.go index 6bb31a99e5f..58d26f34a4d 100644 --- a/pkg/querier/store_gateway_client_test.go +++ b/pkg/querier/store_gateway_client_test.go @@ -16,6 +16,7 @@ import ( "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/tls" ) func Test_newStoreGatewayClientFactory(t *testing.T) { @@ -36,10 +37,11 @@ func Test_newStoreGatewayClientFactory(t *testing.T) { // Create a client factory and query back the mocked service // with different clients. cfg := grpcclient.Config{} + tlsCfg := tls.ClientConfig{} flagext.DefaultValues(&cfg) reg := prometheus.NewPedanticRegistry() - factory := newStoreGatewayClientFactory(cfg, reg) + factory := newStoreGatewayClientFactory(cfg, tlsCfg, reg) for i := 0; i < 2; i++ { client, err := factory(listener.Addr().String()) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 54a15d3fe96..83fcac21766 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -35,6 +35,7 @@ import ( "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/tls" ) var ( @@ -59,6 +60,8 @@ var ( type Config struct { // This is used for template expansion in alerts; must be a valid URL. ExternalURL flagext.URLValue `yaml:"external_url"` + // TLS parameters for the GRPC Client + ClientTLSConfig tls.ClientConfig `yaml:"ruler_client"` // How frequently to evaluate rules by default. EvaluationInterval time.Duration `yaml:"evaluation_interval"` // Delay the evaluation of all rules by a set interval to give a buffer @@ -103,6 +106,7 @@ func (cfg *Config) Validate() error { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.ClientTLSConfig.RegisterFlagsWithPrefix("ruler.client", f) cfg.StoreConfig.RegisterFlags(f) cfg.Ring.RegisterFlags(f) @@ -622,7 +626,11 @@ func (r *Ruler) getShardedRules(ctx context.Context) ([]*GroupStateDesc, error) rgs := []*GroupStateDesc{} for _, rlr := range rulers.Ingesters { - conn, err := grpc.Dial(rlr.Addr, grpc.WithInsecure()) + dialOpts, err := r.cfg.ClientTLSConfig.GetGRPCDialOptions() + if err != nil { + return nil, err + } + conn, err := grpc.Dial(rlr.Addr, dialOpts...) if err != nil { return nil, err } diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 36a92a0e86f..f2ef20e9bde 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -7,6 +7,7 @@ import ( "google.golang.org/grpc" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/tls" ) // Config for a gRPC client. @@ -65,3 +66,25 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamClientInterceptors...)), } } + +// ConfigWithTLS is the config for a grpc client with tls +type ConfigWithTLS struct { + GRPC Config `yaml:",inline"` + TLS tls.ClientConfig `yaml:",inline"` +} + +// RegisterFlagsWithPrefix registers flags with prefix. +func (cfg *ConfigWithTLS) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + cfg.GRPC.RegisterFlagsWithPrefix(prefix, f) + cfg.TLS.RegisterFlagsWithPrefix(prefix, f) +} + +// DialOption returns the config as a grpc.DialOptions +func (cfg *ConfigWithTLS) DialOption(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) ([]grpc.DialOption, error) { + opts, err := cfg.TLS.GetGRPCDialOptions() + if err != nil { + return nil, err + } + + return append(opts, cfg.GRPC.DialOption(unaryClientInterceptors, streamClientInterceptors)...), nil +} diff --git a/pkg/util/tls/tls.go b/pkg/util/tls/tls.go new file mode 100644 index 00000000000..8b4c7d2129c --- /dev/null +++ b/pkg/util/tls/tls.go @@ -0,0 +1,62 @@ +package tls + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "io/ioutil" + + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// ClientConfig is the config for client TLS. +type ClientConfig struct { + CertPath string `yaml:"tls_cert_path"` + KeyPath string `yaml:"tls_key_path"` + CAPath string `yaml:"tls_ca_path"` +} + +// RegisterFlagsWithPrefix registers flags with prefix. +func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.CertPath, prefix+".tls-cert-path", "", "TLS cert path for the client") + f.StringVar(&cfg.KeyPath, prefix+".tls-key-path", "", "TLS key path for the client") + f.StringVar(&cfg.CAPath, prefix+".tls-ca-path", "", "TLS CA path for the client") +} + +// GetTLSConfig initialises tls.Config from config options +func (cfg *ClientConfig) GetTLSConfig() (*tls.Config, error) { + if cfg.CertPath != "" && cfg.KeyPath != "" && cfg.CAPath != "" { + clientCert, err := tls.LoadX509KeyPair(cfg.CertPath, cfg.KeyPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to load TLS certificate %s,%s", cfg.CertPath, cfg.KeyPath) + } + + var caCertPool *x509.CertPool + caCert, err := ioutil.ReadFile(cfg.CAPath) + if err != nil { + return nil, errors.Wrapf(err, "error loading ca cert: %s", cfg.CAPath) + } + caCertPool = x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + if len(clientCert.Certificate) > 0 && caCertPool != nil { + return &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{clientCert}, + RootCAs: caCertPool, + }, nil + } + } + return nil, nil +} + +// GetGRPCDialOptions creates GRPC DialOptions for TLS +func (cfg *ClientConfig) GetGRPCDialOptions() ([]grpc.DialOption, error) { + if tlsConfig, err := cfg.GetTLSConfig(); err != nil { + return nil, errors.Wrap(err, "error creating grpc dial options") + } else if tlsConfig != nil { + return []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))}, nil + } + return []grpc.DialOption{grpc.WithInsecure()}, nil +}