From bc614b314f0b718e99b74e15cf8b26bfd19ed085 Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Tue, 30 Aug 2016 15:08:48 +0200 Subject: [PATCH 1/2] load collector configuration --- cmd/agent/app/common_osx.go | 5 ++ cmd/agent/app/common_unix.go | 5 ++ cmd/agent/app/main.go | 19 +++++ pkg/config/config.go | 115 +++++++++++++++++++++++++++ pkg/config/config_test.go | 46 +++++++++++ pkg/config/file_provider.go | 37 +++++++++ pkg/config/file_provider_test.go | 32 ++++++++ pkg/config/test/datadog.conf | 2 + pkg/config/test/failing/datadog.conf | 1 + 9 files changed, 262 insertions(+) create mode 100644 cmd/agent/app/common_osx.go create mode 100644 cmd/agent/app/common_unix.go create mode 100644 pkg/config/config.go create mode 100644 pkg/config/config_test.go create mode 100644 pkg/config/file_provider.go create mode 100644 pkg/config/file_provider_test.go create mode 100644 pkg/config/test/datadog.conf create mode 100644 pkg/config/test/failing/datadog.conf diff --git a/cmd/agent/app/common_osx.go b/cmd/agent/app/common_osx.go new file mode 100644 index 000000000000..d32b7c4922c0 --- /dev/null +++ b/cmd/agent/app/common_osx.go @@ -0,0 +1,5 @@ +// +build darwin + +package ddagentmain + +const configPath = "/opt/datadog-agent/etc" diff --git a/cmd/agent/app/common_unix.go b/cmd/agent/app/common_unix.go new file mode 100644 index 000000000000..ad5969040196 --- /dev/null +++ b/cmd/agent/app/common_unix.go @@ -0,0 +1,5 @@ +// +build dragonfly freebsd linux nacl netbsd openbsd solaris + +package ddagentmain + +const configPath = "/etc/dd-agent" diff --git a/cmd/agent/app/main.go b/cmd/agent/app/main.go index 68b73e1eacaa..354b9deb5436 100644 --- a/cmd/agent/app/main.go +++ b/cmd/agent/app/main.go @@ -8,6 +8,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/collector/check/core" "github.com/DataDog/datadog-agent/pkg/collector/check/py" "github.com/DataDog/datadog-agent/pkg/collector/loader" + "github.com/DataDog/datadog-agent/pkg/config" "github.com/kardianos/osext" "github.com/op/go-logging" "github.com/sbinet/go-python" @@ -38,6 +39,8 @@ type metric struct { type metrics map[string][]metric +// build a list of providers for checks' configurations, the sequence defines +// the precedence. func getConfigProviders() (providers []loader.ConfigProvider) { confdPath := filepath.Join(distPath, "conf.d") configPaths := []string{confdPath} @@ -48,6 +51,7 @@ func getConfigProviders() (providers []loader.ConfigProvider) { return providers } +// build a list of check loaders, the sequence defines the precedence. func getCheckLoaders() []loader.CheckLoader { return []loader.CheckLoader{ py.NewPythonCheckLoader(), @@ -55,11 +59,26 @@ func getCheckLoaders() []loader.CheckLoader { } } +// define configuration data for the Agent using different providers +func getConfiguration() *config.Config { + cfg := config.NewConfig() + + // for now, we only load configuration from file + fileProvider := config.NewFileProvider(configPath) + fileProvider.Configure(cfg) + + return cfg +} + // Start the main check loop func Start() { log.Infof("Starting Datadog Agent v%v", agentVersion) + // Global Agent configuration + // config := getConfiguration() + + // Create a channel to enqueue the checks pending := make(chan check.Check, 10) // Initialize the CPython interpreter diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 000000000000..a80519289fc6 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,115 @@ +package config + +import "gopkg.in/yaml.v2" + +// Provider is the interface any object gathering configuration +// options from various places has to implement +type Provider interface { + Configure(*Config) error +} + +// Config contains any possible configuration parameter for the Agent +type Config struct { + // The host of the Datadog intake server to send Agent data to + DdURL string `yaml:"dd_url"` + // If you need a proxy to connect to the Internet, provide the settings here (default: disabled) + ProxyHost string `yaml:"proxy_host"` + ProxyPort int `yaml:"proxy_port"` + ProxyUser string `yaml:"proxy_user"` + ProxyPass string `yaml:"proxy_password"` + // To be used with some proxys that return a 302 which make curl switch from POST to GET + // See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get + ProxyForbidMethodSwitch bool `yaml:"proxy_forbid_method_switch"` + + // If you run the agent behind haproxy, you might want to enable this + SkipSSLValidation bool `yaml:"skip_ssl_validation"` + + // The Datadog api key to associate your Agent's data with your organization. + // Can be found here: https://app.datadoghq.com/account/settings + APIKey string `yaml:"api_key"` + + // Force the hostname to whatever you want. (default: auto-detected) + HostName string `yaml:"hostname"` + + // Set the host's tags (optional) + Tags string `yaml:"tags"` + + // Set timeout in seconds for outgoing requests to Datadog. (default: 20) + // When a request timeout, it will be retried after some time. + // It will only be deleted if the forwarder queue becomes too big. (30 MB by default) + ForwarderTimeout int `yaml:"forwarder_timeout"` + + // Set timeout in seconds for integrations that use HTTP to fetch metrics, since + // unbounded timeouts can potentially block the collector indefinitely and cause + // problems! + DefaultIntegrationHTTPTimeout int `yaml:"default_integration_http_timeout"` + + // Add one "dd_check:checkname" tag per running check. It makes it possible to slice + // and dice per monitored app (= running Agent Check) on Datadog's backend. + CreateDDCheckTags bool `yaml:"create_dd_check_tags"` + + // Collect AWS EC2 custom tags as agent tags (requires an IAM role associated with the instance) + CollectEC2Tags bool `yaml:"collect_ec2_tags"` + // Incorporate security-groups into tags collected from AWS EC2 + CollectSecurityGroups bool `yaml:"collect_security_groups"` + + // Enable Agent Developer Mode + // Agent Developer Mode collects and sends more fine-grained metrics about agent and check performance + DeveloperMode bool `yaml:"developer_mode"` + // In developer mode, the number of runs to be included in a single collector profile + CollectorProfileInterval bool `yaml:"collector_profile_interval"` + + // use unique hostname for GCE hosts, see http://dtdg.co/1eAynZk + GCEUpdatedHostname bool `yaml:"gce_updated_hostname"` + + // Set the threshold for accepting points to allow anything + // within recent_point_threshold seconds (default: 30) + RecentPointThreshold int `yaml:"recent_point_threshold"` + + // Additional directory to look for Datadog checks (optional) + AdditionalChecksd string `yaml:"additional_checksd"` + + // If enabled the collector will capture a metric for check run times. + CheckTimings bool `yaml:"check_timings"` + + // If you want to remove the 'ww' flag from ps catching the arguments of processes + // for instance for security reasons + ExcludeProcessArgs bool `yaml:"exclude_process_args"` + + HistogramAggregates string `yaml:"histogram_aggregates"` + HistogramPercentiles float64 `yaml:"histogram_percentiles"` + + // In some environments we may have the procfs file system mounted in a + // miscellaneous location. The procfs_path configuration paramenter allows + // us to override the standard default location '/proc' + ProcfsPath string `yaml:"procfs_path"` + + LogLevel string `yaml:"log_level"` + LogFile string `yaml:"collector_log_file"` + LogToSyslog bool `yaml:"log_to_syslog"` + SyslogHost string `yaml:"syslog_host"` + SyslogPort int `yaml:"syslog_port"` +} + +// NewConfig creates a new Config instance +func NewConfig() *Config { + return &Config{ + DdURL: "https://app.datadoghq.com", + ForwarderTimeout: 20, + RecentPointThreshold: 30, + HistogramAggregates: "max, median, avg, count", + HistogramPercentiles: 0.95, + ProcfsPath: "/proc", + } +} + +// FromYAML tries to load the configration from an array of Bytes +// containing YAML code +func (c *Config) FromYAML(data []byte) error { + err := yaml.Unmarshal(data, c) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 000000000000..22b989fcab48 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,46 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewConfig(t *testing.T) { + c := NewConfig() + assert.Equal(t, c.DdURL, "https://app.datadoghq.com") + assert.Equal(t, c.ForwarderTimeout, 20) + assert.Equal(t, c.RecentPointThreshold, 30) + assert.Equal(t, c.HistogramAggregates, "max, median, avg, count") + assert.Equal(t, c.HistogramPercentiles, 0.95) + assert.Equal(t, c.ProcfsPath, "/proc") +} + +func TestFromYAML(t *testing.T) { + fixture := ` + dd_url: https://app.datadoghq.com + proxy_host: my-proxy.com + proxy_port: 3128 + api_key: abcdefghijklmnopqrstuvwxyz1234567890 + forwarder_timeout: 19 + recent_point_threshold: 29 + histogram_aggregates: foo, bar, baz + histogram_percentiles: 0.94 + procfs_path: /foo/bar + does_not_exist: foo + ` + c := NewConfig() + err := c.FromYAML([]byte(fixture)) + assert.Nil(t, err) + assert.Equal(t, c.ProxyHost, "my-proxy.com") + assert.Equal(t, c.ProxyPort, 3128) + assert.Equal(t, c.APIKey, "abcdefghijklmnopqrstuvwxyz1234567890") + assert.Equal(t, c.ForwarderTimeout, 19) + assert.Equal(t, c.RecentPointThreshold, 29) + assert.Equal(t, c.HistogramAggregates, "foo, bar, baz") + assert.Equal(t, c.HistogramPercentiles, 0.94) + assert.Equal(t, c.ProcfsPath, "/foo/bar") + + err = c.FromYAML([]byte("/")) + assert.NotNil(t, err) +} diff --git a/pkg/config/file_provider.go b/pkg/config/file_provider.go new file mode 100644 index 000000000000..5210a3151b1a --- /dev/null +++ b/pkg/config/file_provider.go @@ -0,0 +1,37 @@ +package config + +import ( + "io/ioutil" + "path/filepath" + + "github.com/op/go-logging" +) + +const configFileName = "datadog.conf" + +var log = logging.MustGetLogger("datadog-agent") + +// FileProvider retrieves configuration data from text files on the +// filesystem containing YAML code. +type FileProvider struct { + searchPath string +} + +// NewFileProvider returns a provider for configuration files. It will +// search for `configFileName` within the given path. +func NewFileProvider(path string) *FileProvider { + return &FileProvider{searchPath: path} +} + +// Configure tries to open the configuration file, read YAML data and +// unmarshal it into the Config instance. +func (p *FileProvider) Configure(config *Config) error { + configFilePath := filepath.Join(p.searchPath, configFileName) + yamlData, err := ioutil.ReadFile(configFilePath) + + if err != nil { + log.Errorf("Unable to read config from file %s, skipping...", configFilePath) + return err + } + return config.FromYAML(yamlData) +} diff --git a/pkg/config/file_provider_test.go b/pkg/config/file_provider_test.go new file mode 100644 index 000000000000..13468ed04cab --- /dev/null +++ b/pkg/config/file_provider_test.go @@ -0,0 +1,32 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewProvider(t *testing.T) { + p := NewFileProvider("foo") + assert.Equal(t, p.searchPath, "foo") +} + +func TestConfigure(t *testing.T) { + c := NewConfig() + p := NewFileProvider("foo") + + err := p.Configure(c) + assert.NotNil(t, err) + assert.EqualError(t, err, "open foo/datadog.conf: no such file or directory") + + p.searchPath = filepath.Join("test", "failing") + err = p.Configure(c) + assert.NotNil(t, err) + assert.EqualError(t, err, "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `not a y...` into config.Config") + + p.searchPath = "test" + err = p.Configure(c) + assert.Nil(t, err) + assert.Equal(t, c.DdURL, "https://foo.datadoghq.com") +} diff --git a/pkg/config/test/datadog.conf b/pkg/config/test/datadog.conf new file mode 100644 index 000000000000..2d768253c007 --- /dev/null +++ b/pkg/config/test/datadog.conf @@ -0,0 +1,2 @@ +# The host of the Datadog intake server to send Agent data to +dd_url: https://foo.datadoghq.com diff --git a/pkg/config/test/failing/datadog.conf b/pkg/config/test/failing/datadog.conf new file mode 100644 index 000000000000..cee5d4b120cc --- /dev/null +++ b/pkg/config/test/failing/datadog.conf @@ -0,0 +1 @@ +not a yaml file From abdb04c978619bfc605bc0cb2b9f926333eab8c7 Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Thu, 8 Sep 2016 16:35:02 +0200 Subject: [PATCH 2/2] use the agent config provider interface in the main --- cmd/agent/app/main.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/agent/app/main.go b/cmd/agent/app/main.go index 354b9deb5436..272641f7ab35 100644 --- a/cmd/agent/app/main.go +++ b/cmd/agent/app/main.go @@ -59,15 +59,12 @@ func getCheckLoaders() []loader.CheckLoader { } } -// define configuration data for the Agent using different providers -func getConfiguration() *config.Config { - cfg := config.NewConfig() - - // for now, we only load configuration from file - fileProvider := config.NewFileProvider(configPath) - fileProvider.Configure(cfg) - - return cfg +// build a list of providers for Agent configuration, the sequence +// define the precedence. +func getAgentConfigProviders() (providers []config.Provider) { + return []config.Provider{ + config.NewFileProvider(configPath), + } } // Start the main check loop @@ -76,7 +73,12 @@ func Start() { log.Infof("Starting Datadog Agent v%v", agentVersion) // Global Agent configuration - // config := getConfiguration() + cfg := config.NewConfig() + for _, provider := range getAgentConfigProviders() { + if err := provider.Configure(cfg); err != nil { + log.Warningf("Unable to load configuration from provider %v: %v", provider, err) + } + } // Create a channel to enqueue the checks pending := make(chan check.Check, 10)