Load collector configuration

cmd/agent/app/common_osx.go

@@ -0,0 +1,5 @@
+// +build darwin
+
+package ddagentmain
+
+const configPath = "/opt/datadog-agent/etc"

cmd/agent/app/common_unix.go

@@ -0,0 +1,5 @@
+// +build dragonfly freebsd linux nacl netbsd openbsd solaris
+
+package ddagentmain
+
+const configPath = "/etc/dd-agent"

cmd/agent/app/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/collector/check/core"
 	"github.com/DataDog/datadog-agent/pkg/collector/check/py"
 	"github.com/DataDog/datadog-agent/pkg/collector/loader"
+	"github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/kardianos/osext"
 	"github.com/op/go-logging"
 	"github.com/sbinet/go-python"
@@ -38,6 +39,8 @@ type metric struct {
 
 type metrics map[string][]metric
 
+// build a list of providers for checks' configurations, the sequence defines
+// the precedence.
 func getConfigProviders() (providers []loader.ConfigProvider) {
 	confdPath := filepath.Join(distPath, "conf.d")
 	configPaths := []string{confdPath}
@@ -48,18 +51,36 @@ func getConfigProviders() (providers []loader.ConfigProvider) {
 	return providers
 }
 
+// build a list of check loaders, the sequence defines the precedence.
 func getCheckLoaders() []loader.CheckLoader {
 	return []loader.CheckLoader{
 		py.NewPythonCheckLoader(),
 		core.NewGoCheckLoader(),
 	}
 }
 
+// build a list of providers for Agent configuration, the sequence
+// define the precedence.
+func getAgentConfigProviders() (providers []config.Provider) {
+	return []config.Provider{
+		config.NewFileProvider(configPath),
+	}
+}
+
 // Start the main check loop
 func Start() {
 
 	log.Infof("Starting Datadog Agent v%v", agentVersion)
 
+	// Global Agent configuration
+	cfg := config.NewConfig()
+	for _, provider := range getAgentConfigProviders() {
+		if err := provider.Configure(cfg); err != nil {
+			log.Warningf("Unable to load configuration from provider %v: %v", provider, err)
+		}
+	}
+
+	// Create a channel to enqueue the checks
 	pending := make(chan check.Check, 10)
 
 	// Initialize the CPython interpreter

pkg/config/config.go

@@ -0,0 +1,115 @@
+package config
+
+import "gopkg.in/yaml.v2"
+
+// Provider is the interface any object gathering configuration
+// options from various places has to implement
+type Provider interface {
+	Configure(*Config) error
+}
+
+// Config contains any possible configuration parameter for the Agent
+type Config struct {
+	// The host of the Datadog intake server to send Agent data to
+	DdURL string `yaml:"dd_url"`
+	// If you need a proxy to connect to the Internet, provide the settings here (default: disabled)
+	ProxyHost string `yaml:"proxy_host"`
+	ProxyPort int    `yaml:"proxy_port"`
+	ProxyUser string `yaml:"proxy_user"`
+	ProxyPass string `yaml:"proxy_password"`
+	// To be used with some proxys that return a 302 which make curl switch from POST to GET
+	// See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get
+	ProxyForbidMethodSwitch bool `yaml:"proxy_forbid_method_switch"`
+
+	// If you run the agent behind haproxy, you might want to enable this
+	SkipSSLValidation bool `yaml:"skip_ssl_validation"`
+
+	// The Datadog api key to associate your Agent's data with your organization.
+	// Can be found here: https://app.datadoghq.com/account/settings
+	APIKey string `yaml:"api_key"`
+
+	// Force the hostname to whatever you want. (default: auto-detected)
+	HostName string `yaml:"hostname"`
+
+	// Set the host's tags (optional)
+	Tags string `yaml:"tags"`
+
+	// Set timeout in seconds for outgoing requests to Datadog. (default: 20)
+	// When a request timeout, it will be retried after some time.
+	// It will only be deleted if the forwarder queue becomes too big. (30 MB by default)
+	ForwarderTimeout int `yaml:"forwarder_timeout"`
+
+	// Set timeout in seconds for integrations that use HTTP to fetch metrics, since
+	// unbounded timeouts can potentially block the collector indefinitely and cause
+	// problems!
+	DefaultIntegrationHTTPTimeout int `yaml:"default_integration_http_timeout"`
+
+	// Add one "dd_check:checkname" tag per running check. It makes it possible to slice
+	// and dice per monitored app (= running Agent Check) on Datadog's backend.
+	CreateDDCheckTags bool `yaml:"create_dd_check_tags"`
+
+	// Collect AWS EC2 custom tags as agent tags (requires an IAM role associated with the instance)
+	CollectEC2Tags bool `yaml:"collect_ec2_tags"`
+	// Incorporate security-groups into tags collected from AWS EC2
+	CollectSecurityGroups bool `yaml:"collect_security_groups"`
+
+	// Enable Agent Developer Mode
+	// Agent Developer Mode collects and sends more fine-grained metrics about agent and check performance
+	DeveloperMode bool `yaml:"developer_mode"`
+	// In developer mode, the number of runs to be included in a single collector profile
+	CollectorProfileInterval bool `yaml:"collector_profile_interval"`
+
+	// use unique hostname for GCE hosts, see http://dtdg.co/1eAynZk
+	GCEUpdatedHostname bool `yaml:"gce_updated_hostname"`
+
+	// Set the threshold for accepting points to allow anything
+	// within recent_point_threshold seconds (default: 30)
+	RecentPointThreshold int `yaml:"recent_point_threshold"`
+
+	// Additional directory to look for Datadog checks (optional)
+	AdditionalChecksd string `yaml:"additional_checksd"`
+
+	// If enabled the collector will capture a metric for check run times.
+	CheckTimings bool `yaml:"check_timings"`
+
+	// If you want to remove the 'ww' flag from ps catching the arguments of processes
+	// for instance for security reasons
+	ExcludeProcessArgs bool `yaml:"exclude_process_args"`
+
+	HistogramAggregates  string  `yaml:"histogram_aggregates"`
+	HistogramPercentiles float64 `yaml:"histogram_percentiles"`
+
+	// In some environments we may have the procfs file system mounted in a
+	// miscellaneous location. The procfs_path configuration paramenter allows
+	// us to override the standard default location '/proc'
+	ProcfsPath string `yaml:"procfs_path"`
+
+	LogLevel    string `yaml:"log_level"`
+	LogFile     string `yaml:"collector_log_file"`
+	LogToSyslog bool   `yaml:"log_to_syslog"`
+	SyslogHost  string `yaml:"syslog_host"`
+	SyslogPort  int    `yaml:"syslog_port"`
+}
+
+// NewConfig creates a new Config instance
+func NewConfig() *Config {
+	return &Config{
+		DdURL:                "https://app.datadoghq.com",
+		ForwarderTimeout:     20,
+		RecentPointThreshold: 30,
+		HistogramAggregates:  "max, median, avg, count",
+		HistogramPercentiles: 0.95,
+		ProcfsPath:           "/proc",
+	}
+}
+
+// FromYAML tries to load the configration from an array of Bytes
+// containing YAML code
+func (c *Config) FromYAML(data []byte) error {
+	err := yaml.Unmarshal(data, c)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/config/config_test.go

@@ -0,0 +1,46 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewConfig(t *testing.T) {
+	c := NewConfig()
+	assert.Equal(t, c.DdURL, "https://app.datadoghq.com")
+	assert.Equal(t, c.ForwarderTimeout, 20)
+	assert.Equal(t, c.RecentPointThreshold, 30)
+	assert.Equal(t, c.HistogramAggregates, "max, median, avg, count")
+	assert.Equal(t, c.HistogramPercentiles, 0.95)
+	assert.Equal(t, c.ProcfsPath, "/proc")
+}
+
+func TestFromYAML(t *testing.T) {
+	fixture := `
+  dd_url: https://app.datadoghq.com
+  proxy_host: my-proxy.com
+  proxy_port: 3128
+  api_key: abcdefghijklmnopqrstuvwxyz1234567890
+  forwarder_timeout: 19
+  recent_point_threshold: 29
+  histogram_aggregates: foo, bar, baz
+  histogram_percentiles: 0.94
+  procfs_path: /foo/bar
+  does_not_exist: foo
+  `
+	c := NewConfig()
+	err := c.FromYAML([]byte(fixture))
+	assert.Nil(t, err)
+	assert.Equal(t, c.ProxyHost, "my-proxy.com")
+	assert.Equal(t, c.ProxyPort, 3128)
+	assert.Equal(t, c.APIKey, "abcdefghijklmnopqrstuvwxyz1234567890")
+	assert.Equal(t, c.ForwarderTimeout, 19)
+	assert.Equal(t, c.RecentPointThreshold, 29)
+	assert.Equal(t, c.HistogramAggregates, "foo, bar, baz")
+	assert.Equal(t, c.HistogramPercentiles, 0.94)
+	assert.Equal(t, c.ProcfsPath, "/foo/bar")
+
+	err = c.FromYAML([]byte("/"))
+	assert.NotNil(t, err)
+}

pkg/config/file_provider.go

@@ -0,0 +1,37 @@
+package config
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/op/go-logging"
+)
+
+const configFileName = "datadog.conf"
+
+var log = logging.MustGetLogger("datadog-agent")
+
+// FileProvider retrieves configuration data from text files on the
+// filesystem containing YAML code.
+type FileProvider struct {
+	searchPath string
+}
+
+// NewFileProvider returns a provider for configuration files. It will
+// search for `configFileName` within the given path.
+func NewFileProvider(path string) *FileProvider {
+	return &FileProvider{searchPath: path}
+}
+
+// Configure tries to open the configuration file, read YAML data and
+// unmarshal it into the Config instance.
+func (p *FileProvider) Configure(config *Config) error {
+	configFilePath := filepath.Join(p.searchPath, configFileName)
+	yamlData, err := ioutil.ReadFile(configFilePath)
+
+	if err != nil {
+		log.Errorf("Unable to read config from file %s, skipping...", configFilePath)
+		return err
+	}
+	return config.FromYAML(yamlData)
+}

pkg/config/file_provider_test.go

@@ -0,0 +1,32 @@
+package config
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewProvider(t *testing.T) {
+	p := NewFileProvider("foo")
+	assert.Equal(t, p.searchPath, "foo")
+}
+
+func TestConfigure(t *testing.T) {
+	c := NewConfig()
+	p := NewFileProvider("foo")
+
+	err := p.Configure(c)
+	assert.NotNil(t, err)
+	assert.EqualError(t, err, "open foo/datadog.conf: no such file or directory")
+
+	p.searchPath = filepath.Join("test", "failing")
+	err = p.Configure(c)
+	assert.NotNil(t, err)
+	assert.EqualError(t, err, "yaml: unmarshal errors:\n  line 1: cannot unmarshal !!str `not a y...` into config.Config")
+
+	p.searchPath = "test"
+	err = p.Configure(c)
+	assert.Nil(t, err)
+	assert.Equal(t, c.DdURL, "https://foo.datadoghq.com")
+}

pkg/config/test/datadog.conf

@@ -0,0 +1,2 @@
+# The host of the Datadog intake server to send Agent data to
+dd_url: https://foo.datadoghq.com

pkg/config/test/failing/datadog.conf

@@ -0,0 +1 @@
+not a yaml file