[core] Config loader: first draft

cmd/agent/app/main.go

@@ -1,12 +1,13 @@
 package ddagentmain
 
 import (
-	"fmt"
+	"path/filepath"
 	"time"
 
 	"github.com/DataDog/datadog-agent/aggregator"
 	"github.com/DataDog/datadog-agent/pkg/checks"
 	"github.com/DataDog/datadog-agent/pkg/checks/system"
+	"github.com/DataDog/datadog-agent/pkg/loader"
 	"github.com/DataDog/datadog-agent/pkg/py"
 	"github.com/kardianos/osext"
 	"github.com/op/go-logging"
@@ -15,6 +16,8 @@ import (
 
 const AGENT_VERSION = "6.0.0"
 
+var here, _ = osext.ExecutableFolder()
+var distPath = filepath.Join(here, "dist")
 var log = logging.MustGetLogger("datadog-agent")
 
 // schedule all the available checks for running
@@ -33,6 +36,16 @@ type metric struct {
 
 type metrics map[string][]metric
 
+func getConfigProviders() (providers []loader.ConfigProvider) {
+	confdPath := filepath.Join(distPath, "conf.d")
+	configPaths := []string{confdPath}
+
+	// File Provider
+	providers = append(providers, loader.NewFileConfigProvider(configPaths))
+
+	return providers
+}
+
 // Start the main check loop
 func Start() {
 
@@ -44,12 +57,12 @@ func Start() {
 	if err != nil {
 		panic(err.Error())
 	}
+
 	// Set the PYTHONPATH
-	here, _ := osext.ExecutableFolder()
-	distPath := fmt.Sprintf("%s/dist", here)
-	confdPath := fmt.Sprintf("%s/conf.d", distPath)
+	checksPath := filepath.Join(distPath, "checks")
 	path := python.PySys_GetObject("path")
 	python.PyList_Append(path, python.PyString_FromString(distPath))
+	python.PyList_Append(path, python.PyString_FromString(checksPath))
 
 	// `python.Initialize` acquires the GIL but we don't need it, let's release it
 	state := python.PyEval_SaveThread()
@@ -60,12 +73,20 @@ func Start() {
 	// Get a single Runner instance, i.e. we process checks sequentially
 	go checks.Runner(pending)
 
-	// Get a list of Python checks we want to run
-	checksNames := []string{"checks.go_expvar"}
+	// Get a list of config checks from the configured providers
+	var configs []loader.CheckConfig
+	for _, provider := range getConfigProviders() {
+		c, _ := provider.Collect()
+		configs = append(configs, c...)
+	}
+
 	// Search for and import all the desired Python checks
-	checks := py.CollectChecks(checksNames, confdPath)
+	// TODO: this functionality will be implemented by a generic Collector able to
+	// search for and load different checks (Python, Go, whatever...)
+	checks := py.CollectChecks(configs)
 
 	// Run memory check, this is a native check, not Python
+	// TODO: see above, this should be done elsewhere, not manually here
 	mc := system.MemoryCheck{}
 	checks = append(checks, &mc)
 

pkg/loader/README.md

@@ -0,0 +1,16 @@
+This package is responsible of scanning different sources searching for Agent checks' configuration files.
+
+Check configurations may be contained within files on disk, environment variables, external databases: for
+each source, the Agent has a specific _Provider_ implementing the `ConfigProvider` interface.
+
+Check configurations may come in different format, for example Yaml code in the case of config files on disk.
+Every configuration, regardless of the format, must be unmarshalled into a `CheckConfig` struct.
+
+Usage example:
+```go
+var configs []loader.CheckConfig
+for _, provider := range configProviders {
+  c, _ := provider.Collect()
+  configs = append(configs, c...)
+}
+```

pkg/loader/file_provider.go

@@ -0,0 +1,77 @@
+package loader
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/op/go-logging"
+
+	"gopkg.in/yaml.v2"
+)
+
+var log = logging.MustGetLogger("datadog-agent")
+
+// FileConfigProvider collect configuration files from disk
+type FileConfigProvider struct {
+	paths []string
+}
+
+// NewFileConfigProvider creates a new FileConfigProvider searching for
+// configuration files on the given paths
+func NewFileConfigProvider(paths []string) *FileConfigProvider {
+	return &FileConfigProvider{paths: paths}
+}
+
+// Collect scans provided paths searching for configuration files. When found,
+// it parses the files and try to unmarshall Yaml contents into a CheckConfig
+// instance
+func (c *FileConfigProvider) Collect() ([]CheckConfig, error) {
+	configs := []CheckConfig{}
+
+	for _, path := range c.paths {
+		log.Debug("Searching for yaml files at:", path)
+
+		files, err := ioutil.ReadDir(path)
+		if err != nil {
+			log.Warningf("Unable to access dir: %s, skipping...", err)
+			continue
+		}
+
+		for _, f := range files {
+			if f.IsDir() {
+				log.Warningf("%s is a dir, skipping...", f.Name())
+				continue
+			}
+
+			fName := f.Name()
+			extName := filepath.Ext(fName)
+			bName := fName[:len(f.Name())-len(extName)]
+			conf, err := getCheckConfig(bName, filepath.Join(path, fName))
+			if err != nil {
+				log.Warningf("%s is not a valid config file: %s", f.Name(), err)
+				continue
+			}
+
+			log.Debug("Found valid configuration in file:", f.Name())
+			configs = append(configs, conf)
+		}
+	}
+
+	return configs, nil
+}
+
+// getCheckConfig returns an instance of CheckConfig if `fpath` points to a valid config file
+func getCheckConfig(name, fpath string) (CheckConfig, error) {
+	conf := CheckConfig{Name: name}
+
+	// Read file contents
+	// FIXME: ReadFile reads the entire file, possible security implications
+	yamlFile, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return conf, err
+	}
+
+	// Parse configuration
+	err = yaml.Unmarshal(yamlFile, &conf.Data)
+	return conf, err
+}

pkg/loader/file_provider_test.go

@@ -0,0 +1,56 @@
+package loader
+
+import "testing"
+
+func Test_GetCheckConfig(t *testing.T) {
+	config, err := getCheckConfig("foo", "tests/wrong.yaml")
+	if err == nil {
+		t.Fatal("Expecting error")
+	}
+
+	config, err = getCheckConfig("foo", "foo.yaml")
+	if err == nil {
+		t.Fatal("Expecting error")
+	}
+
+	config, err = getCheckConfig("foo", "tests/testcheck.yaml")
+	if err != nil {
+		t.Fatalf("Expecting nil, found: %s", err)
+	}
+	if config.Name != "foo" {
+		t.Fatalf("Expecting `foo`, found: %s", config.Name)
+	}
+}
+
+func TestNewYamlConfigProvider(t *testing.T) {
+	paths := []string{"foo", "bar", "foo/bar"}
+	provider := NewFileConfigProvider(paths)
+	if len(provider.paths) != len(paths) {
+		t.Fatalf("Expecting length %d, found: %d", len(provider.paths), len(paths))
+	}
+
+	for i, p := range provider.paths {
+		if p != paths[i] {
+			t.Fatalf("Expecting %s, found: %s", paths[i], p)
+		}
+	}
+}
+
+func TestCollect(t *testing.T) {
+	paths := []string{"tests", "foo/bar"}
+	provider := NewFileConfigProvider(paths)
+	configs, err := provider.Collect()
+
+	if err != nil {
+		t.Fatalf("Expecting nil, found: %s", err)
+	}
+
+	if len(configs) != 1 {
+		t.Fatalf("Expecting length 1, found: %d", len(configs))
+	}
+
+	config := configs[0]
+	if config.Name != "testcheck" {
+		t.Fatalf("Expecting testcheck, found: %s", config.Name)
+	}
+}

pkg/loader/tests/testcheck.yaml

@@ -0,0 +1,5 @@
+init_config:
+
+instances:
+  # No configuration is needed for this check.
+  - foo: bar

pkg/loader/tests/wrong.yaml

@@ -0,0 +1 @@
+// not valid Yaml

pkg/loader/types.go

@@ -0,0 +1,19 @@
+package loader
+
+// CheckConfig is a generic container for configuration files
+type CheckConfig struct {
+	Name string                 // the name of the check
+	Data map[string]interface{} // raw configuration content, unmarshalled from Yaml
+}
+
+// ConfigProvider is the interface that wraps the Collect method
+//
+// Collect is responsible of populating a list of CheckConfig instances
+// by retrieving configuration patterns from external resources: files
+// on disk, databases, environment variables are just few examples.
+//
+// Any type implementing the interface will take care of any dependency
+// or data needed to access the resource providing the configuration.
+type ConfigProvider interface {
+	Collect() ([]CheckConfig, error)
+}

pkg/py/check.go

@@ -5,6 +5,7 @@ import (
 	"runtime"
 
 	"github.com/DataDog/datadog-agent/pkg/checks"
+	"github.com/DataDog/datadog-agent/pkg/loader"
 	"github.com/op/go-logging"
 	"github.com/sbinet/go-python"
 )
@@ -19,13 +20,13 @@ const agentCheckModuleName = "checks"
 type PythonCheck struct {
 	Instance   *python.PyObject
 	ModuleName string
-	Config     CheckConfig
+	Config     loader.CheckConfig
 }
 
 // NewPythonCheck conveniently creates a PythonCheck instance
-func NewPythonCheck(class *python.PyObject, config CheckConfig) *PythonCheck {
+func NewPythonCheck(class *python.PyObject, config loader.CheckConfig) *PythonCheck {
 	// pack arguments
-	kwargs, _ := config.ToPythonDict()
+	kwargs, _ := ToPythonDict(&config)
 
 	// Lock the GIL and release it at the end
 	_gstate := python.PyGILState_Ensure()
@@ -82,7 +83,7 @@ func (c *PythonCheck) String() string {
 }
 
 // CollectChecks return an array of checks to be performed
-func CollectChecks(modules []string, confdPath string) []checks.Check {
+func CollectChecks(configs []loader.CheckConfig) []checks.Check {
 	// Lock the GIL and release it at the end of the run
 	_gstate := python.PyGILState_Ensure()
 	defer func() {
@@ -103,32 +104,27 @@ func CollectChecks(modules []string, confdPath string) []checks.Check {
 		return checks
 	}
 
-	for _, module := range modules {
+	for _, config := range configs {
+		moduleName := config.Name
+
 		// import python module containing the check
-		checkModule := python.PyImport_ImportModuleNoBlock(module)
+		checkModule := python.PyImport_ImportModuleNoBlock(moduleName)
 		if checkModule == nil {
-			log.Warningf("Unable to import %v", module)
-			python.PyErr_Print()
+			log.Warningf("Unable to import %v", moduleName)
+			python.PyErr_Print() // TODO: remove this or redirect to the Go logger
 			python.PyErr_Clear()
 			continue
 		}
 
 		// Try to find a class inheriting from AgentCheck within the module
 		checkClass := findSubclassOf(agentCheckClass, checkModule)
 		if checkClass == nil {
-			log.Warningf("Unable to find a check class in the module %v", module)
-			continue
-		}
-
-		// Search for a configuration file
-		conf, err := getCheckConfig(confdPath, getModuleName(module))
-		if err != nil {
-			log.Warningf("Error reading Config file: %s. Skipping check...", err)
+			log.Warningf("Unable to find a check class in the module %v", checkModule)
 			continue
 		}
 
 		// Get an AgentCheck instance and add it to the registry
-		check := NewPythonCheck(checkClass, conf)
+		check := NewPythonCheck(checkClass, config)
 		if check != nil {
 			log.Infof("Found check: %v", python.PyString_AsString(checkClass.Str()))
 			checks = append(checks, check)

pkg/py/check_test.go

@@ -3,6 +3,7 @@ package py
 import (
 	"testing"
 
+	"github.com/DataDog/datadog-agent/pkg/loader"
 	"github.com/sbinet/go-python"
 )
 
@@ -13,9 +14,9 @@ func getCheckInstance() *PythonCheck {
 		python.PyGILState_Release(_gstate)
 	}()
 
-	module := python.PyImport_ImportModuleNoBlock("tests.testcheck")
+	module := python.PyImport_ImportModuleNoBlock("testcheck")
 	checkClass := module.GetAttrString("TestCheck")
-	checkConfig, _ := getCheckConfig("tests", "testcheck")
+	checkConfig := loader.CheckConfig{Name: "testcheck"}
 	return NewPythonCheck(checkClass, checkConfig)
 }
 
@@ -27,9 +28,9 @@ func TestNewPythonCheck(t *testing.T) {
 		python.PyGILState_Release(_gstate)
 	}()
 
-	module := python.PyImport_ImportModuleNoBlock("tests.testcheck")
+	module := python.PyImport_ImportModuleNoBlock("testcheck")
 	checkClass := module.GetAttrString("TestCheck")
-	check := NewPythonCheck(checkClass, CheckConfig{})
+	check := NewPythonCheck(checkClass, loader.CheckConfig{})
 
 	if check.Instance.IsInstance(checkClass) != 1 {
 		t.Fatalf("Expected instance of class TestCheck, found: %s",
@@ -58,7 +59,13 @@ func TestStr(t *testing.T) {
 }
 
 func TestCollectChecks(t *testing.T) {
-	checks := CollectChecks([]string{"tests.testcheck", "doesnt.exist", "tests.foo", "tests.testcheck2"}, "tests")
+	configs := []loader.CheckConfig{
+		loader.CheckConfig{Name: "testcheck"},
+		loader.CheckConfig{Name: "doesnt.exist"},
+		loader.CheckConfig{Name: "foo"},
+	}
+
+	checks := CollectChecks(configs)
 	if len(checks) != 1 {
 		t.Fatalf("Expected 1 check loaded, found: %d", len(checks))
 	}