diff --git a/cmd/agent/app/main.go b/cmd/agent/app/main.go index 574569de7171..88dc9adf9915 100644 --- a/cmd/agent/app/main.go +++ b/cmd/agent/app/main.go @@ -1,12 +1,13 @@ package ddagentmain import ( - "fmt" + "path/filepath" "time" "github.com/DataDog/datadog-agent/aggregator" "github.com/DataDog/datadog-agent/pkg/checks" "github.com/DataDog/datadog-agent/pkg/checks/system" + "github.com/DataDog/datadog-agent/pkg/loader" "github.com/DataDog/datadog-agent/pkg/py" "github.com/kardianos/osext" "github.com/op/go-logging" @@ -15,6 +16,8 @@ import ( const AGENT_VERSION = "6.0.0" +var here, _ = osext.ExecutableFolder() +var distPath = filepath.Join(here, "dist") var log = logging.MustGetLogger("datadog-agent") // schedule all the available checks for running @@ -33,6 +36,16 @@ type metric struct { type metrics map[string][]metric +func getConfigProviders() (providers []loader.ConfigProvider) { + confdPath := filepath.Join(distPath, "conf.d") + configPaths := []string{confdPath} + + // File Provider + providers = append(providers, loader.NewFileConfigProvider(configPaths)) + + return providers +} + // Start the main check loop func Start() { @@ -44,12 +57,12 @@ func Start() { if err != nil { panic(err.Error()) } + // Set the PYTHONPATH - here, _ := osext.ExecutableFolder() - distPath := fmt.Sprintf("%s/dist", here) - confdPath := fmt.Sprintf("%s/conf.d", distPath) + checksPath := filepath.Join(distPath, "checks") path := python.PySys_GetObject("path") python.PyList_Append(path, python.PyString_FromString(distPath)) + python.PyList_Append(path, python.PyString_FromString(checksPath)) // `python.Initialize` acquires the GIL but we don't need it, let's release it state := python.PyEval_SaveThread() @@ -60,12 +73,20 @@ func Start() { // Get a single Runner instance, i.e. we process checks sequentially go checks.Runner(pending) - // Get a list of Python checks we want to run - checksNames := []string{"checks.go_expvar"} + // Get a list of config checks from the configured providers + var configs []loader.CheckConfig + for _, provider := range getConfigProviders() { + c, _ := provider.Collect() + configs = append(configs, c...) + } + // Search for and import all the desired Python checks - checks := py.CollectChecks(checksNames, confdPath) + // TODO: this functionality will be implemented by a generic Collector able to + // search for and load different checks (Python, Go, whatever...) + checks := py.CollectChecks(configs) // Run memory check, this is a native check, not Python + // TODO: see above, this should be done elsewhere, not manually here mc := system.MemoryCheck{} checks = append(checks, &mc) diff --git a/pkg/loader/README.md b/pkg/loader/README.md new file mode 100644 index 000000000000..d645628356c0 --- /dev/null +++ b/pkg/loader/README.md @@ -0,0 +1,16 @@ +This package is responsible of scanning different sources searching for Agent checks' configuration files. + +Check configurations may be contained within files on disk, environment variables, external databases: for +each source, the Agent has a specific _Provider_ implementing the `ConfigProvider` interface. + +Check configurations may come in different format, for example Yaml code in the case of config files on disk. +Every configuration, regardless of the format, must be unmarshalled into a `CheckConfig` struct. + +Usage example: +```go +var configs []loader.CheckConfig +for _, provider := range configProviders { + c, _ := provider.Collect() + configs = append(configs, c...) +} +``` diff --git a/pkg/loader/file_provider.go b/pkg/loader/file_provider.go new file mode 100644 index 000000000000..9208ed51bcb3 --- /dev/null +++ b/pkg/loader/file_provider.go @@ -0,0 +1,77 @@ +package loader + +import ( + "io/ioutil" + "path/filepath" + + "github.com/op/go-logging" + + "gopkg.in/yaml.v2" +) + +var log = logging.MustGetLogger("datadog-agent") + +// FileConfigProvider collect configuration files from disk +type FileConfigProvider struct { + paths []string +} + +// NewFileConfigProvider creates a new FileConfigProvider searching for +// configuration files on the given paths +func NewFileConfigProvider(paths []string) *FileConfigProvider { + return &FileConfigProvider{paths: paths} +} + +// Collect scans provided paths searching for configuration files. When found, +// it parses the files and try to unmarshall Yaml contents into a CheckConfig +// instance +func (c *FileConfigProvider) Collect() ([]CheckConfig, error) { + configs := []CheckConfig{} + + for _, path := range c.paths { + log.Debug("Searching for yaml files at:", path) + + files, err := ioutil.ReadDir(path) + if err != nil { + log.Warningf("Unable to access dir: %s, skipping...", err) + continue + } + + for _, f := range files { + if f.IsDir() { + log.Warningf("%s is a dir, skipping...", f.Name()) + continue + } + + fName := f.Name() + extName := filepath.Ext(fName) + bName := fName[:len(f.Name())-len(extName)] + conf, err := getCheckConfig(bName, filepath.Join(path, fName)) + if err != nil { + log.Warningf("%s is not a valid config file: %s", f.Name(), err) + continue + } + + log.Debug("Found valid configuration in file:", f.Name()) + configs = append(configs, conf) + } + } + + return configs, nil +} + +// getCheckConfig returns an instance of CheckConfig if `fpath` points to a valid config file +func getCheckConfig(name, fpath string) (CheckConfig, error) { + conf := CheckConfig{Name: name} + + // Read file contents + // FIXME: ReadFile reads the entire file, possible security implications + yamlFile, err := ioutil.ReadFile(fpath) + if err != nil { + return conf, err + } + + // Parse configuration + err = yaml.Unmarshal(yamlFile, &conf.Data) + return conf, err +} diff --git a/pkg/loader/file_provider_test.go b/pkg/loader/file_provider_test.go new file mode 100644 index 000000000000..20bb01917cd3 --- /dev/null +++ b/pkg/loader/file_provider_test.go @@ -0,0 +1,56 @@ +package loader + +import "testing" + +func Test_GetCheckConfig(t *testing.T) { + config, err := getCheckConfig("foo", "tests/wrong.yaml") + if err == nil { + t.Fatal("Expecting error") + } + + config, err = getCheckConfig("foo", "foo.yaml") + if err == nil { + t.Fatal("Expecting error") + } + + config, err = getCheckConfig("foo", "tests/testcheck.yaml") + if err != nil { + t.Fatalf("Expecting nil, found: %s", err) + } + if config.Name != "foo" { + t.Fatalf("Expecting `foo`, found: %s", config.Name) + } +} + +func TestNewYamlConfigProvider(t *testing.T) { + paths := []string{"foo", "bar", "foo/bar"} + provider := NewFileConfigProvider(paths) + if len(provider.paths) != len(paths) { + t.Fatalf("Expecting length %d, found: %d", len(provider.paths), len(paths)) + } + + for i, p := range provider.paths { + if p != paths[i] { + t.Fatalf("Expecting %s, found: %s", paths[i], p) + } + } +} + +func TestCollect(t *testing.T) { + paths := []string{"tests", "foo/bar"} + provider := NewFileConfigProvider(paths) + configs, err := provider.Collect() + + if err != nil { + t.Fatalf("Expecting nil, found: %s", err) + } + + if len(configs) != 1 { + t.Fatalf("Expecting length 1, found: %d", len(configs)) + } + + config := configs[0] + if config.Name != "testcheck" { + t.Fatalf("Expecting testcheck, found: %s", config.Name) + } +} diff --git a/pkg/loader/tests/nested/ignore.yaml b/pkg/loader/tests/nested/ignore.yaml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/pkg/loader/tests/testcheck.yaml b/pkg/loader/tests/testcheck.yaml new file mode 100644 index 000000000000..112316085225 --- /dev/null +++ b/pkg/loader/tests/testcheck.yaml @@ -0,0 +1,5 @@ +init_config: + +instances: + # No configuration is needed for this check. + - foo: bar diff --git a/pkg/loader/tests/wrong.yaml b/pkg/loader/tests/wrong.yaml new file mode 100644 index 000000000000..437ca4fe0e61 --- /dev/null +++ b/pkg/loader/tests/wrong.yaml @@ -0,0 +1 @@ +// not valid Yaml diff --git a/pkg/loader/types.go b/pkg/loader/types.go new file mode 100644 index 000000000000..7ad70edefb6b --- /dev/null +++ b/pkg/loader/types.go @@ -0,0 +1,19 @@ +package loader + +// CheckConfig is a generic container for configuration files +type CheckConfig struct { + Name string // the name of the check + Data map[string]interface{} // raw configuration content, unmarshalled from Yaml +} + +// ConfigProvider is the interface that wraps the Collect method +// +// Collect is responsible of populating a list of CheckConfig instances +// by retrieving configuration patterns from external resources: files +// on disk, databases, environment variables are just few examples. +// +// Any type implementing the interface will take care of any dependency +// or data needed to access the resource providing the configuration. +type ConfigProvider interface { + Collect() ([]CheckConfig, error) +} diff --git a/pkg/py/check.go b/pkg/py/check.go index 35149433769e..0ea20499f762 100644 --- a/pkg/py/check.go +++ b/pkg/py/check.go @@ -5,6 +5,7 @@ import ( "runtime" "github.com/DataDog/datadog-agent/pkg/checks" + "github.com/DataDog/datadog-agent/pkg/loader" "github.com/op/go-logging" "github.com/sbinet/go-python" ) @@ -19,13 +20,13 @@ const agentCheckModuleName = "checks" type PythonCheck struct { Instance *python.PyObject ModuleName string - Config CheckConfig + Config loader.CheckConfig } // NewPythonCheck conveniently creates a PythonCheck instance -func NewPythonCheck(class *python.PyObject, config CheckConfig) *PythonCheck { +func NewPythonCheck(class *python.PyObject, config loader.CheckConfig) *PythonCheck { // pack arguments - kwargs, _ := config.ToPythonDict() + kwargs, _ := ToPythonDict(&config) // Lock the GIL and release it at the end _gstate := python.PyGILState_Ensure() @@ -82,7 +83,7 @@ func (c *PythonCheck) String() string { } // CollectChecks return an array of checks to be performed -func CollectChecks(modules []string, confdPath string) []checks.Check { +func CollectChecks(configs []loader.CheckConfig) []checks.Check { // Lock the GIL and release it at the end of the run _gstate := python.PyGILState_Ensure() defer func() { @@ -103,12 +104,14 @@ func CollectChecks(modules []string, confdPath string) []checks.Check { return checks } - for _, module := range modules { + for _, config := range configs { + moduleName := config.Name + // import python module containing the check - checkModule := python.PyImport_ImportModuleNoBlock(module) + checkModule := python.PyImport_ImportModuleNoBlock(moduleName) if checkModule == nil { - log.Warningf("Unable to import %v", module) - python.PyErr_Print() + log.Warningf("Unable to import %v", moduleName) + python.PyErr_Print() // TODO: remove this or redirect to the Go logger python.PyErr_Clear() continue } @@ -116,19 +119,12 @@ func CollectChecks(modules []string, confdPath string) []checks.Check { // Try to find a class inheriting from AgentCheck within the module checkClass := findSubclassOf(agentCheckClass, checkModule) if checkClass == nil { - log.Warningf("Unable to find a check class in the module %v", module) - continue - } - - // Search for a configuration file - conf, err := getCheckConfig(confdPath, getModuleName(module)) - if err != nil { - log.Warningf("Error reading Config file: %s. Skipping check...", err) + log.Warningf("Unable to find a check class in the module %v", checkModule) continue } // Get an AgentCheck instance and add it to the registry - check := NewPythonCheck(checkClass, conf) + check := NewPythonCheck(checkClass, config) if check != nil { log.Infof("Found check: %v", python.PyString_AsString(checkClass.Str())) checks = append(checks, check) diff --git a/pkg/py/check_test.go b/pkg/py/check_test.go index 3584fa42589d..d6b1e26a516a 100644 --- a/pkg/py/check_test.go +++ b/pkg/py/check_test.go @@ -3,6 +3,7 @@ package py import ( "testing" + "github.com/DataDog/datadog-agent/pkg/loader" "github.com/sbinet/go-python" ) @@ -13,9 +14,9 @@ func getCheckInstance() *PythonCheck { python.PyGILState_Release(_gstate) }() - module := python.PyImport_ImportModuleNoBlock("tests.testcheck") + module := python.PyImport_ImportModuleNoBlock("testcheck") checkClass := module.GetAttrString("TestCheck") - checkConfig, _ := getCheckConfig("tests", "testcheck") + checkConfig := loader.CheckConfig{Name: "testcheck"} return NewPythonCheck(checkClass, checkConfig) } @@ -27,9 +28,9 @@ func TestNewPythonCheck(t *testing.T) { python.PyGILState_Release(_gstate) }() - module := python.PyImport_ImportModuleNoBlock("tests.testcheck") + module := python.PyImport_ImportModuleNoBlock("testcheck") checkClass := module.GetAttrString("TestCheck") - check := NewPythonCheck(checkClass, CheckConfig{}) + check := NewPythonCheck(checkClass, loader.CheckConfig{}) if check.Instance.IsInstance(checkClass) != 1 { t.Fatalf("Expected instance of class TestCheck, found: %s", @@ -58,7 +59,13 @@ func TestStr(t *testing.T) { } func TestCollectChecks(t *testing.T) { - checks := CollectChecks([]string{"tests.testcheck", "doesnt.exist", "tests.foo", "tests.testcheck2"}, "tests") + configs := []loader.CheckConfig{ + loader.CheckConfig{Name: "testcheck"}, + loader.CheckConfig{Name: "doesnt.exist"}, + loader.CheckConfig{Name: "foo"}, + } + + checks := CollectChecks(configs) if len(checks) != 1 { t.Fatalf("Expected 1 check loaded, found: %d", len(checks)) } diff --git a/pkg/py/config.go b/pkg/py/config.go index 29ea69382614..bd5980d75078 100644 --- a/pkg/py/config.go +++ b/pkg/py/config.go @@ -1,30 +1,13 @@ package py import ( - "fmt" - "io/ioutil" - "os" "reflect" - "gopkg.in/yaml.v2" - + "github.com/DataDog/datadog-agent/pkg/loader" "github.com/mitchellh/reflectwalk" "github.com/sbinet/go-python" ) -// CheckConfig is a generic container for YAML configuration files -type CheckConfig struct { - rawData map[string]interface{} -} - -// ToPythonDict copies data into a Python dictionary -func (c *CheckConfig) ToPythonDict() (*python.PyObject, error) { - w := new(walker) - err := reflectwalk.Walk(c.rawData, w) - - return w.result, err -} - // we use this struct to walk through YAML results and convert them to Python stuff type walker struct { result *python.PyObject @@ -33,6 +16,14 @@ type walker struct { currentContainer *python.PyObject } +// ToPythonDict dumps CheckConfig data into a Python dictionary +func ToPythonDict(c *loader.CheckConfig) (*python.PyObject, error) { + w := new(walker) + err := reflectwalk.Walk(c.Data, w) + + return w.result, err +} + // push the old container to the stack and start using the new one func (w *walker) push(newc *python.PyObject) { // special case: init @@ -190,23 +181,3 @@ func (w *walker) SliceElem(i int, v reflect.Value) error { return nil } - -// Read and parse a YAML configuration file from confDir for module `name` -func getCheckConfig(confDir, name string) (CheckConfig, error) { - conf := CheckConfig{} - - // Read file contents - fname := fmt.Sprintf("%s%c%s.yaml", confDir, os.PathSeparator, name) - yamlFile, err := ioutil.ReadFile(fname) - if err != nil { - return conf, err - } - - // Parse configuration - err = yaml.Unmarshal(yamlFile, &conf.rawData) - if err != nil { - return conf, err - } - - return conf, nil -} diff --git a/pkg/py/config_test.go b/pkg/py/config_test.go index 9cf26e9f40db..8ad036e222bc 100644 --- a/pkg/py/config_test.go +++ b/pkg/py/config_test.go @@ -5,6 +5,7 @@ import ( "reflect" "testing" + "github.com/DataDog/datadog-agent/pkg/loader" "github.com/mitchellh/reflectwalk" "github.com/sbinet/go-python" @@ -19,14 +20,14 @@ func TestToPythonDict(t *testing.T) { t.Fatalf("Expected empty error message, found: %s", err) } - c := CheckConfig{} + c := loader.CheckConfig{} - err = yaml.Unmarshal(yamlFile, &c.rawData) + err = yaml.Unmarshal(yamlFile, &c.Data) if err != nil { t.Fatalf("Expected empty error message, found: %s", err) } - res, err := c.ToPythonDict() + res, err := ToPythonDict(&c) if err != nil { t.Fatalf("Expected empty error message, found: %s", err) } @@ -225,29 +226,3 @@ func TestSliceElem(t *testing.T) { t.Fatalf("Expected list lenght 1, found %d", l) } } - -func TestGetCheckConfig(t *testing.T) { - conf, err := getCheckConfig("foo", "bar") - if err == nil { - t.Fatal("Expecting error") - } - if len(conf.rawData) != 0 { - t.Fatalf("Expecting empty Config, found: %d", len(conf.rawData)) - } - - conf, err = getCheckConfig("tests", "bad") - if err == nil { - t.Fatal("Expecting error") - } - if len(conf.rawData) != 0 { - t.Fatalf("Expecting empty Config, found: %v", conf) - } - - conf, err = getCheckConfig("tests", "testcheck") - if err != nil { - t.Fatalf("Error: %v", err) - } - if len(conf.rawData) != 2 { - t.Fatalf("Expecting 2 elem in Config, found: %d", len(conf.rawData)) - } -} diff --git a/pkg/py/tests/bar.py b/pkg/py/tests/bar.py index fd2a84857b6c..4da68f816b94 100644 --- a/pkg/py/tests/bar.py +++ b/pkg/py/tests/bar.py @@ -1,4 +1,4 @@ -from tests import foo +import foo class Bar(foo.Foo): pass diff --git a/pkg/py/utils_test.go b/pkg/py/utils_test.go index b974a0daf7e4..61a3b03c883b 100644 --- a/pkg/py/utils_test.go +++ b/pkg/py/utils_test.go @@ -17,6 +17,7 @@ func TestMain(m *testing.M) { // Set the PYTHONPATH path := python.PySys_GetObject("path") python.PyList_Append(path, python.PyString_FromString(".")) + python.PyList_Append(path, python.PyString_FromString("tests")) python.PyList_Append(path, python.PyString_FromString("dist")) // Initialize acquires the GIL but we don't need it, release it @@ -47,9 +48,9 @@ func TestFindSubclassOf(t *testing.T) { python.PyGILState_Release(_gstate) }() - fooModule := python.PyImport_ImportModuleNoBlock("tests.foo") + fooModule := python.PyImport_ImportModuleNoBlock("foo") fooClass := fooModule.GetAttrString("Foo") - barModule := python.PyImport_ImportModuleNoBlock("tests.bar") + barModule := python.PyImport_ImportModuleNoBlock("bar") barClass := barModule.GetAttrString("Bar") // invalid input