diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index 4c0e0cd382e..19558579196 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -104,7 +104,7 @@ func main() { util.InitEvents(eventSampleRate) // Setting the environment variable JAEGER_AGENT_HOST enables tracing - if trace, err := tracing.NewFromEnv("cortex-" + cfg.Target.String()); err != nil { + if trace, err := tracing.NewFromEnv("cortex-" + cfg.Target); err != nil { level.Error(util.Logger).Log("msg", "Failed to setup tracing", "err", err.Error()) } else { defer trace.Close() diff --git a/cmd/cortex/main_test.go b/cmd/cortex/main_test.go index 75ac3726044..923c43a25a1 100644 --- a/cmd/cortex/main_test.go +++ b/cmd/cortex/main_test.go @@ -26,17 +26,6 @@ func TestFlagParsing(t *testing.T) { stderrMessage: configFileOption, }, - // check that config file is used - "config with unknown target": { - yaml: "target: unknown", - stderrMessage: "unrecognised module name: unknown", - }, - - "argument with unknown target": { - arguments: []string{"-target=unknown"}, - stderrMessage: "unrecognised module name: unknown", - }, - "unknown flag": { arguments: []string{"-unknown.flag"}, stderrMessage: "-unknown.flag", @@ -48,12 +37,6 @@ func TestFlagParsing(t *testing.T) { stdoutMessage: "target: ingester", }, - "config with wrong argument override": { - yaml: "target: ingester", - arguments: []string{"-target=unknown"}, - stderrMessage: "unrecognised module name: unknown", - }, - "default values": { stdoutMessage: "target: all\n", }, @@ -63,11 +46,6 @@ func TestFlagParsing(t *testing.T) { stdoutMessage: "target: ingester\n", }, - "config without expand-env": { - yaml: "target: $TARGET", - stderrMessage: "Error parsing config file: unrecognised module name: $TARGET\n", - }, - "config with expand-env": { arguments: []string{"-config.expand-env"}, yaml: "target: $TARGET", diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 3b279bea44e..cb3e9185e42 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -47,6 +47,7 @@ import ( "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/grpc/healthcheck" + "github.com/cortexproject/cortex/pkg/util/modules" "github.com/cortexproject/cortex/pkg/util/runtimeconfig" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/validation" @@ -71,10 +72,10 @@ import ( // Config is the root config for Cortex. type Config struct { - Target ModuleName `yaml:"target"` - AuthEnabled bool `yaml:"auth_enabled"` - PrintConfig bool `yaml:"-"` - HTTPPrefix string `yaml:"http_prefix"` + Target string `yaml:"target"` + AuthEnabled bool `yaml:"auth_enabled"` + PrintConfig bool `yaml:"-"` + HTTPPrefix string `yaml:"http_prefix"` API api.Config `yaml:"api"` Server server.Config `yaml:"server"` @@ -108,9 +109,8 @@ type Config struct { // RegisterFlags registers flag. func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Server.MetricsNamespace = "cortex" - c.Target = All c.Server.ExcludeRequestInLog = true - f.Var(&c.Target, "target", "The Cortex service to run. Supported values are: all, distributor, ingester, querier, query-frontend, table-manager, ruler, alertmanager, configs.") + f.StringVar(&c.Target, "target", All, "The Cortex service to run. Supported values are: all, distributor, ingester, querier, query-frontend, table-manager, ruler, alertmanager, configs.") f.BoolVar(&c.AuthEnabled, "auth.enabled", true, "Set to false to disable auth.") f.BoolVar(&c.PrintConfig, "print.config", false, "Print the config and exit.") f.StringVar(&c.HTTPPrefix, "http.prefix", "/api/prom", "HTTP path prefix for Cortex API.") @@ -191,7 +191,8 @@ type Cortex struct { Cfg Config // set during initialization - ServiceMap map[ModuleName]services.Service + ServiceMap map[string]services.Service + ModuleManager *modules.Manager API *api.API Server *server.Server @@ -238,14 +239,10 @@ func New(cfg Config) (*Cortex, error) { cortex.setupAuthMiddleware() cortex.setupThanosTracing() - serviceMap, err := cortex.initModuleServices() - if err != nil { + if err := cortex.setupModuleManager(); err != nil { return nil, err } - cortex.ServiceMap = serviceMap - cortex.API.RegisterServiceMapHandler(http.HandlerFunc(cortex.servicesHandler)) - return cortex, nil } @@ -292,40 +289,16 @@ func (t *Cortex) setupThanosTracing() { ) } -func (t *Cortex) initModuleServices() (map[ModuleName]services.Service, error) { - servicesMap := map[ModuleName]services.Service{} - - // initialize all of our dependencies first - deps := orderedDeps(t.Cfg.Target) - deps = append(deps, t.Cfg.Target) // lastly, initialize the requested module - - for ix, n := range deps { - mod := modules[n] - - var serv services.Service - - if mod.wrappedService != nil { - s, err := mod.wrappedService(t) - if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("error initialising module: %s", n)) - } - if s != nil { - // We pass servicesMap, which isn't yet finished. By the time service starts, - // it will be fully built, so there is no need for extra synchronization. - serv = newModuleServiceWrapper(servicesMap, n, s, mod.deps, findInverseDependencies(n, deps[ix+1:])) - } - } - - if serv != nil { - servicesMap[n] = serv - } +// Run starts Cortex running, and blocks until a Cortex stops. +func (t *Cortex) Run() error { + serviceMap, err := t.ModuleManager.InitModuleServices(t.Cfg.Target) + if err != nil { + return err } - return servicesMap, nil -} + t.ServiceMap = serviceMap + t.API.RegisterServiceMapHandler(http.HandlerFunc(t.servicesHandler)) -// Run starts Cortex running, and blocks until a Cortex stops. -func (t *Cortex) Run() error { // get all services, create service manager and tell it to start servs := []services.Service(nil) for _, s := range t.ServiceMap { @@ -426,65 +399,3 @@ func (t *Cortex) readyHandler(sm *services.Manager) http.HandlerFunc { http.Error(w, "ready", http.StatusOK) } } - -// listDeps recursively gets a list of dependencies for a passed moduleName -func listDeps(m ModuleName) []ModuleName { - deps := modules[m].deps - for _, d := range modules[m].deps { - deps = append(deps, listDeps(d)...) - } - return deps -} - -// orderedDeps gets a list of all dependencies ordered so that items are always after any of their dependencies. -func orderedDeps(m ModuleName) []ModuleName { - deps := listDeps(m) - - // get a unique list of moduleNames, with a flag for whether they have been added to our result - uniq := map[ModuleName]bool{} - for _, dep := range deps { - uniq[dep] = false - } - - result := make([]ModuleName, 0, len(uniq)) - - // keep looping through all modules until they have all been added to the result. - - for len(result) < len(uniq) { - OUTER: - for name, added := range uniq { - if added { - continue - } - for _, dep := range modules[name].deps { - // stop processing this module if one of its dependencies has - // not been added to the result yet. - if !uniq[dep] { - continue OUTER - } - } - - // if all of the module's dependencies have been added to the result slice, - // then we can safely add this module to the result slice as well. - uniq[name] = true - result = append(result, name) - } - } - return result -} - -// find modules in the supplied list, that depend on mod -func findInverseDependencies(mod ModuleName, mods []ModuleName) []ModuleName { - result := []ModuleName(nil) - - for _, n := range mods { - for _, d := range modules[n].deps { - if d == mod { - result = append(result, n) - break - } - } - } - - return result -} diff --git a/pkg/cortex/cortex_test.go b/pkg/cortex/cortex_test.go index 589d1b44f85..d484675f9ae 100644 --- a/pkg/cortex/cortex_test.go +++ b/pkg/cortex/cortex_test.go @@ -52,19 +52,19 @@ func TestCortex(t *testing.T) { c, err := New(cfg) require.NoError(t, err) - require.NotNil(t, c.ServiceMap) - for m, s := range c.ServiceMap { + serviceMap, err := c.ModuleManager.InitModuleServices(c.Cfg.Target) + require.NoError(t, err) + require.NotNil(t, serviceMap) + + for m, s := range serviceMap { // make sure each service is still New require.Equal(t, services.New, s.State(), "module: %s", m) } // check random modules that we expect to be configured when using Target=All - require.NotNil(t, c.ServiceMap[Server]) - require.NotNil(t, c.ServiceMap[Ingester]) - require.NotNil(t, c.ServiceMap[Ring]) - require.NotNil(t, c.ServiceMap[Distributor]) - - // check that findInverseDependencie for Ring -- querier and distributor depend on Ring, so should be returned. - require.ElementsMatch(t, []ModuleName{Distributor, Querier}, findInverseDependencies(Ring, modules[cfg.Target].deps)) + require.NotNil(t, serviceMap[Server]) + require.NotNil(t, serviceMap[Ingester]) + require.NotNil(t, serviceMap[Ring]) + require.NotNil(t, serviceMap[Distributor]) } diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index e298a1f41c2..bd0b562a383 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -3,7 +3,6 @@ package cortex import ( "fmt" "os" - "strings" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -31,65 +30,38 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/modules" "github.com/cortexproject/cortex/pkg/util/runtimeconfig" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/validation" ) -// ModuleName is used to describe a running module -type ModuleName string - // The various modules that make up Cortex. const ( - API ModuleName = "api" - Ring ModuleName = "ring" - RuntimeConfig ModuleName = "runtime-config" - Overrides ModuleName = "overrides" - Server ModuleName = "server" - Distributor ModuleName = "distributor" - Ingester ModuleName = "ingester" - Flusher ModuleName = "flusher" - Querier ModuleName = "querier" - StoreQueryable ModuleName = "store-queryable" - QueryFrontend ModuleName = "query-frontend" - Store ModuleName = "store" - DeleteRequestsStore ModuleName = "delete-requests-store" - TableManager ModuleName = "table-manager" - Ruler ModuleName = "ruler" - Configs ModuleName = "configs" - AlertManager ModuleName = "alertmanager" - Compactor ModuleName = "compactor" - StoreGateway ModuleName = "store-gateway" - MemberlistKV ModuleName = "memberlist-kv" - DataPurger ModuleName = "data-purger" - All ModuleName = "all" + API string = "api" + Ring string = "ring" + RuntimeConfig string = "runtime-config" + Overrides string = "overrides" + Server string = "server" + Distributor string = "distributor" + Ingester string = "ingester" + Flusher string = "flusher" + Querier string = "querier" + StoreQueryable string = "store-queryable" + QueryFrontend string = "query-frontend" + Store string = "store" + DeleteRequestsStore string = "delete-requests-store" + TableManager string = "table-manager" + Ruler string = "ruler" + Configs string = "configs" + AlertManager string = "alertmanager" + Compactor string = "compactor" + StoreGateway string = "store-gateway" + MemberlistKV string = "memberlist-kv" + DataPurger string = "data-purger" + All string = "all" ) -func (m ModuleName) String() string { - return string(m) -} - -func (m *ModuleName) Set(s string) error { - l := ModuleName(strings.ToLower(s)) - if _, ok := modules[l]; !ok { - return fmt.Errorf("unrecognised module name: %s", s) - } - *m = l - return nil -} - -func (m ModuleName) MarshalYAML() (interface{}, error) { - return m.String(), nil -} - -func (m *ModuleName) UnmarshalYAML(unmarshal func(interface{}) error) error { - var s string - if err := unmarshal(&s); err != nil { - return err - } - return m.Set(s) -} - func (t *Cortex) initAPI() (services.Service, error) { t.Cfg.API.ServerPrefix = t.Cfg.Server.PathPrefix t.Cfg.API.LegacyHTTPPrefix = t.Cfg.HTTPPrefix @@ -515,117 +487,63 @@ func (t *Cortex) initDataPurger() (services.Service, error) { return t.DataPurger, nil } -type module struct { - deps []ModuleName +func (t *Cortex) setupModuleManager() error { + mm := modules.NewManager() + + // Register all modules here. + // RegisterModule(name string, initFn func()(services.Service, error)) + mm.RegisterModule(Server, t.initServer) + mm.RegisterModule(API, t.initAPI) + mm.RegisterModule(RuntimeConfig, t.initRuntimeConfig) + mm.RegisterModule(MemberlistKV, t.initMemberlistKV) + mm.RegisterModule(Ring, t.initRing) + mm.RegisterModule(Overrides, t.initOverrides) + mm.RegisterModule(Distributor, t.initDistributor) + mm.RegisterModule(Store, t.initStore) + mm.RegisterModule(DeleteRequestsStore, t.initDeleteRequestsStore) + mm.RegisterModule(Ingester, t.initIngester) + mm.RegisterModule(Flusher, t.initFlusher) + mm.RegisterModule(Querier, t.initQuerier) + mm.RegisterModule(StoreQueryable, t.initStoreQueryable) + mm.RegisterModule(QueryFrontend, t.initQueryFrontend) + mm.RegisterModule(TableManager, t.initTableManager) + mm.RegisterModule(Ruler, t.initRuler) + mm.RegisterModule(Configs, t.initConfig) + mm.RegisterModule(AlertManager, t.initAlertManager) + mm.RegisterModule(Compactor, t.initCompactor) + mm.RegisterModule(StoreGateway, t.initStoreGateway) + mm.RegisterModule(DataPurger, t.initDataPurger) + mm.RegisterModule(All, nil) + mm.RegisterModule(StoreGateway, t.initStoreGateway) + + // Add dependencies + deps := map[string][]string{ + API: {Server}, + Ring: {API, RuntimeConfig, MemberlistKV}, + Overrides: {RuntimeConfig}, + Distributor: {Ring, API, Overrides}, + Store: {Overrides, DeleteRequestsStore}, + Ingester: {Overrides, Store, API, RuntimeConfig, MemberlistKV}, + Flusher: {Store, API}, + Querier: {Distributor, Store, Ring, API, StoreQueryable}, + StoreQueryable: {Store}, + QueryFrontend: {API, Overrides, DeleteRequestsStore}, + TableManager: {API}, + Ruler: {Distributor, Store, StoreQueryable}, + Configs: {API}, + AlertManager: {API}, + Compactor: {API}, + StoreGateway: {API}, + DataPurger: {Store, DeleteRequestsStore, API}, + All: {QueryFrontend, Querier, Ingester, Distributor, TableManager, DataPurger, StoreGateway}, + } + for mod, targets := range deps { + if err := mm.AddDependency(mod, targets...); err != nil { + return err + } + } - // Service that will be wrapped into moduleServiceWrapper, to wait for dependencies to start / end - // (can return nil) - wrappedService func(t *Cortex) (services.Service, error) -} + t.ModuleManager = mm -var modules = map[ModuleName]module{ - Server: { - wrappedService: (*Cortex).initServer, - }, - - API: { - deps: []ModuleName{Server}, - wrappedService: (*Cortex).initAPI, - }, - - RuntimeConfig: { - wrappedService: (*Cortex).initRuntimeConfig, - }, - - MemberlistKV: { - wrappedService: (*Cortex).initMemberlistKV, - }, - - Ring: { - deps: []ModuleName{API, RuntimeConfig, MemberlistKV}, - wrappedService: (*Cortex).initRing, - }, - - Overrides: { - deps: []ModuleName{RuntimeConfig}, - wrappedService: (*Cortex).initOverrides, - }, - - Distributor: { - deps: []ModuleName{Ring, API, Overrides}, - wrappedService: (*Cortex).initDistributor, - }, - - Store: { - deps: []ModuleName{Overrides, DeleteRequestsStore}, - wrappedService: (*Cortex).initStore, - }, - - DeleteRequestsStore: { - wrappedService: (*Cortex).initDeleteRequestsStore, - }, - - Ingester: { - deps: []ModuleName{Overrides, Store, API, RuntimeConfig, MemberlistKV}, - wrappedService: (*Cortex).initIngester, - }, - - Flusher: { - deps: []ModuleName{Store, API}, - wrappedService: (*Cortex).initFlusher, - }, - - Querier: { - deps: []ModuleName{Distributor, Store, Ring, API, StoreQueryable}, - wrappedService: (*Cortex).initQuerier, - }, - - StoreQueryable: { - deps: []ModuleName{Store}, - wrappedService: (*Cortex).initStoreQueryable, - }, - - QueryFrontend: { - deps: []ModuleName{API, Overrides, DeleteRequestsStore}, - wrappedService: (*Cortex).initQueryFrontend, - }, - - TableManager: { - deps: []ModuleName{API}, - wrappedService: (*Cortex).initTableManager, - }, - - Ruler: { - deps: []ModuleName{Distributor, Store, StoreQueryable}, - wrappedService: (*Cortex).initRuler, - }, - - Configs: { - deps: []ModuleName{API}, - wrappedService: (*Cortex).initConfig, - }, - - AlertManager: { - deps: []ModuleName{API}, - wrappedService: (*Cortex).initAlertManager, - }, - - Compactor: { - deps: []ModuleName{API}, - wrappedService: (*Cortex).initCompactor, - }, - - StoreGateway: { - deps: []ModuleName{API}, - wrappedService: (*Cortex).initStoreGateway, - }, - - DataPurger: { - deps: []ModuleName{Store, DeleteRequestsStore, API}, - wrappedService: (*Cortex).initDataPurger, - }, - - All: { - deps: []ModuleName{QueryFrontend, Querier, Ingester, Distributor, TableManager, DataPurger, StoreGateway}, - }, + return nil } diff --git a/pkg/cortex/module_service_wrapper.go b/pkg/util/modules/module_service_wrapper.go similarity index 63% rename from pkg/cortex/module_service_wrapper.go rename to pkg/util/modules/module_service_wrapper.go index 2ac4cc87e53..73f1014bd58 100644 --- a/pkg/cortex/module_service_wrapper.go +++ b/pkg/util/modules/module_service_wrapper.go @@ -1,4 +1,4 @@ -package cortex +package modules import ( "github.com/cortexproject/cortex/pkg/util" @@ -7,19 +7,19 @@ import ( // This function wraps module service, and adds waiting for dependencies to start before starting, // and dependant modules to stop before stopping this module service. -func newModuleServiceWrapper(serviceMap map[ModuleName]services.Service, mod ModuleName, modServ services.Service, startDeps []ModuleName, stopDeps []ModuleName) services.Service { - getDeps := func(deps []ModuleName) map[string]services.Service { +func newModuleServiceWrapper(serviceMap map[string]services.Service, mod string, modServ services.Service, startDeps []string, stopDeps []string) services.Service { + getDeps := func(deps []string) map[string]services.Service { r := map[string]services.Service{} for _, m := range deps { s := serviceMap[m] if s != nil { - r[string(m)] = s + r[m] = s } } return r } - return util.NewModuleService(string(mod), modServ, + return util.NewModuleService(mod, modServ, func(_ string) map[string]services.Service { return getDeps(startDeps) }, diff --git a/pkg/util/modules/modules.go b/pkg/util/modules/modules.go new file mode 100644 index 00000000000..b021227c42f --- /dev/null +++ b/pkg/util/modules/modules.go @@ -0,0 +1,152 @@ +package modules + +import ( + "fmt" + + "github.com/pkg/errors" + + "github.com/cortexproject/cortex/pkg/util/services" +) + +// module is the basic building block of the application +type module struct { + // dependencies of this module + deps []string + + // initFn for this module (can return nil) + initFn func() (services.Service, error) +} + +// Manager is a component that initialises modules of the application +// in the right order of dependencies. +type Manager struct { + modules map[string]*module +} + +// NewManager creates a new Manager +func NewManager() *Manager { + return &Manager{ + modules: make(map[string]*module), + } +} + +// RegisterModule registers a new module with name and init function +// name must be unique to avoid overwriting modules +// if initFn is nil, the module will not initialise +func (m *Manager) RegisterModule(name string, initFn func() (services.Service, error)) { + m.modules[name] = &module{ + initFn: initFn, + } +} + +// AddDependency adds a dependency from name(source) to dependsOn(targets) +// An error is returned if the source module name is not found +func (m *Manager) AddDependency(name string, dependsOn ...string) error { + if mod, ok := m.modules[name]; ok { + mod.deps = append(mod.deps, dependsOn...) + } else { + return fmt.Errorf("no such module: %s", name) + } + return nil +} + +// InitModuleServices initialises the target module by initialising all its dependencies +// in the right order. Modules are wrapped in such a way that they start after their +// dependencies have been started and stop before their dependencies are stopped. +func (m *Manager) InitModuleServices(target string) (map[string]services.Service, error) { + if _, ok := m.modules[target]; !ok { + return nil, fmt.Errorf("unrecognised module name: %s", target) + } + servicesMap := map[string]services.Service{} + + // initialize all of our dependencies first + deps := m.orderedDeps(target) + deps = append(deps, target) // lastly, initialize the requested module + + for ix, n := range deps { + mod := m.modules[n] + + var serv services.Service + + if mod.initFn != nil { + s, err := mod.initFn() + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("error initialising module: %s", n)) + } + + if s != nil { + // We pass servicesMap, which isn't yet complete. By the time service starts, + // it will be fully built, so there is no need for extra synchronization. + serv = newModuleServiceWrapper(servicesMap, n, s, mod.deps, m.findInverseDependencies(n, deps[ix+1:])) + } + } + + if serv != nil { + servicesMap[n] = serv + } + } + + return servicesMap, nil +} + +// listDeps recursively gets a list of dependencies for a passed moduleName +func (m *Manager) listDeps(mod string) []string { + deps := m.modules[mod].deps + for _, d := range m.modules[mod].deps { + deps = append(deps, m.listDeps(d)...) + } + return deps +} + +// orderedDeps gets a list of all dependencies ordered so that items are always after any of their dependencies. +func (m *Manager) orderedDeps(mod string) []string { + deps := m.listDeps(mod) + + // get a unique list of moduleNames, with a flag for whether they have been added to our result + uniq := map[string]bool{} + for _, dep := range deps { + uniq[dep] = false + } + + result := make([]string, 0, len(uniq)) + + // keep looping through all modules until they have all been added to the result. + + for len(result) < len(uniq) { + OUTER: + for name, added := range uniq { + if added { + continue + } + for _, dep := range m.modules[name].deps { + // stop processing this module if one of its dependencies has + // not been added to the result yet. + if !uniq[dep] { + continue OUTER + } + } + + // if all of the module's dependencies have been added to the result slice, + // then we can safely add this module to the result slice as well. + uniq[name] = true + result = append(result, name) + } + } + return result +} + +// find modules in the supplied list, that depend on mod +func (m *Manager) findInverseDependencies(mod string, mods []string) []string { + result := []string(nil) + + for _, n := range mods { + for _, d := range m.modules[n].deps { + if d == mod { + result = append(result, n) + break + } + } + } + + return result +} diff --git a/pkg/util/modules/modules_test.go b/pkg/util/modules/modules_test.go new file mode 100644 index 00000000000..3812779c2d2 --- /dev/null +++ b/pkg/util/modules/modules_test.go @@ -0,0 +1,49 @@ +package modules + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/util/services" +) + +func mockInitFunc() (services.Service, error) { return nil, nil } + +func TestDependencies(t *testing.T) { + var testModules = map[string]module{ + "serviceA": { + initFn: mockInitFunc, + }, + + "serviceB": { + initFn: mockInitFunc, + }, + + "serviceC": { + initFn: mockInitFunc, + }, + } + + mm := NewManager() + for name, mod := range testModules { + mm.RegisterModule(name, mod.initFn) + } + assert.NoError(t, mm.AddDependency("serviceB", "serviceA")) + assert.NoError(t, mm.AddDependency("serviceC", "serviceB")) + assert.Equal(t, mm.modules["serviceB"].deps, []string{"serviceA"}) + + invDeps := mm.findInverseDependencies("serviceA", []string{"serviceB", "serviceC"}) + require.Len(t, invDeps, 1) + assert.Equal(t, invDeps[0], "serviceB") + + svcs, err := mm.InitModuleServices("serviceC") + assert.NotNil(t, svcs) + assert.NoError(t, err) + + svcs, err = mm.InitModuleServices("service_unknown") + assert.Nil(t, svcs) + assert.Error(t, err, fmt.Errorf("unrecognised module name: service_unknown")) +}