From bb7dd2c84a0ec7840b55a6f28ebcd9344d118924 Mon Sep 17 00:00:00 2001 From: Yasir Ali Date: Mon, 6 Mar 2023 16:58:12 -0500 Subject: [PATCH 1/3] Adds support for odp options in config.yaml --- cmd/optimizely/main_test.go | 10 ++++++++++ cmd/optimizely/testdata/default.yaml | 3 +++ config.yaml | 6 ++++++ config/config.go | 26 ++++++++++++++++---------- config/config_test.go | 3 +++ pkg/optimizely/cache.go | 3 +++ pkg/optimizely/cache_test.go | 13 ++++++++----- 7 files changed, 49 insertions(+), 15 deletions(-) diff --git a/cmd/optimizely/main_test.go b/cmd/optimizely/main_test.go index 19652e47..2cefbf2d 100644 --- a/cmd/optimizely/main_test.go +++ b/cmd/optimizely/main_test.go @@ -64,6 +64,9 @@ func assertClient(t *testing.T, actual config.ClientConfig) { assert.Equal(t, "https://localhost/v1/%s.json", actual.DatafileURLTemplate) assert.Equal(t, "https://logx.localhost.com/v1", actual.EventURL) assert.Equal(t, "custom-regex", actual.SdkKeyRegex) + assert.True(t, actual.DisableOdp) + assert.Equal(t, 100, actual.SegmentsCacheSize) + assert.Equal(t, 1*time.Minute, actual.SegmentsCacheTimeout) assert.Equal(t, "in-memory", actual.UserProfileService["default"]) userProfileServices := map[string]interface{}{ @@ -220,6 +223,9 @@ func TestViperProps(t *testing.T) { v.Set("client.datafileURLTemplate", "https://localhost/v1/%s.json") v.Set("client.eventURL", "https://logx.localhost.com/v1") v.Set("client.sdkKeyRegex", "custom-regex") + v.Set("client.disableOdp", true) + v.Set("client.segmentsCacheSize", 100) + v.Set("client.segmentsCacheTimeout", 1*time.Minute) upsServices := map[string]interface{}{ "in-memory": map[string]interface{}{ "storageStrategy": "fifo", @@ -349,6 +355,10 @@ func TestViperEnv(t *testing.T) { _ = os.Setenv("OPTIMIZELY_CLIENT_DATAFILEURLTEMPLATE", "https://localhost/v1/%s.json") _ = os.Setenv("OPTIMIZELY_CLIENT_EVENTURL", "https://logx.localhost.com/v1") _ = os.Setenv("OPTIMIZELY_CLIENT_SDKKEYREGEX", "custom-regex") + _ = os.Setenv("OPTIMIZELY_CLIENT_DISABLEODP", "true") + _ = os.Setenv("OPTIMIZELY_CLIENT_SEGMENTSCACHESIZE", "100") + _ = os.Setenv("OPTIMIZELY_CLIENT_SEGMENTSCACHETIMEOUT", "1m") + _ = os.Setenv("OPTIMIZELY_CLIENT_USERPROFILESERVICE", `{"default":"in-memory","services":{"in-memory":{"storagestrategy":"fifo"},"redis":{"host":"localhost:6379","password":""},"rest":{"host":"http://localhost","lookuppath":"/ups/lookup","savepath":"/ups/save","headers":{"content-type":"application/json"},"async":true},"custom":{"path":"http://test2.com"}}}`) _ = os.Setenv("OPTIMIZELY_CLIENT_ODPCACHE", `{"default":"in-memory","services":{"in-memory":{"size":100,"timeout":5},"redis":{"host":"localhost:6379","password":""},"custom":{"path":"http://test2.com"}}}`) diff --git a/cmd/optimizely/testdata/default.yaml b/cmd/optimizely/testdata/default.yaml index e50f83e9..5e1e701d 100644 --- a/cmd/optimizely/testdata/default.yaml +++ b/cmd/optimizely/testdata/default.yaml @@ -32,6 +32,9 @@ client: datafileURLTemplate: "https://localhost/v1/%s.json" eventURL: "https://logx.localhost.com/v1" sdkKeyRegex: "custom-regex" + disableOdp: true + segmentsCacheSize: 100 + segmentsCacheTimeout: 1m userProfileService: default: "in-memory" services: diff --git a/config.yaml b/config.yaml index 4adbe614..7ea28a20 100644 --- a/config.yaml +++ b/config.yaml @@ -135,6 +135,12 @@ client: ## By default Agent assumes only alphanumeric characters as part of the SDK Key string. ## https://github.com/google/re2/wiki/Syntax sdkKeyRegex: "^\\w+(:\\w+)?$" + ## Disable odp + disableOdp: false + ## Max number of items that can be stored in odp segments cache. + segmentsCacheSize: 10000 + ## Timeout in seconds after which an item stored in odp segments cache will be deleted. + segmentsCacheTimeout: 600s ## configure optional User profile service userProfileService: default: "" diff --git a/config/config.go b/config/config.go index 3ac66fe9..39ec3d72 100644 --- a/config/config.go +++ b/config/config.go @@ -78,7 +78,10 @@ func NewDefaultConfig() *AgentConfig { DatafileURLTemplate: "https://cdn.optimizely.com/datafiles/%s.json", EventURL: "https://logx.optimizely.com/v1/events", // https://github.com/google/re2/wiki/Syntax - SdkKeyRegex: "^\\w+(:\\w+)?$", + SdkKeyRegex: "^\\w+(:\\w+)?$", + DisableOdp: false, + SegmentsCacheSize: 10000, + SegmentsCacheTimeout: 600 * time.Second, UserProfileService: UserProfileServiceConfigs{ "default": "", "services": map[string]interface{}{}, @@ -162,15 +165,18 @@ type ODPCacheConfigs map[string]interface{} // ClientConfig holds the configuration options for the Optimizely Client. type ClientConfig struct { - PollingInterval time.Duration `json:"pollingInterval"` - BatchSize int `json:"batchSize" default:"10"` - QueueSize int `json:"queueSize" default:"1000"` - FlushInterval time.Duration `json:"flushInterval" default:"30s"` - DatafileURLTemplate string `json:"datafileURLTemplate"` - EventURL string `json:"eventURL"` - SdkKeyRegex string `json:"sdkKeyRegex"` - UserProfileService UserProfileServiceConfigs `json:"userProfileService"` - ODPCache ODPCacheConfigs `json:"odpCache"` + PollingInterval time.Duration `json:"pollingInterval"` + BatchSize int `json:"batchSize" default:"10"` + QueueSize int `json:"queueSize" default:"1000"` + FlushInterval time.Duration `json:"flushInterval" default:"30s"` + DatafileURLTemplate string `json:"datafileURLTemplate"` + EventURL string `json:"eventURL"` + SdkKeyRegex string `json:"sdkKeyRegex"` + DisableOdp bool `json:"disableOdp"` + SegmentsCacheSize int `json:"segmentsCacheSize"` + SegmentsCacheTimeout time.Duration `json:"segmentsCacheTimeout"` + UserProfileService UserProfileServiceConfigs `json:"userProfileService"` + ODPCache ODPCacheConfigs `json:"odpCache"` } // LogConfig holds the log configuration diff --git a/config/config_test.go b/config/config_test.go index fb2bc84b..5f018b23 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -83,6 +83,9 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, "https://logx.optimizely.com/v1/events", conf.Client.EventURL) assert.Equal(t, "^\\w+(:\\w+)?$", conf.Client.SdkKeyRegex) assert.Equal(t, "", conf.Client.UserProfileService["default"]) + assert.Equal(t, false, conf.Client.DisableOdp) + assert.Equal(t, 10000, conf.Client.SegmentsCacheSize) + assert.Equal(t, 600*time.Second, conf.Client.SegmentsCacheTimeout) assert.Equal(t, map[string]interface{}{}, conf.Client.UserProfileService["services"]) assert.Equal(t, "", conf.Client.ODPCache["default"]) assert.Equal(t, map[string]interface{}{}, conf.Client.ODPCache["services"]) diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 77991814..262d59d8 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -242,6 +242,9 @@ func defaultLoader( client.WithConfigManager(configManager), client.WithExperimentOverrides(forcedVariations), client.WithEventProcessor(ep), + client.WithOdpDisabled(conf.DisableOdp), + client.WithSegmentsCacheSize(conf.SegmentsCacheSize), + client.WithSegmentsCacheTimeout(conf.SegmentsCacheTimeout), } var clientUserProfileService decision.UserProfileService diff --git a/pkg/optimizely/cache_test.go b/pkg/optimizely/cache_test.go index 736322fd..14f298aa 100644 --- a/pkg/optimizely/cache_test.go +++ b/pkg/optimizely/cache_test.go @@ -385,11 +385,14 @@ func (s *DefaultLoaderTestSuite) SetupTest() { func (s *DefaultLoaderTestSuite) TestDefaultLoader() { conf := config.ClientConfig{ - FlushInterval: 321 * time.Second, - BatchSize: 1234, - QueueSize: 5678, - EventURL: "https://localhost/events", - SdkKeyRegex: "sdkkey", + FlushInterval: 321 * time.Second, + BatchSize: 1234, + QueueSize: 5678, + EventURL: "https://localhost/events", + SdkKeyRegex: "sdkkey", + DisableOdp: true, + SegmentsCacheTimeout: 5 * time.Minute, + SegmentsCacheSize: 100, UserProfileService: map[string]interface{}{"default": "in-memory", "services": map[string]interface{}{ "in-memory": map[string]interface{}{ "capacity": 0, From e044e10837861c3f77ba7ac3e0b05849c466f796 Mon Sep 17 00:00:00 2001 From: Yasir Ali Date: Mon, 6 Mar 2023 20:09:23 -0500 Subject: [PATCH 2/3] Added a message. --- config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.yaml b/config.yaml index 4917c8d5..ec9bba67 100644 --- a/config.yaml +++ b/config.yaml @@ -164,6 +164,7 @@ client: segmentsCacheSize: 10000 ## Timeout in seconds after which an item stored in odp segments cache will be deleted. segmentsCacheTimeout: 600s + ## This will override all other settings cache: default: "" services: From 3fb5d3f4cce7bf241a3230f20f261cac9c2c8486 Mon Sep 17 00:00:00 2001 From: Yasir Ali Date: Tue, 7 Mar 2023 14:06:35 -0500 Subject: [PATCH 3/3] Suggested changes made. --- cmd/optimizely/main_test.go | 17 +++++++++--- cmd/optimizely/testdata/default.yaml | 3 +++ config.yaml | 6 +++++ config/config.go | 20 +++++++++----- config/config_test.go | 3 +++ pkg/optimizely/cache.go | 40 ++++++++++++++++++++++------ pkg/optimizely/cache_test.go | 9 ++++--- 7 files changed, 76 insertions(+), 22 deletions(-) diff --git a/cmd/optimizely/main_test.go b/cmd/optimizely/main_test.go index fb1f4d3b..8928982b 100644 --- a/cmd/optimizely/main_test.go +++ b/cmd/optimizely/main_test.go @@ -65,8 +65,11 @@ func assertClient(t *testing.T, actual config.ClientConfig) { assert.Equal(t, "https://logx.localhost.com/v1", actual.EventURL) assert.Equal(t, "custom-regex", actual.SdkKeyRegex) assert.True(t, actual.ODP.Disable) + assert.Equal(t, 5*time.Second, actual.ODP.EventsFlushInterval) + assert.Equal(t, 5*time.Second, actual.ODP.EventsRequestTimeout) assert.Equal(t, 100, actual.ODP.SegmentsCacheSize) assert.Equal(t, 1*time.Minute, actual.ODP.SegmentsCacheTimeout) + assert.Equal(t, 5*time.Second, actual.ODP.SegmentsRequestTimeout) assert.Equal(t, "in-memory", actual.UserProfileService["default"]) userProfileServices := map[string]interface{}{ @@ -266,10 +269,13 @@ func TestViperProps(t *testing.T) { "services": odpCacheServices, } odpConfig := map[string]interface{}{ - "cache": odpCache, - "disable": true, - "segmentsCacheSize": 100, - "segmentsCacheTimeout": 1 * time.Minute, + "disable": true, + "eventsRequestTimeout": 5 * time.Second, + "eventsFlushInterval": 5 * time.Second, + "segmentsCacheSize": 100, + "segmentsCacheTimeout": 1 * time.Minute, + "segmentsRequestTimeout": 5 * time.Second, + "cache": odpCache, } v.Set("client.odp", odpConfig) v.Set("log.pretty", true) @@ -361,8 +367,11 @@ func TestViperEnv(t *testing.T) { _ = os.Setenv("OPTIMIZELY_CLIENT_USERPROFILESERVICE", `{"default":"in-memory","services":{"in-memory":{"storagestrategy":"fifo"},"redis":{"host":"localhost:6379","password":""},"rest":{"host":"http://localhost","lookuppath":"/ups/lookup","savepath":"/ups/save","headers":{"content-type":"application/json"},"async":true},"custom":{"path":"http://test2.com"}}}`) _ = os.Setenv("OPTIMIZELY_CLIENT_ODP_CACHE", `{"default":"in-memory","services":{"in-memory":{"size":100,"timeout":5},"redis":{"host":"localhost:6379","password":""},"custom":{"path":"http://test2.com"}}}`) _ = os.Setenv("OPTIMIZELY_CLIENT_ODP_DISABLE", `true`) + _ = os.Setenv("OPTIMIZELY_CLIENT_ODP_EVENTSREQUESTTIMEOUT", `5s`) + _ = os.Setenv("OPTIMIZELY_CLIENT_ODP_EVENTSFLUSHINTERVAL", `5s`) _ = os.Setenv("OPTIMIZELY_CLIENT_ODP_SEGMENTSCACHESIZE", `100`) _ = os.Setenv("OPTIMIZELY_CLIENT_ODP_SEGMENTSCACHETIMEOUT", `1m`) + _ = os.Setenv("OPTIMIZELY_CLIENT_ODP_SEGMENTSREQUESTTIMEOUT", `5s`) _ = os.Setenv("OPTIMIZELY_LOG_PRETTY", "true") _ = os.Setenv("OPTIMIZELY_LOG_INCLUDESDKKEY", "false") diff --git a/cmd/optimizely/testdata/default.yaml b/cmd/optimizely/testdata/default.yaml index 2db247dc..3cd43c53 100644 --- a/cmd/optimizely/testdata/default.yaml +++ b/cmd/optimizely/testdata/default.yaml @@ -51,8 +51,11 @@ client: path: "http://test2.com" odp: disable: true + eventsRequestTimeout: 5s + eventsFlushInterval: 5s segmentsCacheSize: 100 segmentsCacheTimeout: 1m + segmentsRequestTimeout: 5s cache: default: "in-memory" services: diff --git a/config.yaml b/config.yaml index ec9bba67..9ac4e090 100644 --- a/config.yaml +++ b/config.yaml @@ -160,10 +160,16 @@ client: odp: ## Disable odp disable: false + ## Timeout in seconds after which event requests will timeout. + eventsRequestTimeout: 10s + ## Flush interval in seconds for odp events + eventsFlushInterval: 1s ## Max number of items that can be stored in odp segments cache. segmentsCacheSize: 10000 ## Timeout in seconds after which an item stored in odp segments cache will be deleted. segmentsCacheTimeout: 600s + ## Timeout in seconds after which segment requests will timeout. + segmentsRequestTimeout: 10s ## This will override all other settings cache: default: "" diff --git a/config/config.go b/config/config.go index c5ac9591..845d147f 100644 --- a/config/config.go +++ b/config/config.go @@ -84,13 +84,16 @@ func NewDefaultConfig() *AgentConfig { "services": map[string]interface{}{}, }, ODP: OdpConfig{ + Disable: false, + EventsRequestTimeout: 10 * time.Second, + EventsFlushInterval: 1 * time.Second, + SegmentsCacheSize: 10000, + SegmentsCacheTimeout: 600 * time.Second, + SegmentsRequestTimeout: 10 * time.Second, Cache: ODPCacheConfigs{ "default": "", "services": map[string]interface{}{}, }, - Disable: false, - SegmentsCacheSize: 10000, - SegmentsCacheTimeout: 600 * time.Second, }, }, Runtime: RuntimeConfig{ @@ -180,10 +183,13 @@ type ClientConfig struct { // OdpConfig holds the odp configuration type OdpConfig struct { - Cache ODPCacheConfigs `json:"cache"` - Disable bool `json:"disable"` - SegmentsCacheSize int `json:"segmentsCacheSize"` - SegmentsCacheTimeout time.Duration `json:"segmentsCacheTimeout"` + EventsRequestTimeout time.Duration `json:"eventsRequestTimeout"` + EventsFlushInterval time.Duration `json:"eventsFlushInterval"` + Disable bool `json:"disable"` + SegmentsCacheSize int `json:"segmentsCacheSize"` + SegmentsCacheTimeout time.Duration `json:"segmentsCacheTimeout"` + SegmentsRequestTimeout time.Duration `json:"segmentsRequestTimeout"` + Cache ODPCacheConfigs `json:"cache"` } // LogConfig holds the log configuration diff --git a/config/config_test.go b/config/config_test.go index f0d49c91..6192c17b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -84,8 +84,11 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, "^\\w+(:\\w+)?$", conf.Client.SdkKeyRegex) assert.Equal(t, "", conf.Client.UserProfileService["default"]) assert.Equal(t, false, conf.Client.ODP.Disable) + assert.Equal(t, 1*time.Second, conf.Client.ODP.EventsFlushInterval) + assert.Equal(t, 10*time.Second, conf.Client.ODP.EventsRequestTimeout) assert.Equal(t, 10000, conf.Client.ODP.SegmentsCacheSize) assert.Equal(t, 600*time.Second, conf.Client.ODP.SegmentsCacheTimeout) + assert.Equal(t, 10*time.Second, conf.Client.ODP.SegmentsRequestTimeout) assert.Equal(t, map[string]interface{}{}, conf.Client.UserProfileService["services"]) assert.Equal(t, "", conf.Client.ODP.Cache["default"]) assert.Equal(t, map[string]interface{}{}, conf.Client.ODP.Cache["services"]) diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 98e5512c..4c4fbd8e 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -33,7 +33,9 @@ import ( "github.com/optimizely/go-sdk/pkg/decision" "github.com/optimizely/go-sdk/pkg/event" "github.com/optimizely/go-sdk/pkg/odp" + odpEventPkg "github.com/optimizely/go-sdk/pkg/odp/event" odpSegmentPkg "github.com/optimizely/go-sdk/pkg/odp/segment" + "github.com/optimizely/go-sdk/pkg/utils" odpCachePkg "github.com/optimizely/go-sdk/pkg/odp/cache" cmap "github.com/orcaman/concurrent-map" @@ -243,8 +245,6 @@ func defaultLoader( client.WithExperimentOverrides(forcedVariations), client.WithEventProcessor(ep), client.WithOdpDisabled(conf.ODP.Disable), - client.WithSegmentsCacheSize(conf.ODP.SegmentsCacheSize), - client.WithSegmentsCacheTimeout(conf.ODP.SegmentsCacheTimeout), } var clientUserProfileService decision.UserProfileService @@ -265,15 +265,39 @@ func defaultLoader( // convert odpCache to Cache interface if convertedODPCache, ok := rawODPCache.(odpCachePkg.Cache); ok && convertedODPCache != nil { clientODPCache = convertedODPCache - odpSegmentOptions := []odpSegmentPkg.SMOptionFunc{odpSegmentPkg.WithSegmentsCache(clientODPCache)} - segmentManager := odpSegmentPkg.NewSegmentManager(sdkKey, odpSegmentOptions...) - odpOptions := []odp.OMOptionFunc{odp.WithSegmentManager(segmentManager)} - odpManager := odp.NewOdpManager(sdkKey, - false, odpOptions...) - clientOptions = append(clientOptions, client.WithOdpManager(odpManager)) } } + // Create segment manager with odpConfig and custom cache + segmentManager := odpSegmentPkg.NewSegmentManager( + sdkKey, + odpSegmentPkg.WithAPIManager( + odpSegmentPkg.NewSegmentAPIManager(sdkKey, utils.NewHTTPRequester(nil, utils.Timeout(conf.ODP.SegmentsRequestTimeout))), + ), + odpSegmentPkg.WithSegmentsCacheSize(conf.ODP.SegmentsCacheSize), + odpSegmentPkg.WithSegmentsCacheTimeout(conf.ODP.SegmentsCacheTimeout), + odpSegmentPkg.WithSegmentsCache(clientODPCache), + ) + + // Create event manager with odpConfig + eventManager := odpEventPkg.NewBatchEventManager( + odpEventPkg.WithAPIManager( + odpEventPkg.NewEventAPIManager( + sdkKey, utils.NewHTTPRequester(nil, utils.Timeout(conf.ODP.EventsRequestTimeout)), + ), + ), + odpEventPkg.WithFlushInterval(conf.ODP.EventsFlushInterval), + ) + + // Create odp manager with custom segment and event manager + odpManager := odp.NewOdpManager( + sdkKey, + conf.ODP.Disable, + odp.WithSegmentManager(segmentManager), + odp.WithEventManager(eventManager), + ) + clientOptions = append(clientOptions, client.WithOdpManager(odpManager)) + optimizelyClient, err := optimizelyFactory.Client( clientOptions..., ) diff --git a/pkg/optimizely/cache_test.go b/pkg/optimizely/cache_test.go index 157a856a..8813b734 100644 --- a/pkg/optimizely/cache_test.go +++ b/pkg/optimizely/cache_test.go @@ -407,15 +407,18 @@ func (s *DefaultLoaderTestSuite) TestDefaultLoader() { }}, }, ODP: config.OdpConfig{ + EventsRequestTimeout: 10 * time.Second, + EventsFlushInterval: 1 * time.Second, Cache: map[string]interface{}{"default": "in-memory", "services": map[string]interface{}{ "in-memory": map[string]interface{}{ "size": 100, "timeout": 5, }}, }, - Disable: true, - SegmentsCacheTimeout: 5 * time.Minute, - SegmentsCacheSize: 100, + Disable: true, + SegmentsCacheTimeout: 5 * time.Minute, + SegmentsCacheSize: 100, + SegmentsRequestTimeout: 10 * time.Second, }, }