From 63b978f6062b5c5b4dcb1f241f92d83c328858d6 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Fri, 16 Aug 2019 12:05:54 +0300 Subject: [PATCH 01/13] Select database on connection --- src/redis/driver_impl.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/redis/driver_impl.go b/src/redis/driver_impl.go index eac51fbf5..92e832a65 100644 --- a/src/redis/driver_impl.go +++ b/src/redis/driver_impl.go @@ -64,9 +64,20 @@ func (this *poolImpl) Put(c Connection) { } } -func NewPoolImpl(scope stats.Scope, socketType string, url string, poolSize int) Pool { +func NewPoolImpl(scope stats.Scope, socketType string, url string, poolSize int, db int) Pool { logger.Warnf("connecting to redis on %s %s with pool size %d", socketType, url, poolSize) - pool, err := pool.New(socketType, url, poolSize) + pool, err := pool.NewCustom(socketType, url, poolSize, func(network, addr string) (*redis.Client, error) { + c, err := redis.Dial(network, addr) + if err != nil { + return nil, err + } + + if err := c.Cmd("select", db).Err; err != nil { + return nil, err + } + + return c, nil + }) checkError(err) return &poolImpl{ pool: pool, From 3fce7d072626f8f78f15fbc3f4b242b451b31b83 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Fri, 16 Aug 2019 12:06:17 +0300 Subject: [PATCH 02/13] Add database for redis connections to settings --- src/settings/settings.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/settings/settings.go b/src/settings/settings.go index 890955477..7ff42cf9d 100644 --- a/src/settings/settings.go +++ b/src/settings/settings.go @@ -21,10 +21,12 @@ type Settings struct { LogLevel string `envconfig:"LOG_LEVEL" default:"WARN"` RedisSocketType string `envconfig:"REDIS_SOCKET_TYPE" default:"unix"` RedisUrl string `envconfig:"REDIS_URL" default:"/var/run/nutcracker/ratelimit.sock"` + RedisDatabase int `envconfig:"REDIS_DB" default:0` RedisPoolSize int `envconfig:"REDIS_POOL_SIZE" default:"10"` RedisPerSecond bool `envconfig:"REDIS_PERSECOND" default:"false"` RedisPerSecondSocketType string `envconfig:"REDIS_PERSECOND_SOCKET_TYPE" default:"unix"` RedisPerSecondUrl string `envconfig:"REDIS_PERSECOND_URL" default:"/var/run/nutcracker/ratelimitpersecond.sock"` + RedisPerSecondDatabase int `envconfig:"REDIS_PERSECOND_DB" default:0` RedisPerSecondPoolSize int `envconfig:"REDIS_PERSECOND_POOL_SIZE" default:"10"` ExpirationJitterMaxSeconds int64 `envconfig:"EXPIRATION_JITTER_MAX_SECONDS" default:"300"` } From 8d35e731a51662d784c49019b982172ca0ec00ad Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Fri, 16 Aug 2019 12:19:06 +0300 Subject: [PATCH 03/13] Pass redis database args on pool initialization --- src/service_cmd/runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service_cmd/runner/runner.go b/src/service_cmd/runner/runner.go index cd8535176..1f5dfa26d 100644 --- a/src/service_cmd/runner/runner.go +++ b/src/service_cmd/runner/runner.go @@ -31,14 +31,14 @@ func Run() { var perSecondPool redis.Pool if s.RedisPerSecond { - perSecondPool = redis.NewPoolImpl(srv.Scope().Scope("redis_per_second_pool"), s.RedisPerSecondSocketType, s.RedisPerSecondUrl, s.RedisPerSecondPoolSize) + perSecondPool = redis.NewPoolImpl(srv.Scope().Scope("redis_per_second_pool"), s.RedisPerSecondSocketType, s.RedisPerSecondUrl, s.RedisPerSecondPoolSize, s.RedisPerSecondDatabase) } service := ratelimit.NewService( srv.Runtime(), redis.NewRateLimitCacheImpl( - redis.NewPoolImpl(srv.Scope().Scope("redis_pool"), s.RedisSocketType, s.RedisUrl, s.RedisPoolSize), + redis.NewPoolImpl(srv.Scope().Scope("redis_pool"), s.RedisSocketType, s.RedisUrl, s.RedisPoolSize, s.RedisDatabase), perSecondPool, redis.NewTimeSourceImpl(), rand.New(redis.NewLockedSource(time.Now().Unix())), From 939f68fefe952649f930c77a44b45983f2100d27 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Fri, 16 Aug 2019 13:08:12 +0300 Subject: [PATCH 04/13] Improve logs --- src/redis/driver_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis/driver_impl.go b/src/redis/driver_impl.go index 92e832a65..e3945a50a 100644 --- a/src/redis/driver_impl.go +++ b/src/redis/driver_impl.go @@ -65,7 +65,7 @@ func (this *poolImpl) Put(c Connection) { } func NewPoolImpl(scope stats.Scope, socketType string, url string, poolSize int, db int) Pool { - logger.Warnf("connecting to redis on %s %s with pool size %d", socketType, url, poolSize) + logger.Warnf("connecting to redis on %s %s database %d with pool size %d", socketType, url, db, poolSize) pool, err := pool.NewCustom(socketType, url, poolSize, func(network, addr string) (*redis.Client, error) { c, err := redis.Dial(network, addr) if err != nil { From 162cd3728a7e8164fb3afd716bc3327088d8fd46 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 22:34:59 +0300 Subject: [PATCH 05/13] Add database number docs --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4b263195a..8bf562863 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,8 @@ Ratelimit uses Redis as its caching layer. Ratelimit supports two operation mode 1. One Redis server for all limits. 1. Two Redis instances: one for per second limits and another one for all other limits. +1. `REDIS_DB` and `REDIS_PERSECOND_DB` could be used to configure redis database number + ## One Redis Instance To configure one Redis instance use the following environment variables: @@ -387,6 +389,7 @@ To configure one Redis instance use the following environment variables: 1. `REDIS_SOCKET_TYPE` 1. `REDIS_URL` 1. `REDIS_POOL_SIZE` +1. `REDIS_DB` This setup will use the same Redis server for all limits. @@ -397,10 +400,12 @@ To configure two Redis instances use the following environment variables: 1. `REDIS_SOCKET_TYPE` 1. `REDIS_URL` 1. `REDIS_POOL_SIZE` +1. `REDIS_DB` 1. `REDIS_PERSECOND`: set this to `"true"`. 1. `REDIS_PERSECOND_SOCKET_TYPE` 1. `REDIS_PERSECOND_URL` 1. `REDIS_PERSECOND_POOL_SIZE` +1. `REDIS_PERSECOND_DB` This setup will use the Redis server configured with the `_PERSECOND_` vars for per second limits, and the other Redis server for all other limits. From 867fcd257e3946b216593828c079ab78679bd3e7 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 23:10:18 +0300 Subject: [PATCH 06/13] Rework redis database connection --- src/redis/driver_impl.go | 41 +++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/redis/driver_impl.go b/src/redis/driver_impl.go index e3945a50a..8c94e1519 100644 --- a/src/redis/driver_impl.go +++ b/src/redis/driver_impl.go @@ -64,26 +64,41 @@ func (this *poolImpl) Put(c Connection) { } } -func NewPoolImpl(scope stats.Scope, socketType string, url string, poolSize int, db int) Pool { - logger.Warnf("connecting to redis on %s %s database %d with pool size %d", socketType, url, db, poolSize) - pool, err := pool.NewCustom(socketType, url, poolSize, func(network, addr string) (*redis.Client, error) { - c, err := redis.Dial(network, addr) - if err != nil { - return nil, err +type DialFunc func(*redis.Client) error + +func NewPoolImpl(scope stats.Scope, socketType string, url string, poolSize int, dfs ...DialFunc) Pool { + logger.Warnf("connecting to redis on %s %s with pool size %d", socketType, url, poolSize) + + var df pool.DialFunc + if len(dfs) != 0 { + df = func(network, addr string) (*redis.Client, error) { + c, err := redis.Dial(network, addr) + if err != nil { + return nil, err + } + for _, f := range dfs { + dialErr := f(c) + if dialErr != nil { + return nil, dialErr + } + } + return c, nil } - - if err := c.Cmd("select", db).Err; err != nil { - return nil, err - } - - return c, nil - }) + } + pool, err := pool.NewCustom(socketType, url, poolSize, df) checkError(err) return &poolImpl{ pool: pool, stats: newPoolStats(scope)} } +func WithDatabase(db int) DialFunc { + return func(c *redis.Client) error { + logger.Warnf("connecting to redis database %d", db) + return c.Cmd("select", db).Err + } +} + func (this *connectionImpl) PipeAppend(cmd string, args ...interface{}) { this.client.PipeAppend(cmd, args...) this.pending++ From 14602ecc3a23e2b6353fcafd3eda709da8a8b00c Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 23:10:47 +0300 Subject: [PATCH 07/13] Fix compilation errors for reworked redis connection --- src/service_cmd/runner/runner.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/service_cmd/runner/runner.go b/src/service_cmd/runner/runner.go index 1f5dfa26d..1a45239b0 100644 --- a/src/service_cmd/runner/runner.go +++ b/src/service_cmd/runner/runner.go @@ -29,16 +29,39 @@ func Run() { srv := server.NewServer("ratelimit", settings.GrpcUnaryInterceptor(nil)) + var pool redis.Pool + var dials []redis.DialFunc + if s.RedisDatabase != 0 { + dials = append(dials, redis.WithDatabase(s.RedisDatabase)) + } + pool = redis.NewPoolImpl( + srv.Scope().Scope("redis_pool"), + s.RedisSocketType, + s.RedisUrl, + s.RedisPoolSize, + dials..., + ) + var perSecondPool redis.Pool if s.RedisPerSecond { - perSecondPool = redis.NewPoolImpl(srv.Scope().Scope("redis_per_second_pool"), s.RedisPerSecondSocketType, s.RedisPerSecondUrl, s.RedisPerSecondPoolSize, s.RedisPerSecondDatabase) + var perSecondDials []redis.DialFunc + if s.RedisPerSecondDatabase != 0 { + perSecondDials = append(perSecondDials, redis.WithDatabase(s.RedisPerSecondDatabase)) + } + perSecondPool = redis.NewPoolImpl( + srv.Scope().Scope("redis_per_second_pool"), + s.RedisPerSecondSocketType, + s.RedisPerSecondUrl, + s.RedisPerSecondPoolSize, + perSecondDials..., + ) } service := ratelimit.NewService( srv.Runtime(), redis.NewRateLimitCacheImpl( - redis.NewPoolImpl(srv.Scope().Scope("redis_pool"), s.RedisSocketType, s.RedisUrl, s.RedisPoolSize, s.RedisDatabase), + pool, perSecondPool, redis.NewTimeSourceImpl(), rand.New(redis.NewLockedSource(time.Now().Unix())), From 712f7094164bf27672026489950192888c9a5f2a Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 23:26:16 +0300 Subject: [PATCH 08/13] Try to add red test --- test/integration/integration_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index c48387f3e..1bdd60141 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -44,6 +44,12 @@ func newDescriptorStatusLegacy( func TestBasicConfig(t *testing.T) { t.Run("WithoutPerSecondRedis", testBasicConfig("8083", "false")) t.Run("WithPerSecondRedis", testBasicConfig("8085", "true")) + t.Run("WithoutPerSecondRedisDbNumber", testBasicConfig("8083", "false")) + // t.Run("WithPerSecondRedisDdNumber", testBasicConfig("8083", "false")) +} + +func testDbNumber(grpcPort, perSecond string) func(*testing.T) { + return testBasicConfig("8083", "false") } func testBasicConfig(grpcPort, perSecond string) func(*testing.T) { From 0d4e4b20e31283189293431e8dd4c5692e105a75 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 23:32:22 +0300 Subject: [PATCH 09/13] Fix nil pointer --- src/redis/driver_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis/driver_impl.go b/src/redis/driver_impl.go index 8c94e1519..032560710 100644 --- a/src/redis/driver_impl.go +++ b/src/redis/driver_impl.go @@ -69,7 +69,7 @@ type DialFunc func(*redis.Client) error func NewPoolImpl(scope stats.Scope, socketType string, url string, poolSize int, dfs ...DialFunc) Pool { logger.Warnf("connecting to redis on %s %s with pool size %d", socketType, url, poolSize) - var df pool.DialFunc + df := redis.Dial if len(dfs) != 0 { df = func(network, addr string) (*redis.Client, error) { c, err := redis.Dial(network, addr) From f3974737692bd6a0c10450bcb62788113dab152c Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 23:38:56 +0300 Subject: [PATCH 10/13] Fix red test and add comments --- test/integration/integration_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 1bdd60141..94565a18a 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -44,12 +44,15 @@ func newDescriptorStatusLegacy( func TestBasicConfig(t *testing.T) { t.Run("WithoutPerSecondRedis", testBasicConfig("8083", "false")) t.Run("WithPerSecondRedis", testBasicConfig("8085", "true")) - t.Run("WithoutPerSecondRedisDbNumber", testBasicConfig("8083", "false")) - // t.Run("WithPerSecondRedisDdNumber", testBasicConfig("8083", "false")) + // Use same redis configuration + // If database number configuration doesn't work it will lead to key collisions + t.Run("WithPerSecondRedisDbNumber", testDbNumber("8085", "true")) } func testDbNumber(grpcPort, perSecond string) func(*testing.T) { - return testBasicConfig("8083", "false") + os.Setenv("REDIS_DB", "10") + os.Setenv("REDIS_PERSECOND_DB", "10") + return testBasicConfig(grpcPort, perSecond) } func testBasicConfig(grpcPort, perSecond string) func(*testing.T) { From d6a8e5dd71c589d344c72cc20fc64bcd3d11c1e4 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 23:44:24 +0300 Subject: [PATCH 11/13] Try use different port --- test/integration/integration_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 94565a18a..85b1f4008 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -46,12 +46,12 @@ func TestBasicConfig(t *testing.T) { t.Run("WithPerSecondRedis", testBasicConfig("8085", "true")) // Use same redis configuration // If database number configuration doesn't work it will lead to key collisions - t.Run("WithPerSecondRedisDbNumber", testDbNumber("8085", "true")) + t.Run("WithPerSecondRedisDbNumber", testDbNumber("8087", "true")) } func testDbNumber(grpcPort, perSecond string) func(*testing.T) { - os.Setenv("REDIS_DB", "10") - os.Setenv("REDIS_PERSECOND_DB", "10") + // os.Setenv("REDIS_DB", "10") + // os.Setenv("REDIS_PERSECOND_DB", "10") return testBasicConfig(grpcPort, perSecond) } From 577b827dc581417d39ca4a67f86d3cd3ddaee3dc Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Mon, 16 Sep 2019 23:50:32 +0300 Subject: [PATCH 12/13] Put back database numbers to fix red test --- test/integration/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 85b1f4008..073af2dad 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -50,8 +50,8 @@ func TestBasicConfig(t *testing.T) { } func testDbNumber(grpcPort, perSecond string) func(*testing.T) { - // os.Setenv("REDIS_DB", "10") - // os.Setenv("REDIS_PERSECOND_DB", "10") + os.Setenv("REDIS_DB", "10") + os.Setenv("REDIS_PERSECOND_DB", "10") return testBasicConfig(grpcPort, perSecond) } From 882c356887ffdd7a18bd071b20742da06bc93d16 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Tue, 17 Sep 2019 09:20:04 +0300 Subject: [PATCH 13/13] Move to separate test case --- test/integration/integration_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 073af2dad..899ab5d77 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -44,9 +44,13 @@ func newDescriptorStatusLegacy( func TestBasicConfig(t *testing.T) { t.Run("WithoutPerSecondRedis", testBasicConfig("8083", "false")) t.Run("WithPerSecondRedis", testBasicConfig("8085", "true")) +} + +func TestWithDbNumber(t *testing.T) { // Use same redis configuration // If database number configuration doesn't work it will lead to key collisions - t.Run("WithPerSecondRedisDbNumber", testDbNumber("8087", "true")) + t.Run("WithoutPerSecondRedisDbNumber", testDbNumber("8087", "false")) + t.Run("WithPerSecondRedisDbNumber", testDbNumber("8089", "true")) } func testDbNumber(grpcPort, perSecond string) func(*testing.T) {