From 1bb82f6995532d153d19a67b10b76865a14f3b2f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 12 Apr 2020 20:00:10 -0500 Subject: [PATCH 01/23] Add SentinelConnect and SentinelMasterConnect to ConnectionMultiplexer for working with sentinel setups (#1427) Fix issue with duplicate endpoints being added in the UpdateSentinelAddressList method (#1430). Add string configuration overloads for sentinel connect methods. Remove password from sentinel servers as it seems the windows port does not support it. Add some new tests. --- StackExchange.Redis.sln | 9 ++ .../ConnectionMultiplexer.cs | 64 +++++++- src/StackExchange.Redis/EndPointCollection.cs | 20 +++ .../RedisConfigs/Docker/docker-entrypoint.sh | 2 +- tests/RedisConfigs/Sentinel/redis-7010.conf | 2 + tests/RedisConfigs/Sentinel/redis-7011.conf | 4 +- .../RedisConfigs/Sentinel/sentinel-26379.conf | 2 + .../RedisConfigs/Sentinel/sentinel-26380.conf | 2 + .../RedisConfigs/Sentinel/sentinel-26381.conf | 4 +- tests/StackExchange.Redis.Tests/Sentinel.cs | 138 +++++++++++++++--- 10 files changed, 224 insertions(+), 23 deletions(-) diff --git a/StackExchange.Redis.sln b/StackExchange.Redis.sln index 23908aebb..cd4d9b060 100644 --- a/StackExchange.Redis.sln +++ b/StackExchange.Redis.sln @@ -23,6 +23,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "RedisConfigs", "RedisConfig tests\RedisConfigs\cli-master.cmd = tests\RedisConfigs\cli-master.cmd tests\RedisConfigs\cli-secure.cmd = tests\RedisConfigs\cli-secure.cmd tests\RedisConfigs\cli-slave.cmd = tests\RedisConfigs\cli-slave.cmd + tests\RedisConfigs\docker-compose.yml = tests\RedisConfigs\docker-compose.yml + tests\RedisConfigs\Dockerfile = tests\RedisConfigs\Dockerfile tests\RedisConfigs\start-all.cmd = tests\RedisConfigs\start-all.cmd tests\RedisConfigs\start-all.sh = tests\RedisConfigs\start-all.sh tests\RedisConfigs\start-basic.cmd = tests\RedisConfigs\start-basic.cmd @@ -126,6 +128,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestConsoleBaseline", "toys EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = ".github", ".github\.github.csproj", "{8FB98E7D-DAE2-4465-BD9A-104000E0A2D4}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docker", "Docker", "{A9F81DA3-DA82-423E-A5DD-B11C37548E06}" + ProjectSection(SolutionItems) = preProject + tests\RedisConfigs\Docker\docker-entrypoint.sh = tests\RedisConfigs\Docker\docker-entrypoint.sh + tests\RedisConfigs\Docker\supervisord.conf = tests\RedisConfigs\Docker\supervisord.conf + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -197,6 +205,7 @@ Global {3DA1EEED-E9FE-43D9-B293-E000CFCCD91A} = {E25031D3-5C64-430D-B86F-697B66816FD8} {153A10E4-E668-41AD-9E0F-6785CE7EED66} = {3AD17044-6BFF-4750-9AC2-2CA466375F2A} {D58114AE-4998-4647-AFCA-9353D20495AE} = {E25031D3-5C64-430D-B86F-697B66816FD8} + {A9F81DA3-DA82-423E-A5DD-B11C37548E06} = {96E891CD-2ED7-4293-A7AB-4C6F5D8D2B05} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {193AA352-6748-47C1-A5FC-C9AA6B5F000B} diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index f8e4dd7d4..d5dbfb819 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -994,7 +994,7 @@ public static ConnectionMultiplexer Connect(string configuration, TextWriter log /// /// Create a new ConnectionMultiplexer instance /// - /// The configurtion options to use for this multiplexer. + /// The configuration options to use for this multiplexer. /// The to log to. public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, TextWriter log = null) { @@ -1002,6 +1002,66 @@ public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, return ConnectImpl(configuration, log); } + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server + /// + /// The string configuration to use for this multiplexer. + /// The to log to. + public static ConnectionMultiplexer SentinelConnect(string configuration, TextWriter log = null) + { + var options = ConfigurationOptions.Parse(configuration); + return SentinelConnect(options); + } + + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server + /// + /// The configuration options to use for this multiplexer. + /// The to log to. + public static ConnectionMultiplexer SentinelConnect(ConfigurationOptions configuration, TextWriter log = null) + { + if (string.IsNullOrEmpty(configuration.ServiceName)) + throw new ArgumentException("A ServiceName must be specified."); + + var sentinelConfigurationOptions = configuration.Clone(); + + // this is required when connecting to sentinel servers + sentinelConfigurationOptions.TieBreaker = ""; + sentinelConfigurationOptions.CommandMap = CommandMap.Sentinel; + + // use default sentinel port + sentinelConfigurationOptions.EndPoints.SetDefaultPorts(26379); + + return Connect(sentinelConfigurationOptions, log); + } + + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server, discovers the current master server + /// for the specified ServiceName in the config and returns a managed connection to the current master server + /// + /// The string configuration to use for this multiplexer. + /// The to log to. + public static ConnectionMultiplexer SentinelMasterConnect(string configuration, TextWriter log = null) + { + var options = ConfigurationOptions.Parse(configuration); + var sentinelConnection = SentinelConnect(options, log); + + return sentinelConnection.GetSentinelMasterConnection(options, log); + } + + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server, discovers the current master server + /// for the specified ServiceName in the config and returns a managed connection to the current master server + /// + /// The configuration options to use for this multiplexer. + /// The to log to. + public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions configuration, TextWriter log = null) + { + var sentinelConnection = SentinelConnect(configuration, log); + + return sentinelConnection.GetSentinelMasterConnection(configuration, log); + } + private static ConnectionMultiplexer ConnectImpl(object configuration, TextWriter log) { IDisposable killMe = null; @@ -2400,7 +2460,7 @@ internal void UpdateSentinelAddressList(string serviceName) foreach (EndPoint newSentinel in firstCompleteRequest.Where(x => !RawConfig.EndPoints.Contains(x))) { hasNew = true; - RawConfig.EndPoints.Add(newSentinel); + RawConfig.EndPoints.TryAdd(newSentinel); } if (hasNew) diff --git a/src/StackExchange.Redis/EndPointCollection.cs b/src/StackExchange.Redis/EndPointCollection.cs index 8a9ae28d9..8b25930ad 100644 --- a/src/StackExchange.Redis/EndPointCollection.cs +++ b/src/StackExchange.Redis/EndPointCollection.cs @@ -59,6 +59,26 @@ public void Add(string hostAndPort) /// The port for to add. public void Add(IPAddress host, int port) => Add(new IPEndPoint(host, port)); + /// + /// Try adding a new endpoint to the list. + /// + /// The endpoint to add. + /// True if the endpoint was added or false if not. + public bool TryAdd(EndPoint endpoint) + { + if (endpoint == null) throw new ArgumentNullException(nameof(endpoint)); + + if (!Contains(endpoint)) + { + base.InsertItem(Count, endpoint); + return true; + } + else + { + return false; + } + } + /// /// See Collection<T>.InsertItem() /// diff --git a/tests/RedisConfigs/Docker/docker-entrypoint.sh b/tests/RedisConfigs/Docker/docker-entrypoint.sh index 5ab35b2c8..6e925d375 100644 --- a/tests/RedisConfigs/Docker/docker-entrypoint.sh +++ b/tests/RedisConfigs/Docker/docker-entrypoint.sh @@ -4,7 +4,7 @@ if [ "$#" -ne 0 ]; then exec "$@" else mkdir -p /var/log/supervisor - mkdir Temp/ + mkdir -p Temp/ supervisord -c /etc/supervisord.conf sleep 3 diff --git a/tests/RedisConfigs/Sentinel/redis-7010.conf b/tests/RedisConfigs/Sentinel/redis-7010.conf index 0e27680b2..807682537 100644 --- a/tests/RedisConfigs/Sentinel/redis-7010.conf +++ b/tests/RedisConfigs/Sentinel/redis-7010.conf @@ -1,4 +1,6 @@ port 7010 +#requirepass changeme +#masterauth changeme repl-diskless-sync yes repl-diskless-sync-delay 0 maxmemory 100mb diff --git a/tests/RedisConfigs/Sentinel/redis-7011.conf b/tests/RedisConfigs/Sentinel/redis-7011.conf index 6d02eb150..382247b75 100644 --- a/tests/RedisConfigs/Sentinel/redis-7011.conf +++ b/tests/RedisConfigs/Sentinel/redis-7011.conf @@ -1,9 +1,11 @@ port 7011 +#requirepass changeme slaveof 127.0.0.1 7010 +#masterauth changeme repl-diskless-sync yes repl-diskless-sync-delay 0 maxmemory 100mb appendonly no dir "../Temp" dbfilename "sentinel-target-7011.rdb" -save "" \ No newline at end of file +save "" diff --git a/tests/RedisConfigs/Sentinel/sentinel-26379.conf b/tests/RedisConfigs/Sentinel/sentinel-26379.conf index 6d10f6030..4a63fab9d 100644 --- a/tests/RedisConfigs/Sentinel/sentinel-26379.conf +++ b/tests/RedisConfigs/Sentinel/sentinel-26379.conf @@ -1,5 +1,7 @@ port 26379 +#requirepass "changeme" sentinel monitor mymaster 127.0.0.1 7010 1 +#sentinel auth-pass mymaster changeme sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 1000 sentinel config-epoch mymaster 0 diff --git a/tests/RedisConfigs/Sentinel/sentinel-26380.conf b/tests/RedisConfigs/Sentinel/sentinel-26380.conf index fa044227e..57f544c8f 100644 --- a/tests/RedisConfigs/Sentinel/sentinel-26380.conf +++ b/tests/RedisConfigs/Sentinel/sentinel-26380.conf @@ -1,5 +1,7 @@ port 26380 +#requirepass "changeme" sentinel monitor mymaster 127.0.0.1 7010 1 +#sentinel auth-pass mymaster changeme sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 1000 sentinel config-epoch mymaster 0 diff --git a/tests/RedisConfigs/Sentinel/sentinel-26381.conf b/tests/RedisConfigs/Sentinel/sentinel-26381.conf index fa49c9e14..e8159bcdb 100644 --- a/tests/RedisConfigs/Sentinel/sentinel-26381.conf +++ b/tests/RedisConfigs/Sentinel/sentinel-26381.conf @@ -1,6 +1,8 @@ port 26381 +#requirepass "changeme" sentinel monitor mymaster 127.0.0.1 7010 1 +#sentinel auth-pass mymaster changeme sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 1000 sentinel config-epoch mymaster 0 -dir "../Temp" +dir "../Temp" \ No newline at end of file diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index e3613babc..dcf229cb4 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Net; @@ -12,7 +12,7 @@ namespace StackExchange.Redis.Tests public class Sentinel : TestBase { private string ServiceName => TestConfig.Current.SentinelSeviceName; - private ConfigurationOptions ServiceOptions => new ConfigurationOptions { ServiceName = ServiceName, AllowAdmin = true }; + private ConfigurationOptions ServiceOptions => new ConfigurationOptions { ServiceName = ServiceName, AllowAdmin = true, Password = "changeme" }; private ConnectionMultiplexer Conn { get; } private IServer SentinelServerA { get; } @@ -28,24 +28,16 @@ public Sentinel(ITestOutputHelper output) : base(output) Skip.IfNoConfig(nameof(TestConfig.Config.SentinelServer), TestConfig.Current.SentinelServer); Skip.IfNoConfig(nameof(TestConfig.Config.SentinelSeviceName), TestConfig.Current.SentinelSeviceName); - var options = new ConfigurationOptions() - { - CommandMap = CommandMap.Sentinel, - EndPoints = { - { TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA }, - { TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortB }, - { TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortC } - }, - AllowAdmin = true, - TieBreaker = "", - ServiceName = TestConfig.Current.SentinelSeviceName, - SyncTimeout = 5000 - }; - Conn = ConnectionMultiplexer.Connect(options, ConnectionLog); + var options = ServiceOptions.Clone(); + options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); + options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortB); + options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortC); + + Conn = ConnectionMultiplexer.SentinelConnect(options, ConnectionLog); for (var i = 0; i < 150; i++) { Thread.Sleep(20); - if (Conn.IsConnected && Conn.GetSentinelMasterConnection(ServiceOptions).IsConnected) + if (Conn.IsConnected && Conn.GetSentinelMasterConnection(options).IsConnected) { break; } @@ -57,6 +49,84 @@ public Sentinel(ITestOutputHelper output) : base(output) SentinelsServers = new IServer[] { SentinelServerA, SentinelServerB, SentinelServerC }; } + [Fact] + public void MasterConnectTest() + { + var options = ServiceOptions.Clone(); + options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); + + var conn = ConnectionMultiplexer.SentinelMasterConnect(options); + var db = conn.GetDatabase(); + + var test = db.Ping(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + } + + [Fact] + public void MasterConnectWithDefaultPortTest() + { + var options = ServiceOptions.Clone(); + options.EndPoints.Add(TestConfig.Current.SentinelServer); + + var conn = ConnectionMultiplexer.SentinelMasterConnect(options); + var db = conn.GetDatabase(); + + var test = db.Ping(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + } + + [Fact] + public void MasterConnectWithStringConfigurationTest() + { + var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName}"; + var conn = ConnectionMultiplexer.SentinelMasterConnect(connectionString); + var db = conn.GetDatabase(); + + var test = db.Ping(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + } + + [Fact] + public async Task MasterConnectFailoverTest() + { + var options = ServiceOptions.Clone(); + options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); + + // connection is managed and should switch to current master when failover happens + var conn = ConnectionMultiplexer.SentinelMasterConnect(options); + conn.ConfigurationChanged += (s, e) => { + Log($"Configuration changed: {e.EndPoint}"); + }; + var db = conn.GetDatabase(); + + var test = await db.PingAsync(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + + // set string value on current master + var expected = DateTime.Now.Ticks.ToString(); + Log("Tick Key: " + expected); + var key = Me(); + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + await db.StringSetAsync(key, expected); + + // forces and verifies failover + await DoFailoverAsync(); + + var value = await db.StringGetAsync(key); + Assert.Equal(expected, value); + + await db.StringSetAsync(key, expected); + } + + private void Conn_ConfigurationChanged(object sender, EndPointEventArgs e) + { + throw new NotImplementedException(); + } + [Fact] public void PingTest() { @@ -584,7 +654,8 @@ public async Task ReadOnlyConnectionSlavesTest() var config = new ConfigurationOptions { TieBreaker = "", - ServiceName = TestConfig.Current.SentinelSeviceName, + ServiceName = ServiceOptions.ServiceName, + Password = ServiceOptions.Password }; foreach (var kv in slaves) @@ -604,5 +675,36 @@ public async Task ReadOnlyConnectionSlavesTest() //Assert.StartsWith("No connection is available to service this operation", ex.Message); } + + private async Task DoFailoverAsync() + { + // capture current master and slave + var master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); + var slaves = SentinelServerA.SentinelSlaves(ServiceName); + + await Task.Delay(1000).ForAwait(); + try + { + Log("Failover attempted initiated"); + SentinelServerA.SentinelFailover(ServiceName); + Log(" Success!"); + } + catch (RedisServerException ex) when (ex.Message.Contains("NOGOODSLAVE")) + { + // Retry once + Log(" Retry initiated"); + await Task.Delay(1000).ForAwait(); + SentinelServerA.SentinelFailover(ServiceName); + Log(" Retry complete"); + } + await Task.Delay(2000).ForAwait(); + + var newMaster = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); + var newSlave = SentinelServerA.SentinelSlaves(ServiceName); + + // make sure master changed + Assert.Equal(slaves[0].ToDictionary()["name"], newMaster.ToString()); + Assert.Equal(master.ToString(), newSlave[0].ToDictionary()["name"]); + } } } From 6a6325cf876fab245600e961525e1466e11abe51 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 12 Apr 2020 20:44:57 -0500 Subject: [PATCH 02/23] Update configuration docs to show a sample of connecting to sentinel service --- docs/Configuration.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 6d535632f..3f256e8b2 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -29,6 +29,13 @@ This will connect to a single server on the local machine using the default redi var conn = ConnectionMultiplexer.Connect("redis0:6380,redis1:6380,allowAdmin=true"); ``` +This will connect to a sentinel server on the local machine using the default sentinel port (26379), discover the current master server for the `mymaster` service and return a managed connection +pointing to that master server that will automatically be updated if the master changes: + +```csharp +var conn = ConnectionMultiplexer.SentinelMasterConnect("localhost,serviceName=mymaster"); +``` + An overview of mapping between the `string` and `ConfigurationOptions` representation is shown below, but you can switch between them trivially: ```csharp @@ -79,7 +86,7 @@ The `ConfigurationOptions` object has a wide range of properties, all of which a | proxy={proxy type} | `Proxy` | `Proxy.None` | Type of proxy in use (if any); for example "twemproxy" | | resolveDns={bool} | `ResolveDns` | `false` | Specifies that DNS resolution should be explicit and eager, rather than implicit | | responseTimeout={int} | `ResponseTimeout` | `SyncTimeout` | Time (ms) to decide whether the socket is unhealthy | -| serviceName={string} | `ServiceName` | `null` | Not currently implemented (intended for use with sentinel) | +| serviceName={string} | `ServiceName` | `null` | Used for connecting to a sentinel master service | | ssl={bool} | `Ssl` | `false` | Specifies that SSL encryption should be used | | sslHost={string} | `SslHost` | `null` | Enforces a particular SSL host identity on the server's certificate | | sslProtocols={enum} | `SslProtocols` | `null` | Ssl/Tls versions supported when using an encrypted connection. Use '\|' to provide multiple values. | From 6786a9e4fbd02a79bba20f5e881977480f8db91d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 13 Apr 2020 14:19:58 -0500 Subject: [PATCH 03/23] Remove dead code --- tests/StackExchange.Redis.Tests/Sentinel.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index dcf229cb4..2a644964d 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -122,11 +122,6 @@ public async Task MasterConnectFailoverTest() await db.StringSetAsync(key, expected); } - private void Conn_ConfigurationChanged(object sender, EndPointEventArgs e) - { - throw new NotImplementedException(); - } - [Fact] public void PingTest() { From e7e756a636c336df1be7ec35052645439a0ef6de Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 29 Apr 2020 14:48:33 -0500 Subject: [PATCH 04/23] Some progress, but moving over to Windows because dev on OSX doesn't seem to work --- .../ConnectionMultiplexer.cs | 109 +++++++++++------- tests/StackExchange.Redis.Tests/Sentinel.cs | 62 +++++++++- 2 files changed, 126 insertions(+), 45 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index d5dbfb819..e313dead6 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -839,10 +839,11 @@ internal ServerEndPoint AnyConnected(ServerType serverType, uint startOffset, Re public static Task ConnectAsync(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImplAsync(configuration, log); + var config = ConfigurationOptions.Parse(configuration); + return ConnectImplAsync(config, log); } - private static async Task ConnectImplAsync(object configuration, TextWriter log = null) + private static async Task ConnectImplAsync(ConfigurationOptions configuration, TextWriter log = null) { IDisposable killMe = null; EventHandler connectHandler = null; @@ -882,25 +883,33 @@ public static Task ConnectAsync(ConfigurationOptions conf return ConnectImplAsync(configuration, log); } - internal static ConfigurationOptions PrepareConfig(object configuration) + internal static ConfigurationOptions PrepareConfig(string configuration, bool sentinel = false) + { + return PrepareConfig(ConfigurationOptions.Parse(configuration)); + } + + internal static ConfigurationOptions PrepareConfig(ConfigurationOptions configuration, bool sentinel = false) { if (configuration == null) throw new ArgumentNullException(nameof(configuration)); - ConfigurationOptions config; - if (configuration is string s) - { - config = ConfigurationOptions.Parse(s); - } - else if (configuration is ConfigurationOptions configurationOptions) + + if (configuration.EndPoints.Count == 0) + throw new ArgumentException("No endpoints specified", nameof(configuration)); + + if (!sentinel) { - config = (configurationOptions).Clone(); + configuration.SetDefaultPorts(); } else { - throw new ArgumentException("Invalid configuration object", nameof(configuration)); + // this is required when connecting to sentinel servers + configuration.TieBreaker = ""; + configuration.CommandMap = CommandMap.Sentinel; + + // use default sentinel port + configuration.EndPoints.SetDefaultPorts(26379); } - if (config.EndPoints.Count == 0) throw new ArgumentException("No endpoints specified", nameof(configuration)); - config.SetDefaultPorts(); - return config; + + return configuration; } internal class LogProxy : IDisposable @@ -952,9 +961,9 @@ public void Dispose() } } } - private static ConnectionMultiplexer CreateMultiplexer(object configuration, LogProxy log, out EventHandler connectHandler) + private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions configuration, LogProxy log, out EventHandler connectHandler) { - var muxer = new ConnectionMultiplexer(PrepareConfig(configuration)); + var muxer = new ConnectionMultiplexer(PrepareConfig(configuration.Clone())); connectHandler = null; if (log != null) { @@ -988,7 +997,8 @@ private static ConnectionMultiplexer CreateMultiplexer(object configuration, Log public static ConnectionMultiplexer Connect(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(configuration, log); + var config = ConfigurationOptions.Parse(configuration); + return ConnectImpl(config, false, log); } /// @@ -999,7 +1009,7 @@ public static ConnectionMultiplexer Connect(string configuration, TextWriter log public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(configuration, log); + return ConnectImpl(configuration, false, log); } /// @@ -1009,8 +1019,21 @@ public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, /// The to log to. public static ConnectionMultiplexer SentinelConnect(string configuration, TextWriter log = null) { - var options = ConfigurationOptions.Parse(configuration); - return SentinelConnect(options); + SocketConnection.AssertDependencies(); + var config = ConfigurationOptions.Parse(configuration); + return ConnectImpl(config, true, log); + } + + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server + /// + /// The string configuration to use for this multiplexer. + /// The to log to. + public static Task SentinelConnectAsync(string configuration, TextWriter log = null) + { + SocketConnection.AssertDependencies(); + var config = ConfigurationOptions.Parse(configuration); + return ConnectImplAsync(config, log); } /// @@ -1020,19 +1043,19 @@ public static ConnectionMultiplexer SentinelConnect(string configuration, TextWr /// The to log to. public static ConnectionMultiplexer SentinelConnect(ConfigurationOptions configuration, TextWriter log = null) { - if (string.IsNullOrEmpty(configuration.ServiceName)) - throw new ArgumentException("A ServiceName must be specified."); - - var sentinelConfigurationOptions = configuration.Clone(); - - // this is required when connecting to sentinel servers - sentinelConfigurationOptions.TieBreaker = ""; - sentinelConfigurationOptions.CommandMap = CommandMap.Sentinel; - - // use default sentinel port - sentinelConfigurationOptions.EndPoints.SetDefaultPorts(26379); + SocketConnection.AssertDependencies(); + return ConnectImpl(configuration, true, log); + } - return Connect(sentinelConfigurationOptions, log); + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server + /// + /// The configuration options to use for this multiplexer. + /// The to log to. + public static Task SentinelConnectAsync(ConfigurationOptions configuration, TextWriter log = null) + { + SocketConnection.AssertDependencies(); + return ConnectImplAsync(configuration, log); } /// @@ -1043,10 +1066,8 @@ public static ConnectionMultiplexer SentinelConnect(ConfigurationOptions configu /// The to log to. public static ConnectionMultiplexer SentinelMasterConnect(string configuration, TextWriter log = null) { - var options = ConfigurationOptions.Parse(configuration); - var sentinelConnection = SentinelConnect(options, log); - - return sentinelConnection.GetSentinelMasterConnection(options, log); + var config = ConfigurationOptions.Parse(configuration); + return SentinelMasterConnect(config, log); } /// @@ -1062,7 +1083,7 @@ public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions c return sentinelConnection.GetSentinelMasterConnection(configuration, log); } - private static ConnectionMultiplexer ConnectImpl(object configuration, TextWriter log) + private static ConnectionMultiplexer ConnectImpl(ConfigurationOptions configuration, bool sentinel, TextWriter log) { IDisposable killMe = null; EventHandler connectHandler = null; @@ -1616,7 +1637,7 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, LogP int iterCount = first ? 2 : 1; // this is fix for https://github.com/StackExchange/StackExchange.Redis/issues/300 - // auto discoverability of cluster nodes is made synchronous. + // auto discoverability of cluster nodes is made synchronous. // we try to connect to endpoints specified inside the user provided configuration // and when we encounter one such endpoint to which we are able to successfully connect, // we get the list of cluster nodes from this endpoint and try to proactively connect @@ -1706,7 +1727,7 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, LogP if (clusterCount > 0 && !encounteredConnectedClusterServer) { - // we have encountered a connected server with clustertype for the first time. + // we have encountered a connected server with clustertype for the first time. // so we will get list of other nodes from this server using "CLUSTER NODES" command // and try to connect to these other nodes in the next iteration encounteredConnectedClusterServer = true; @@ -2204,6 +2225,7 @@ internal void InitializeSentinel(LogProxy logProxy) // Subscribe to sentinel change events ISubscriber sub = GetSubscriber(); + if (sub.SubscribedEndpoint("+switch-master") == null) { sub.Subscribe("+switch-master", (channel, message) => @@ -2297,6 +2319,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co } ConnectionMultiplexer connection = Connect(config, log); + // TODO: Verify ROLE is master as specified here: https://redis.io/topics/sentinel-clients // Attach to reconnect event to ensure proper connection to the new master connection.ConnectionRestored += OnManagedConnectionRestored; @@ -2327,7 +2350,7 @@ internal void OnManagedConnectionRestored(object sender, ConnectionFailedEventAr try { - // Run a switch to make sure we have update-to-date + // Run a switch to make sure we have update-to-date // information about which master we should connect to SwitchMaster(e.EndPoint, connection); @@ -2398,7 +2421,7 @@ internal EndPoint GetConfiguredMasterForService(string serviceName) => /// /// Switches the SentinelMasterConnection over to a new master. /// - /// The endpoing responsible for the switch + /// The endpoint responsible for the switch /// The connection that should be switched over to a new master endpoint /// Log to write to, if any internal void SwitchMaster(EndPoint switchBlame, ConnectionMultiplexer connection, TextWriter log = null) @@ -2427,9 +2450,9 @@ internal void SwitchMaster(EndPoint switchBlame, ConnectionMultiplexer connectio { connection.RawConfig.EndPoints.Clear(); connection.servers.Clear(); - connection.RawConfig.EndPoints.Add(newMasterEndPoint); + connection.RawConfig.EndPoints.TryAdd(newMasterEndPoint); Trace(string.Format("Switching master to {0}", newMasterEndPoint)); - // Trigger a reconfigure + // Trigger a reconfigure connection.ReconfigureAsync(false, false, logProxy, switchBlame, string.Format("master switch {0}", serviceName), false, CommandFlags.PreferMaster).Wait(); } diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index 2a644964d..69e3ddf05 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -55,7 +55,7 @@ public void MasterConnectTest() var options = ServiceOptions.Clone(); options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); - var conn = ConnectionMultiplexer.SentinelMasterConnect(options); + var conn = ConnectionMultiplexer.Connect(options); var db = conn.GetDatabase(); var test = db.Ping(); @@ -63,6 +63,36 @@ public void MasterConnectTest() TestConfig.Current.SentinelPortA, test.TotalMilliseconds); } + [Fact] + public async Task MasterConnectWithConnectionStringFailoverTest() + { + var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName}"; + var conn = ConnectionMultiplexer.Connect(connectionString); + conn.ConfigurationChanged += (s, e) => { + Log($"Configuration changed: {e.EndPoint}"); + }; + var db = conn.GetDatabase(); + + var test = await db.PingAsync(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + + // set string value on current master + var expected = DateTime.Now.Ticks.ToString(); + Log("Tick Key: " + expected); + var key = Me(); + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + await db.StringSetAsync(key, expected); + + // forces and verifies failover + await DoFailoverAsync(); + + var value = await db.StringGetAsync(key); + Assert.Equal(expected, value); + + await db.StringSetAsync(key, expected); + } + [Fact] public void MasterConnectWithDefaultPortTest() { @@ -81,7 +111,21 @@ public void MasterConnectWithDefaultPortTest() public void MasterConnectWithStringConfigurationTest() { var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName}"; - var conn = ConnectionMultiplexer.SentinelMasterConnect(connectionString); + var conn = ConnectionMultiplexer.Connect(connectionString); + var db = conn.GetDatabase(); + + var test = db.Ping(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + } + + [Fact] + public void SentinelConnectTest() + { + var options = ServiceOptions.Clone(); + options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); + + var conn = ConnectionMultiplexer.SentinelConnect(options); var db = conn.GetDatabase(); var test = db.Ping(); @@ -89,6 +133,20 @@ public void MasterConnectWithStringConfigurationTest() TestConfig.Current.SentinelPortA, test.TotalMilliseconds); } + [Fact] + public async Task SentinelConnectAsyncTest() + { + var options = ServiceOptions.Clone(); + options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); + + var conn = await ConnectionMultiplexer.SentinelConnectAsync(options); + var db = conn.GetDatabase(); + + var test = await db.PingAsync(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + } + [Fact] public async Task MasterConnectFailoverTest() { From c3243d429a7c1a4498a68a388ac6d9673039e346 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 30 Apr 2020 15:21:25 -0500 Subject: [PATCH 05/23] Get all tests passing --- .../ConnectionMultiplexer.cs | 71 +++++++++++-------- tests/StackExchange.Redis.Tests/Sentinel.cs | 16 +---- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index e313dead6..0738734b9 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -839,8 +839,7 @@ internal ServerEndPoint AnyConnected(ServerType serverType, uint startOffset, Re public static Task ConnectAsync(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - var config = ConfigurationOptions.Parse(configuration); - return ConnectImplAsync(config, log); + return ConnectImplAsync(PrepareConfig(configuration), log); } private static async Task ConnectImplAsync(ConfigurationOptions configuration, TextWriter log = null) @@ -880,36 +879,45 @@ private static async Task ConnectImplAsync(ConfigurationO public static Task ConnectAsync(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImplAsync(configuration, log); + return ConnectImplAsync(PrepareConfig(configuration), log); } - internal static ConfigurationOptions PrepareConfig(string configuration, bool sentinel = false) - { - return PrepareConfig(ConfigurationOptions.Parse(configuration)); - } - - internal static ConfigurationOptions PrepareConfig(ConfigurationOptions configuration, bool sentinel = false) + internal static ConfigurationOptions PrepareConfig(object configuration, bool sentinel = false) { if (configuration == null) throw new ArgumentNullException(nameof(configuration)); - if (configuration.EndPoints.Count == 0) + ConfigurationOptions config; + if (configuration is string c) + { + config = ConfigurationOptions.Parse(c); + } + else if (configuration is ConfigurationOptions configurationOptions) + { + config = configurationOptions.Clone(); + } + else + { + throw new ArgumentException("Invalid configuration object", nameof(configuration)); + } + + if (config.EndPoints.Count == 0) throw new ArgumentException("No endpoints specified", nameof(configuration)); if (!sentinel) { - configuration.SetDefaultPorts(); + config.SetDefaultPorts(); } else { // this is required when connecting to sentinel servers - configuration.TieBreaker = ""; - configuration.CommandMap = CommandMap.Sentinel; + config.TieBreaker = ""; + config.CommandMap = CommandMap.Sentinel; // use default sentinel port - configuration.EndPoints.SetDefaultPorts(26379); + config.EndPoints.SetDefaultPorts(26379); } - return configuration; + return config; } internal class LogProxy : IDisposable @@ -963,7 +971,7 @@ public void Dispose() } private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions configuration, LogProxy log, out EventHandler connectHandler) { - var muxer = new ConnectionMultiplexer(PrepareConfig(configuration.Clone())); + var muxer = new ConnectionMultiplexer(configuration); connectHandler = null; if (log != null) { @@ -997,8 +1005,16 @@ private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions conf public static ConnectionMultiplexer Connect(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - var config = ConfigurationOptions.Parse(configuration); - return ConnectImpl(config, false, log); + var config = PrepareConfig(configuration); + bool sentinel = !String.IsNullOrEmpty(config.ServiceName); + + if (!sentinel) + return ConnectImpl(config, log); + + var sentinelConfig = PrepareConfig(configuration, true); + var conn = ConnectImpl(sentinelConfig, log); + + return conn.GetSentinelMasterConnection(config, log); } /// @@ -1009,7 +1025,7 @@ public static ConnectionMultiplexer Connect(string configuration, TextWriter log public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(configuration, false, log); + return ConnectImpl(PrepareConfig(configuration), log); } /// @@ -1020,8 +1036,7 @@ public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, public static ConnectionMultiplexer SentinelConnect(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - var config = ConfigurationOptions.Parse(configuration); - return ConnectImpl(config, true, log); + return ConnectImpl(PrepareConfig(configuration), log); } /// @@ -1032,8 +1047,7 @@ public static ConnectionMultiplexer SentinelConnect(string configuration, TextWr public static Task SentinelConnectAsync(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - var config = ConfigurationOptions.Parse(configuration); - return ConnectImplAsync(config, log); + return ConnectImplAsync(PrepareConfig(configuration, true), log); } /// @@ -1044,7 +1058,7 @@ public static Task SentinelConnectAsync(string configurat public static ConnectionMultiplexer SentinelConnect(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(configuration, true, log); + return ConnectImpl(PrepareConfig(configuration, true), log); } /// @@ -1055,7 +1069,7 @@ public static ConnectionMultiplexer SentinelConnect(ConfigurationOptions configu public static Task SentinelConnectAsync(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImplAsync(configuration, log); + return ConnectImplAsync(PrepareConfig(configuration, true), log); } /// @@ -1066,8 +1080,7 @@ public static Task SentinelConnectAsync(ConfigurationOpti /// The to log to. public static ConnectionMultiplexer SentinelMasterConnect(string configuration, TextWriter log = null) { - var config = ConfigurationOptions.Parse(configuration); - return SentinelMasterConnect(config, log); + return SentinelMasterConnect(PrepareConfig(configuration, true), log); } /// @@ -1083,7 +1096,7 @@ public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions c return sentinelConnection.GetSentinelMasterConnection(configuration, log); } - private static ConnectionMultiplexer ConnectImpl(ConfigurationOptions configuration, bool sentinel, TextWriter log) + private static ConnectionMultiplexer ConnectImpl(ConfigurationOptions configuration, TextWriter log) { IDisposable killMe = null; EventHandler connectHandler = null; @@ -2318,7 +2331,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co config.EndPoints.Add(newMasterEndPoint); } - ConnectionMultiplexer connection = Connect(config, log); + ConnectionMultiplexer connection = ConnectImpl(config, log); // TODO: Verify ROLE is master as specified here: https://redis.io/topics/sentinel-clients // Attach to reconnect event to ensure proper connection to the new master diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index 69e3ddf05..194c595cc 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Net; @@ -49,20 +49,6 @@ public Sentinel(ITestOutputHelper output) : base(output) SentinelsServers = new IServer[] { SentinelServerA, SentinelServerB, SentinelServerC }; } - [Fact] - public void MasterConnectTest() - { - var options = ServiceOptions.Clone(); - options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); - - var conn = ConnectionMultiplexer.Connect(options); - var db = conn.GetDatabase(); - - var test = db.Ping(); - Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, - TestConfig.Current.SentinelPortA, test.TotalMilliseconds); - } - [Fact] public async Task MasterConnectWithConnectionStringFailoverTest() { From 891b4d263792f497c1d20e1fe966f1df4c30e257 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 30 Apr 2020 15:58:54 -0500 Subject: [PATCH 06/23] Add some missing overloads and fix issue with connectasync with sentinel --- .../ConnectionMultiplexer.cs | 76 +++++++++++++------ tests/StackExchange.Redis.Tests/Sentinel.cs | 32 +++++++- 2 files changed, 84 insertions(+), 24 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 0738734b9..f6b657abd 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -839,7 +839,7 @@ internal ServerEndPoint AnyConnected(ServerType serverType, uint startOffset, Re public static Task ConnectAsync(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImplAsync(PrepareConfig(configuration), log); + return ConnectAsync(PrepareConfig(configuration), log); } private static async Task ConnectImplAsync(ConfigurationOptions configuration, TextWriter log = null) @@ -861,6 +861,12 @@ private static async Task ConnectImplAsync(ConfigurationO } killMe = null; Interlocked.Increment(ref muxer._connectCompletedCount); + + if (muxer.ServerSelectionStrategy.ServerType == ServerType.Sentinel) + { + // Initialize the Sentinel handlers + muxer.InitializeSentinel(logProxy); + } return muxer; } finally @@ -876,32 +882,36 @@ private static async Task ConnectImplAsync(ConfigurationO /// /// The configuration options to use for this multiplexer. /// The to log to. - public static Task ConnectAsync(ConfigurationOptions configuration, TextWriter log = null) + public static async Task ConnectAsync(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImplAsync(PrepareConfig(configuration), log); + + bool sentinel = !String.IsNullOrEmpty(configuration.ServiceName); + + if (!sentinel) + return await ConnectImplAsync(PrepareConfig(configuration), log).ForAwait(); + + var conn = await ConnectImplAsync(PrepareConfig(configuration, true), log).ForAwait(); + return conn.GetSentinelMasterConnection(PrepareConfig(configuration), log); } internal static ConfigurationOptions PrepareConfig(object configuration, bool sentinel = false) { if (configuration == null) throw new ArgumentNullException(nameof(configuration)); - ConfigurationOptions config; - if (configuration is string c) + if (configuration is string s) { - config = ConfigurationOptions.Parse(c); + config = ConfigurationOptions.Parse(s); } else if (configuration is ConfigurationOptions configurationOptions) { - config = configurationOptions.Clone(); + config = (configurationOptions).Clone(); } else { throw new ArgumentException("Invalid configuration object", nameof(configuration)); } - - if (config.EndPoints.Count == 0) - throw new ArgumentException("No endpoints specified", nameof(configuration)); + if (config.EndPoints.Count == 0) throw new ArgumentException("No endpoints specified", nameof(configuration)); if (!sentinel) { @@ -1004,17 +1014,8 @@ private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions conf /// The to log to. public static ConnectionMultiplexer Connect(string configuration, TextWriter log = null) { - SocketConnection.AssertDependencies(); var config = PrepareConfig(configuration); - bool sentinel = !String.IsNullOrEmpty(config.ServiceName); - - if (!sentinel) - return ConnectImpl(config, log); - - var sentinelConfig = PrepareConfig(configuration, true); - var conn = ConnectImpl(sentinelConfig, log); - - return conn.GetSentinelMasterConnection(config, log); + return Connect(config, log); } /// @@ -1025,7 +1026,14 @@ public static ConnectionMultiplexer Connect(string configuration, TextWriter log public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(PrepareConfig(configuration), log); + + bool sentinel = !String.IsNullOrEmpty(configuration.ServiceName); + + if (!sentinel) + return ConnectImpl(PrepareConfig(configuration), log); + + var conn = ConnectImpl(PrepareConfig(configuration, true), log); + return conn.GetSentinelMasterConnection(PrepareConfig(configuration), log); } /// @@ -1036,7 +1044,7 @@ public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, public static ConnectionMultiplexer SentinelConnect(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(PrepareConfig(configuration), log); + return ConnectImpl(PrepareConfig(configuration, true), log); } /// @@ -1096,6 +1104,30 @@ public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions c return sentinelConnection.GetSentinelMasterConnection(configuration, log); } + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server, discovers the current master server + /// for the specified ServiceName in the config and returns a managed connection to the current master server + /// + /// The string configuration to use for this multiplexer. + /// The to log to. + public static Task SentinelMasterConnectAsync(string configuration, TextWriter log = null) + { + return SentinelMasterConnectAsync(PrepareConfig(configuration, true), log); + } + + /// + /// Create a new ConnectionMultiplexer instance that connects to a sentinel server, discovers the current master server + /// for the specified ServiceName in the config and returns a managed connection to the current master server + /// + /// The configuration options to use for this multiplexer. + /// The to log to. + public static async Task SentinelMasterConnectAsync(ConfigurationOptions configuration, TextWriter log = null) + { + var sentinelConnection = await SentinelConnectAsync(configuration, log).ForAwait(); + + return sentinelConnection.GetSentinelMasterConnection(configuration, log); + } + private static ConnectionMultiplexer ConnectImpl(ConfigurationOptions configuration, TextWriter log) { IDisposable killMe = null; diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index 194c595cc..63a5d3c13 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -59,6 +59,36 @@ public async Task MasterConnectWithConnectionStringFailoverTest() }; var db = conn.GetDatabase(); + var test = db.Ping(); + Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, + TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + + // set string value on current master + var expected = DateTime.Now.Ticks.ToString(); + Log("Tick Key: " + expected); + var key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + db.StringSet(key, expected); + + // forces and verifies failover + await DoFailoverAsync(); + + var value = db.StringGet(key); + Assert.Equal(expected, value); + + db.StringSet(key, expected); + } + + [Fact] + public async Task MasterConnectAsyncWithConnectionStringFailoverTest() + { + var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName}"; + var conn = await ConnectionMultiplexer.ConnectAsync(connectionString); + conn.ConfigurationChanged += (s, e) => { + Log($"Configuration changed: {e.EndPoint}"); + }; + var db = conn.GetDatabase(); + var test = await db.PingAsync(); Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA, test.TotalMilliseconds); @@ -692,8 +722,6 @@ public async Task ReadOnlyConnectionSlavesTest() var slaves = SentinelServerA.SentinelSlaves(ServiceName); var config = new ConfigurationOptions { - TieBreaker = "", - ServiceName = ServiceOptions.ServiceName, Password = ServiceOptions.Password }; From 48c0a094b3bfa1f5e0bce6b28bb1e8e7abbc0888 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 30 Apr 2020 16:05:30 -0500 Subject: [PATCH 07/23] Sentinel doc update --- docs/Configuration.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 3f256e8b2..480e5324b 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -29,11 +29,12 @@ This will connect to a single server on the local machine using the default redi var conn = ConnectionMultiplexer.Connect("redis0:6380,redis1:6380,allowAdmin=true"); ``` -This will connect to a sentinel server on the local machine using the default sentinel port (26379), discover the current master server for the `mymaster` service and return a managed connection +If you specify a serviceName in the connection string, it will trigger sentinel mode. This will connect to a sentinel server on the local machine +using the default sentinel port (26379), discover the current master server for the `mymaster` service and return a managed connection pointing to that master server that will automatically be updated if the master changes: ```csharp -var conn = ConnectionMultiplexer.SentinelMasterConnect("localhost,serviceName=mymaster"); +var conn = ConnectionMultiplexer.Connect("localhost,serviceName=mymaster"); ``` An overview of mapping between the `string` and `ConfigurationOptions` representation is shown below, but you can switch between them trivially: From 4276b53c702517f134cc440694a024a0470a9646 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 1 May 2020 21:28:29 -0500 Subject: [PATCH 08/23] Update docs/Configuration.md Co-authored-by: Nick Craver --- docs/Configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 480e5324b..6c808ee06 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -29,7 +29,7 @@ This will connect to a single server on the local machine using the default redi var conn = ConnectionMultiplexer.Connect("redis0:6380,redis1:6380,allowAdmin=true"); ``` -If you specify a serviceName in the connection string, it will trigger sentinel mode. This will connect to a sentinel server on the local machine +If you specify a serviceName in the connection string, it will trigger sentinel mode. This example will connect to a sentinel server on the local machine using the default sentinel port (26379), discover the current master server for the `mymaster` service and return a managed connection pointing to that master server that will automatically be updated if the master changes: From a32f61167f6d3177bdc6a4bda8b067e06eee565b Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 1 May 2020 21:41:52 -0500 Subject: [PATCH 09/23] Making suggested changed from @NickCraver --- .../ConnectionMultiplexer.cs | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index f6b657abd..d724263ad 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -891,7 +891,7 @@ public static async Task ConnectAsync(ConfigurationOption if (!sentinel) return await ConnectImplAsync(PrepareConfig(configuration), log).ForAwait(); - var conn = await ConnectImplAsync(PrepareConfig(configuration, true), log).ForAwait(); + var conn = await ConnectImplAsync(PrepareConfig(configuration, sentinel: true), log).ForAwait(); return conn.GetSentinelMasterConnection(PrepareConfig(configuration), log); } @@ -913,11 +913,7 @@ internal static ConfigurationOptions PrepareConfig(object configuration, bool se } if (config.EndPoints.Count == 0) throw new ArgumentException("No endpoints specified", nameof(configuration)); - if (!sentinel) - { - config.SetDefaultPorts(); - } - else + if (sentinel) { // this is required when connecting to sentinel servers config.TieBreaker = ""; @@ -925,8 +921,12 @@ internal static ConfigurationOptions PrepareConfig(object configuration, bool se // use default sentinel port config.EndPoints.SetDefaultPorts(26379); + + return config; } + config.SetDefaultPorts(); + return config; } @@ -1014,8 +1014,7 @@ private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions conf /// The to log to. public static ConnectionMultiplexer Connect(string configuration, TextWriter log = null) { - var config = PrepareConfig(configuration); - return Connect(config, log); + return Connect(PrepareConfig(configuration), log); } /// @@ -1027,13 +1026,15 @@ public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, { SocketConnection.AssertDependencies(); - bool sentinel = !String.IsNullOrEmpty(configuration.ServiceName); + bool sentinel = !string.IsNullOrEmpty(configuration?.ServiceName); - if (!sentinel) - return ConnectImpl(PrepareConfig(configuration), log); + if (sentinel) + { + var conn = ConnectImpl(PrepareConfig(configuration, sentinel: true), log); + return conn.GetSentinelMasterConnection(PrepareConfig(configuration), log); + } - var conn = ConnectImpl(PrepareConfig(configuration, true), log); - return conn.GetSentinelMasterConnection(PrepareConfig(configuration), log); + return ConnectImpl(PrepareConfig(configuration), log); } /// @@ -1044,7 +1045,7 @@ public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, public static ConnectionMultiplexer SentinelConnect(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(PrepareConfig(configuration, true), log); + return ConnectImpl(PrepareConfig(configuration, sentinel: true), log); } /// @@ -1066,7 +1067,7 @@ public static Task SentinelConnectAsync(string configurat public static ConnectionMultiplexer SentinelConnect(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImpl(PrepareConfig(configuration, true), log); + return ConnectImpl(PrepareConfig(configuration, sentinel: true), log); } /// @@ -1077,7 +1078,7 @@ public static ConnectionMultiplexer SentinelConnect(ConfigurationOptions configu public static Task SentinelConnectAsync(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImplAsync(PrepareConfig(configuration, true), log); + return ConnectImplAsync(PrepareConfig(configuration, sentinel: true), log); } /// @@ -1088,7 +1089,7 @@ public static Task SentinelConnectAsync(ConfigurationOpti /// The to log to. public static ConnectionMultiplexer SentinelMasterConnect(string configuration, TextWriter log = null) { - return SentinelMasterConnect(PrepareConfig(configuration, true), log); + return SentinelMasterConnect(PrepareConfig(configuration, sentinel: true), log); } /// @@ -1112,7 +1113,7 @@ public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions c /// The to log to. public static Task SentinelMasterConnectAsync(string configuration, TextWriter log = null) { - return SentinelMasterConnectAsync(PrepareConfig(configuration, true), log); + return SentinelMasterConnectAsync(PrepareConfig(configuration, sentinel: true), log); } /// From 659637897609f04f1a3ca65417884c6c4293b93c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 2 May 2020 14:10:19 -0500 Subject: [PATCH 10/23] More updates from feedback --- .../ConnectionMultiplexer.cs | 21 +++++++++---------- tests/RedisConfigs/Sentinel/redis-7010.conf | 2 -- tests/RedisConfigs/Sentinel/redis-7011.conf | 2 -- .../RedisConfigs/Sentinel/sentinel-26379.conf | 2 -- .../RedisConfigs/Sentinel/sentinel-26380.conf | 2 -- .../RedisConfigs/Sentinel/sentinel-26381.conf | 2 -- tests/StackExchange.Redis.Tests/Sentinel.cs | 2 +- 7 files changed, 11 insertions(+), 22 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index d724263ad..13ca2abe5 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -882,17 +882,19 @@ private static async Task ConnectImplAsync(ConfigurationO /// /// The configuration options to use for this multiplexer. /// The to log to. - public static async Task ConnectAsync(ConfigurationOptions configuration, TextWriter log = null) + public static Task ConnectAsync(ConfigurationOptions configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - bool sentinel = !String.IsNullOrEmpty(configuration.ServiceName); + if (IsSentinel(configuration)) + return SentinelMasterConnectAsync(configuration, log); - if (!sentinel) - return await ConnectImplAsync(PrepareConfig(configuration), log).ForAwait(); + return ConnectImplAsync(PrepareConfig(configuration), log); + } - var conn = await ConnectImplAsync(PrepareConfig(configuration, sentinel: true), log).ForAwait(); - return conn.GetSentinelMasterConnection(PrepareConfig(configuration), log); + private static bool IsSentinel(ConfigurationOptions configuration) + { + return !string.IsNullOrEmpty(configuration?.ServiceName); } internal static ConfigurationOptions PrepareConfig(object configuration, bool sentinel = false) @@ -1026,12 +1028,9 @@ public static ConnectionMultiplexer Connect(ConfigurationOptions configuration, { SocketConnection.AssertDependencies(); - bool sentinel = !string.IsNullOrEmpty(configuration?.ServiceName); - - if (sentinel) + if (IsSentinel(configuration)) { - var conn = ConnectImpl(PrepareConfig(configuration, sentinel: true), log); - return conn.GetSentinelMasterConnection(PrepareConfig(configuration), log); + return SentinelMasterConnect(configuration, log); } return ConnectImpl(PrepareConfig(configuration), log); diff --git a/tests/RedisConfigs/Sentinel/redis-7010.conf b/tests/RedisConfigs/Sentinel/redis-7010.conf index 807682537..0e27680b2 100644 --- a/tests/RedisConfigs/Sentinel/redis-7010.conf +++ b/tests/RedisConfigs/Sentinel/redis-7010.conf @@ -1,6 +1,4 @@ port 7010 -#requirepass changeme -#masterauth changeme repl-diskless-sync yes repl-diskless-sync-delay 0 maxmemory 100mb diff --git a/tests/RedisConfigs/Sentinel/redis-7011.conf b/tests/RedisConfigs/Sentinel/redis-7011.conf index 382247b75..fd013cda2 100644 --- a/tests/RedisConfigs/Sentinel/redis-7011.conf +++ b/tests/RedisConfigs/Sentinel/redis-7011.conf @@ -1,7 +1,5 @@ port 7011 -#requirepass changeme slaveof 127.0.0.1 7010 -#masterauth changeme repl-diskless-sync yes repl-diskless-sync-delay 0 maxmemory 100mb diff --git a/tests/RedisConfigs/Sentinel/sentinel-26379.conf b/tests/RedisConfigs/Sentinel/sentinel-26379.conf index 4a63fab9d..6d10f6030 100644 --- a/tests/RedisConfigs/Sentinel/sentinel-26379.conf +++ b/tests/RedisConfigs/Sentinel/sentinel-26379.conf @@ -1,7 +1,5 @@ port 26379 -#requirepass "changeme" sentinel monitor mymaster 127.0.0.1 7010 1 -#sentinel auth-pass mymaster changeme sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 1000 sentinel config-epoch mymaster 0 diff --git a/tests/RedisConfigs/Sentinel/sentinel-26380.conf b/tests/RedisConfigs/Sentinel/sentinel-26380.conf index 57f544c8f..fa044227e 100644 --- a/tests/RedisConfigs/Sentinel/sentinel-26380.conf +++ b/tests/RedisConfigs/Sentinel/sentinel-26380.conf @@ -1,7 +1,5 @@ port 26380 -#requirepass "changeme" sentinel monitor mymaster 127.0.0.1 7010 1 -#sentinel auth-pass mymaster changeme sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 1000 sentinel config-epoch mymaster 0 diff --git a/tests/RedisConfigs/Sentinel/sentinel-26381.conf b/tests/RedisConfigs/Sentinel/sentinel-26381.conf index e8159bcdb..b286e5098 100644 --- a/tests/RedisConfigs/Sentinel/sentinel-26381.conf +++ b/tests/RedisConfigs/Sentinel/sentinel-26381.conf @@ -1,7 +1,5 @@ port 26381 -#requirepass "changeme" sentinel monitor mymaster 127.0.0.1 7010 1 -#sentinel auth-pass mymaster changeme sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 1000 sentinel config-epoch mymaster 0 diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index 63a5d3c13..ce7b8a1a7 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -12,7 +12,7 @@ namespace StackExchange.Redis.Tests public class Sentinel : TestBase { private string ServiceName => TestConfig.Current.SentinelSeviceName; - private ConfigurationOptions ServiceOptions => new ConfigurationOptions { ServiceName = ServiceName, AllowAdmin = true, Password = "changeme" }; + private ConfigurationOptions ServiceOptions => new ConfigurationOptions { ServiceName = ServiceName, AllowAdmin = true }; private ConnectionMultiplexer Conn { get; } private IServer SentinelServerA { get; } From 3a8a5d8f09ff87a217af651d6376d0726fb3590a Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 2 May 2020 14:14:25 -0500 Subject: [PATCH 11/23] More config reverts --- tests/RedisConfigs/Sentinel/redis-7011.conf | 2 +- tests/RedisConfigs/Sentinel/sentinel-26381.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/RedisConfigs/Sentinel/redis-7011.conf b/tests/RedisConfigs/Sentinel/redis-7011.conf index fd013cda2..6d02eb150 100644 --- a/tests/RedisConfigs/Sentinel/redis-7011.conf +++ b/tests/RedisConfigs/Sentinel/redis-7011.conf @@ -6,4 +6,4 @@ maxmemory 100mb appendonly no dir "../Temp" dbfilename "sentinel-target-7011.rdb" -save "" +save "" \ No newline at end of file diff --git a/tests/RedisConfigs/Sentinel/sentinel-26381.conf b/tests/RedisConfigs/Sentinel/sentinel-26381.conf index b286e5098..fa49c9e14 100644 --- a/tests/RedisConfigs/Sentinel/sentinel-26381.conf +++ b/tests/RedisConfigs/Sentinel/sentinel-26381.conf @@ -3,4 +3,4 @@ sentinel monitor mymaster 127.0.0.1 7010 1 sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 1000 sentinel config-epoch mymaster 0 -dir "../Temp" \ No newline at end of file +dir "../Temp" From ba2d9392461933e0381a550139a340ec1fff4b45 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 5 May 2020 10:49:31 -0500 Subject: [PATCH 12/23] Missed a spot for using sentinel parameter label. --- src/StackExchange.Redis/ConnectionMultiplexer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 13ca2abe5..394cf0b9b 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1055,7 +1055,7 @@ public static ConnectionMultiplexer SentinelConnect(string configuration, TextWr public static Task SentinelConnectAsync(string configuration, TextWriter log = null) { SocketConnection.AssertDependencies(); - return ConnectImplAsync(PrepareConfig(configuration, true), log); + return ConnectImplAsync(PrepareConfig(configuration, sentinel: true), log); } /// From 3557c76897947f093d8cb4b818ec73713e5febb4 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 7 May 2020 01:35:22 -0500 Subject: [PATCH 13/23] More complete sentinel implementation. Add slaves to endpoints when connecting to sentinel. Adding ROLE command. --- src/StackExchange.Redis/CommandMap.cs | 2 +- .../ConnectionMultiplexer.cs | 125 +++++++++++++----- src/StackExchange.Redis/EndPointCollection.cs | 2 +- src/StackExchange.Redis/Enums/RedisCommand.cs | 1 + src/StackExchange.Redis/Interfaces/IServer.cs | 42 +++++- src/StackExchange.Redis/Message.cs | 1 + src/StackExchange.Redis/RedisLiterals.cs | 2 + src/StackExchange.Redis/RedisServer.cs | 24 ++++ src/StackExchange.Redis/ResultProcessor.cs | 77 +++++++++++ tests/StackExchange.Redis.Tests/Sentinel.cs | 47 ++++++- 10 files changed, 279 insertions(+), 44 deletions(-) diff --git a/src/StackExchange.Redis/CommandMap.cs b/src/StackExchange.Redis/CommandMap.cs index 5d2178d48..a47d96860 100644 --- a/src/StackExchange.Redis/CommandMap.cs +++ b/src/StackExchange.Redis/CommandMap.cs @@ -72,7 +72,7 @@ internal CommandMap(CommandBytes[] map) /// https://redis.io/topics/sentinel public static CommandMap Sentinel { get; } = Create(new HashSet { // see https://redis.io/topics/sentinel - "auth", "ping", "info", "sentinel", "subscribe", "shutdown", "psubscribe", "unsubscribe", "punsubscribe" }, true); + "auth", "ping", "info", "role", "sentinel", "subscribe", "shutdown", "psubscribe", "unsubscribe", "punsubscribe" }, true); /// /// Create a new CommandMap, customizing some commands diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 394cf0b9b..a1cee4d10 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -1101,7 +1101,11 @@ public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions c { var sentinelConnection = SentinelConnect(configuration, log); - return sentinelConnection.GetSentinelMasterConnection(configuration, log); + var muxer = sentinelConnection.GetSentinelMasterConnection(configuration, log); + // set reference to sentinel connection so that we can dispose it + muxer.sentinelConnection = sentinelConnection; + + return muxer; } /// @@ -1125,7 +1129,11 @@ public static async Task SentinelMasterConnectAsync(Confi { var sentinelConnection = await SentinelConnectAsync(configuration, log).ForAwait(); - return sentinelConnection.GetSentinelMasterConnection(configuration, log); + var muxer = sentinelConnection.GetSentinelMasterConnection(configuration, log); + // set reference to sentinel connection so that we can dispose it + muxer.sentinelConnection = sentinelConnection; + + return muxer; } private static ConnectionMultiplexer ConnectImpl(ConfigurationOptions configuration, TextWriter log) @@ -1169,7 +1177,7 @@ private static ConnectionMultiplexer ConnectImpl(ConfigurationOptions configurat } finally { - if (connectHandler != null) muxer.ConnectionFailed -= connectHandler; + if (connectHandler != null && muxer != null) muxer.ConnectionFailed -= connectHandler; if (killMe != null) try { killMe.Dispose(); } catch { } } } @@ -2252,6 +2260,7 @@ public bool IsConnecting internal Timer sentinelMasterReconnectTimer; internal Dictionary sentinelConnectionChildren; + internal ConnectionMultiplexer sentinelConnection = null; /// /// Initializes the connection as a Sentinel connection and adds @@ -2341,31 +2350,61 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co return sentinelConnectionChildren[config.ServiceName]; } - // Get an initial endpoint - try twice - EndPoint newMasterEndPoint = GetConfiguredMasterForService(config.ServiceName) - ?? GetConfiguredMasterForService(config.ServiceName); + int attempts = 0; + bool success = false; + ConnectionMultiplexer connection = null; - if (newMasterEndPoint == null) + do { - throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, - $"Sentinel: Failed connecting to configured master for service: {config.ServiceName}"); - } + attempts++; - // Replace the master endpoint, if we found another one - // If not, assume the last state is the best we have and minimize the race - if (config.EndPoints.Count == 1) - { - config.EndPoints[0] = newMasterEndPoint; - } - else + // Get an initial endpoint - try twice + EndPoint newMasterEndPoint = GetConfiguredMasterForService(config.ServiceName) + ?? GetConfiguredMasterForService(config.ServiceName); + + if (newMasterEndPoint == null) + { + throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, + $"Sentinel: Failed connecting to configured master for service: {config.ServiceName}"); + } + + EndPoint[] slaveEndPoints = GetSlavesForService(config.ServiceName) + ?? GetSlavesForService(config.ServiceName); + + // Replace the master endpoint, if we found another one + // If not, assume the last state is the best we have and minimize the race + if (config.EndPoints.Count == 1) + { + config.EndPoints[0] = newMasterEndPoint; + } + else + { + config.EndPoints.Clear(); + config.EndPoints.TryAdd(newMasterEndPoint); + } + + foreach (var slaveEndPoint in slaveEndPoints) + config.EndPoints.TryAdd(slaveEndPoint); + + connection = ConnectImpl(config, log); + + // verify role is master according to: + // https://redis.io/topics/sentinel-clients + if (connection.GetServer(newMasterEndPoint).Role() == RedisLiterals.master) + { + success = true; + break; + } + + Thread.Sleep(100); + } while (attempts < 3); + + if (!success) { - config.EndPoints.Clear(); - config.EndPoints.Add(newMasterEndPoint); + throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, + $"Sentinel: Failed connecting to configured master for service: {config.ServiceName}"); } - ConnectionMultiplexer connection = ConnectImpl(config, log); - // TODO: Verify ROLE is master as specified here: https://redis.io/topics/sentinel-clients - // Attach to reconnect event to ensure proper connection to the new master connection.ConnectionRestored += OnManagedConnectionRestored; @@ -2463,6 +2502,18 @@ internal EndPoint GetConfiguredMasterForService(string serviceName) => internal EndPoint currentSentinelMasterEndPoint; + internal EndPoint[] GetSlavesForService(string serviceName) => + GetServerSnapshot() + .ToArray() + .Where(s => s.ServerType == ServerType.Sentinel) + .AsParallel() + .Select(s => + { + try { return GetServer(s.EndPoint).SentinelGetSlaveAddresses(serviceName); } + catch { return null; } + }) + .FirstOrDefault(r => r != null); + /// /// Switches the SentinelMasterConnection over to a new master. /// @@ -2487,19 +2538,22 @@ internal void SwitchMaster(EndPoint switchBlame, ConnectionMultiplexer connectio $"Sentinel: Failed connecting to switch master for service: {serviceName}"); } - if (newMasterEndPoint != null) - { - connection.currentSentinelMasterEndPoint = newMasterEndPoint; + connection.currentSentinelMasterEndPoint = newMasterEndPoint; - if (!connection.servers.Contains(newMasterEndPoint)) - { - connection.RawConfig.EndPoints.Clear(); - connection.servers.Clear(); - connection.RawConfig.EndPoints.TryAdd(newMasterEndPoint); - Trace(string.Format("Switching master to {0}", newMasterEndPoint)); - // Trigger a reconfigure - connection.ReconfigureAsync(false, false, logProxy, switchBlame, string.Format("master switch {0}", serviceName), false, CommandFlags.PreferMaster).Wait(); - } + if (!connection.servers.Contains(newMasterEndPoint)) + { + EndPoint[] slaveEndPoints = GetSlavesForService(serviceName) + ?? GetSlavesForService(serviceName); + + connection.servers.Clear(); + connection.RawConfig.EndPoints.Clear(); + connection.RawConfig.EndPoints.TryAdd(newMasterEndPoint); + foreach (var slaveEndPoint in slaveEndPoints) + connection.RawConfig.EndPoints.TryAdd(slaveEndPoint); + Trace(string.Format("Switching master to {0}", newMasterEndPoint)); + // Trigger a reconfigure + connection.ReconfigureAsync(false, false, logProxy, switchBlame, + string.Format("master switch {0}", serviceName), false, CommandFlags.PreferMaster).Wait(); UpdateSentinelAddressList(serviceName); } @@ -2626,6 +2680,7 @@ public void Dispose() { GC.SuppressFinalize(this); Close(!_isDisposed); + sentinelConnection?.Dispose(); } internal Task ExecuteAsyncImpl(Message message, ResultProcessor processor, object state, ServerEndPoint server) diff --git a/src/StackExchange.Redis/EndPointCollection.cs b/src/StackExchange.Redis/EndPointCollection.cs index 8b25930ad..2ec0b5fe0 100644 --- a/src/StackExchange.Redis/EndPointCollection.cs +++ b/src/StackExchange.Redis/EndPointCollection.cs @@ -9,7 +9,7 @@ namespace StackExchange.Redis /// /// A list of endpoints /// - public sealed class EndPointCollection : Collection, IEnumerable, IEnumerable + public sealed class EndPointCollection : Collection, IEnumerable { /// /// Create a new EndPointCollection diff --git a/src/StackExchange.Redis/Enums/RedisCommand.cs b/src/StackExchange.Redis/Enums/RedisCommand.cs index 5f5602517..0ced54933 100644 --- a/src/StackExchange.Redis/Enums/RedisCommand.cs +++ b/src/StackExchange.Redis/Enums/RedisCommand.cs @@ -121,6 +121,7 @@ internal enum RedisCommand RENAME, RENAMENX, RESTORE, + ROLE, RPOP, RPOPLPUSH, RPUSH, diff --git a/src/StackExchange.Redis/Interfaces/IServer.cs b/src/StackExchange.Redis/Interfaces/IServer.cs index 2064b6405..7427bf8c2 100644 --- a/src/StackExchange.Redis/Interfaces/IServer.cs +++ b/src/StackExchange.Redis/Interfaces/IServer.cs @@ -638,6 +638,22 @@ public partial interface IServer : IRedis /// https://redis.io/commands/time Task TimeAsync(CommandFlags flags = CommandFlags.None); + /// + /// The ROLE command returns the current server role which can be master, slave or sentinel. + /// + /// The command flags to use. + /// The server's current role. + /// https://redis.io/commands/role + string Role(CommandFlags flags = CommandFlags.None); + + /// + /// The ROLE command returns the current server role which can be master, slave or sentinel. + /// + /// The command flags to use. + /// The server's current role. + /// https://redis.io/commands/role + Task RoleAsync(CommandFlags flags = CommandFlags.None); + /// /// Gets a text-based latency diagnostic /// @@ -733,7 +749,7 @@ public partial interface IServer : IRedis #region Sentinel /// - /// Returns the ip and port number of the master with that name. + /// Returns the ip and port number of the master with that name. /// If a failover is in progress or terminated successfully for this master it returns the address and port of the promoted slave. /// /// The sentinel service name. @@ -743,7 +759,7 @@ public partial interface IServer : IRedis EndPoint SentinelGetMasterAddressByName(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Returns the ip and port number of the master with that name. + /// Returns the ip and port number of the master with that name. /// If a failover is in progress or terminated successfully for this master it returns the address and port of the promoted slave. /// /// The sentinel service name. @@ -770,6 +786,24 @@ public partial interface IServer : IRedis /// a list of the sentinel ips and ports Task SentinelGetSentinelAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None); + /// + /// Returns the ip and port numbers of all known Sentinel slaves + /// for the given service name. + /// + /// the sentinel service name + /// + /// a list of the slave ips and ports + EndPoint[] SentinelGetSlaveAddresses(string serviceName, CommandFlags flags = CommandFlags.None); + + /// + /// Returns the ip and port numbers of all known Sentinel slaves + /// for the given service name. + /// + /// the sentinel service name + /// + /// a list of the slave ips and ports + Task SentinelGetSlaveAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None); + /// /// Show the state and info of the specified master. /// @@ -823,7 +857,7 @@ public partial interface IServer : IRedis Task[][]> SentinelSlavesAsync(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels + /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels /// (however a new version of the configuration will be published so that the other Sentinels will update their configurations). /// /// The sentinel service name. @@ -832,7 +866,7 @@ public partial interface IServer : IRedis void SentinelFailover(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels + /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels /// (however a new version of the configuration will be published so that the other Sentinels will update their configurations). /// /// The sentinel service name. diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index e340e71cd..77e823a68 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -571,6 +571,7 @@ internal static bool RequiresDatabase(RedisCommand command) case RedisCommand.QUIT: case RedisCommand.READONLY: case RedisCommand.READWRITE: + case RedisCommand.ROLE: case RedisCommand.SAVE: case RedisCommand.SCRIPT: case RedisCommand.SHUTDOWN: diff --git a/src/StackExchange.Redis/RedisLiterals.cs b/src/StackExchange.Redis/RedisLiterals.cs index 1dda150cc..3ec0d9c4c 100644 --- a/src/StackExchange.Redis/RedisLiterals.cs +++ b/src/StackExchange.Redis/RedisLiterals.cs @@ -123,7 +123,9 @@ public static readonly RedisValue pubsub = "pubsub", replication = "replication", server = "server", + master = "master", slave = "slave", + sentinel = "sentinel", slave_read_only = "slave-read-only", timeout = "timeout", yes = "yes"; diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index f66236d85..3455c02d2 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -543,6 +543,18 @@ public Task TimeAsync(CommandFlags flags = CommandFlags.None) return ExecuteAsync(msg, ResultProcessor.DateTime); } + public string Role(CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(-1, flags, RedisCommand.ROLE); + return ExecuteSync(msg, ResultProcessor.Role); + } + + public Task RoleAsync(CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(-1, flags, RedisCommand.ROLE); + return ExecuteAsync(msg, ResultProcessor.Role); + } + internal static Message CreateSlaveOfMessage(EndPoint endpoint, CommandFlags flags = CommandFlags.None) { RedisValue host, port; @@ -819,6 +831,18 @@ public Task SentinelGetSentinelAddressesAsync(string serviceName, Co return ExecuteAsync(msg, ResultProcessor.SentinelAddressesEndPoints); } + public EndPoint[] SentinelGetSlaveAddresses(string serviceName, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(-1, flags, RedisCommand.SENTINEL, RedisLiterals.SLAVES, (RedisValue)serviceName); + return ExecuteSync(msg, ResultProcessor.SentinelAddressesEndPoints); + } + + public Task SentinelGetSlaveAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(-1, flags, RedisCommand.SENTINEL, RedisLiterals.SLAVES, (RedisValue)serviceName); + return ExecuteAsync(msg, ResultProcessor.SentinelAddressesEndPoints); + } + public KeyValuePair[] SentinelMaster(string serviceName, CommandFlags flags = CommandFlags.None) { var msg = Message.Create(-1, flags, RedisCommand.SENTINEL, RedisLiterals.MASTER, (RedisValue)serviceName); diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 478c0df27..1cbc98421 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -85,6 +85,9 @@ public static readonly ResultProcessor public static readonly ResultProcessor ResponseTimer = new TimingProcessor(); + public static readonly ResultProcessor + Role = new RoleProcessor(); + public static readonly ResultProcessor ScriptResult = new ScriptResultProcessor(); @@ -128,6 +131,9 @@ public static readonly ResultProcessor public static readonly ResultProcessor SentinelAddressesEndPoints = new SentinelGetSentinelAddresses(); + public static readonly ResultProcessor + SentinelSlaveEndPoints = new SentinelGetSlaveAddresses(); + public static readonly ResultProcessor[][]> SentinelArrayOfArrays = new SentinelArrayOfArraysProcessor(); @@ -892,6 +898,22 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } + private sealed class RoleProcessor : ResultProcessor + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + switch (result.Type) + { + case ResultType.MultiBulk: + var arr = result.GetItems(); + var role = arr[0].GetString(); + SetResult(message, role); + return true; + } + return false; + } + } + private sealed class ConnectionIdentityProcessor : ResultProcessor { protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) @@ -2043,6 +2065,61 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } + private sealed class SentinelGetSlaveAddresses : ResultProcessor + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + List endPoints = new List(); + + switch (result.Type) + { + case ResultType.MultiBulk: + foreach (RawResult item in result.GetItems()) + { + var arr = item.GetItemsAsValues(); + string ip = null; + string portStr = null; + + for (int i = 0; i < arr.Length && (ip == null || portStr == null); i += 2) + { + string name = arr[i]; + string value = arr[i + 1]; + + switch (name) + { + case "ip": + ip = value; + break; + case "port": + portStr = value; + break; + } + } + + if (ip != null && portStr != null && int.TryParse(portStr, out int port)) + { + endPoints.Add(Format.ParseEndPoint(ip, port)); + } + } + break; + + case ResultType.SimpleString: + //We don't want to blow up if the master is not found + if (result.IsNull) + return true; + break; + } + + if (endPoints.Count > 0) + { + SetResult(message, endPoints.ToArray()); + return true; + } + + return false; + } + } + private sealed class SentinelArrayOfArraysProcessor : ResultProcessor[][]> { protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index ce7b8a1a7..742ac3fae 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; +using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -52,13 +54,27 @@ public Sentinel(ITestOutputHelper output) : base(output) [Fact] public async Task MasterConnectWithConnectionStringFailoverTest() { - var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName}"; + var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName},allowAdmin=true"; var conn = ConnectionMultiplexer.Connect(connectionString); + + // should have 1 master and 1 slave + var endpoints = conn.GetEndPoints(); + Assert.Equal(2, endpoints.Length); + + var servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); + Assert.Equal(2, servers.Length); + Assert.Single(servers, s => s.IsSlave); + + var server1 = servers.First(); + var server2 = servers.Last(); + Assert.Equal("master", server1.Role()); + Assert.Equal("slave", server2.Role()); + conn.ConfigurationChanged += (s, e) => { Log($"Configuration changed: {e.EndPoint}"); }; - var db = conn.GetDatabase(); + var db = conn.GetDatabase(); var test = db.Ping(); Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA, test.TotalMilliseconds); @@ -70,13 +86,38 @@ public async Task MasterConnectWithConnectionStringFailoverTest() db.KeyDelete(key, CommandFlags.FireAndForget); db.StringSet(key, expected); + // force read from slave + var value = db.StringGet(key, CommandFlags.DemandSlave); + Assert.Equal(expected, value); + // forces and verifies failover + var sw = Stopwatch.StartNew(); await DoFailoverAsync(); - var value = db.StringGet(key); + endpoints = conn.GetEndPoints(); + Assert.Equal(2, endpoints.Length); + + servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); + Assert.Equal(2, servers.Length); + + server1 = servers.First(); + server2 = servers.Last(); + + // check to make sure roles have swapped + Assert.Equal("master", server2.Role()); + while (server1.Role() != "slave" || sw.Elapsed > TimeSpan.FromSeconds(30)) + { + await Task.Delay(1000); + } + Log($"Time to swap: {sw.Elapsed}"); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(30)); + + value = db.StringGet(key); Assert.Equal(expected, value); db.StringSet(key, expected); + + conn.Dispose(); } [Fact] From e5f58ed8fcb2a64dd64d65ae124c939fa3480db3 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 8 May 2020 01:03:00 -0500 Subject: [PATCH 14/23] Make test more reliable --- tests/StackExchange.Redis.Tests/Sentinel.cs | 51 +++++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index 742ac3fae..b1041b416 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -63,12 +63,11 @@ public async Task MasterConnectWithConnectionStringFailoverTest() var servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); Assert.Equal(2, servers.Length); - Assert.Single(servers, s => s.IsSlave); var server1 = servers.First(); var server2 = servers.Last(); Assert.Equal("master", server1.Role()); - Assert.Equal("slave", server2.Role()); + Assert.True(await WaitForRoleAsync(server2, "slave")); conn.ConfigurationChanged += (s, e) => { Log($"Configuration changed: {e.EndPoint}"); @@ -80,13 +79,14 @@ public async Task MasterConnectWithConnectionStringFailoverTest() TestConfig.Current.SentinelPortA, test.TotalMilliseconds); // set string value on current master - var expected = DateTime.Now.Ticks.ToString(); - Log("Tick Key: " + expected); + var expected = DateTime.Now.ToShortTimeString(); var key = Me(); + Log(string.Concat(key, ":", expected)); db.KeyDelete(key, CommandFlags.FireAndForget); db.StringSet(key, expected); - // force read from slave + // force read from slave, replication has some lag + await WaitForReplication(server1); var value = db.StringGet(key, CommandFlags.DemandSlave); Assert.Equal(expected, value); @@ -105,12 +105,9 @@ public async Task MasterConnectWithConnectionStringFailoverTest() // check to make sure roles have swapped Assert.Equal("master", server2.Role()); - while (server1.Role() != "slave" || sw.Elapsed > TimeSpan.FromSeconds(30)) - { - await Task.Delay(1000); - } + Assert.True(await WaitForRoleAsync(server1, "slave")); Log($"Time to swap: {sw.Elapsed}"); - Assert.True(sw.Elapsed < TimeSpan.FromSeconds(30)); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(60)); value = db.StringGet(key); Assert.Equal(expected, value); @@ -814,5 +811,39 @@ private async Task DoFailoverAsync() Assert.Equal(slaves[0].ToDictionary()["name"], newMaster.ToString()); Assert.Equal(master.ToString(), newSlave[0].ToDictionary()["name"]); } + + private async Task WaitForRoleAsync(IServer server, string role) + { + var sw = Stopwatch.StartNew(); + while (sw.Elapsed < TimeSpan.FromSeconds(30)) + { + if (await server.RoleAsync() == role) + return true; + + await Task.Delay(1000); + } + + return false; + } + + private async Task WaitForReplication(IServer master) + { + var sw = Stopwatch.StartNew(); + while (sw.Elapsed < TimeSpan.FromSeconds(10)) + { + var info = await master.InfoAsync("replication"); + var replicationInfo = info.FirstOrDefault(f => f.Key == "Replication").ToArray().ToDictionary(); + var slaveInfo = replicationInfo.FirstOrDefault(i => i.Key.StartsWith("slave")).Value?.Split(',').ToDictionary(i => i.Split('=').First(), i => i.Split('=').Last()); + var slaveOffset = slaveInfo?["offset"]; + var masterOffset = replicationInfo["master_repl_offset"]; + + if (slaveOffset == masterOffset) + return true; + + await Task.Delay(200); + } + + return false; + } } } From 9883862bb295f4712df89e15515417113de25ca0 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 8 May 2020 01:09:32 -0500 Subject: [PATCH 15/23] Revert some unintended whitespace changes --- src/StackExchange.Redis/Interfaces/IServer.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/StackExchange.Redis/Interfaces/IServer.cs b/src/StackExchange.Redis/Interfaces/IServer.cs index 7427bf8c2..1bef4f11a 100644 --- a/src/StackExchange.Redis/Interfaces/IServer.cs +++ b/src/StackExchange.Redis/Interfaces/IServer.cs @@ -749,7 +749,7 @@ public partial interface IServer : IRedis #region Sentinel /// - /// Returns the ip and port number of the master with that name. + /// Returns the ip and port number of the master with that name. /// If a failover is in progress or terminated successfully for this master it returns the address and port of the promoted slave. /// /// The sentinel service name. @@ -759,7 +759,7 @@ public partial interface IServer : IRedis EndPoint SentinelGetMasterAddressByName(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Returns the ip and port number of the master with that name. + /// Returns the ip and port number of the master with that name. /// If a failover is in progress or terminated successfully for this master it returns the address and port of the promoted slave. /// /// The sentinel service name. @@ -857,7 +857,7 @@ public partial interface IServer : IRedis Task[][]> SentinelSlavesAsync(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels + /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels /// (however a new version of the configuration will be published so that the other Sentinels will update their configurations). /// /// The sentinel service name. @@ -866,7 +866,7 @@ public partial interface IServer : IRedis void SentinelFailover(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels + /// Force a failover as if the master was not reachable, and without asking for agreement to other Sentinels /// (however a new version of the configuration will be published so that the other Sentinels will update their configurations). /// /// The sentinel service name. From c8107e1188ab4ad3b5abeabd8f9f69447841ba01 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 10 May 2020 21:03:48 -0500 Subject: [PATCH 16/23] Couple small updates from feedback --- src/StackExchange.Redis/Interfaces/IServer.cs | 8 ++++---- src/StackExchange.Redis/RedisLiterals.cs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/StackExchange.Redis/Interfaces/IServer.cs b/src/StackExchange.Redis/Interfaces/IServer.cs index 1bef4f11a..646f9cd38 100644 --- a/src/StackExchange.Redis/Interfaces/IServer.cs +++ b/src/StackExchange.Redis/Interfaces/IServer.cs @@ -773,7 +773,7 @@ public partial interface IServer : IRedis /// for the given service name. /// /// the sentinel service name - /// + /// The command flags to use. /// a list of the sentinel ips and ports EndPoint[] SentinelGetSentinelAddresses(string serviceName, CommandFlags flags = CommandFlags.None); @@ -782,7 +782,7 @@ public partial interface IServer : IRedis /// for the given service name. /// /// the sentinel service name - /// + /// The command flags to use. /// a list of the sentinel ips and ports Task SentinelGetSentinelAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None); @@ -791,7 +791,7 @@ public partial interface IServer : IRedis /// for the given service name. /// /// the sentinel service name - /// + /// The command flags to use. /// a list of the slave ips and ports EndPoint[] SentinelGetSlaveAddresses(string serviceName, CommandFlags flags = CommandFlags.None); @@ -800,7 +800,7 @@ public partial interface IServer : IRedis /// for the given service name. /// /// the sentinel service name - /// + /// The command flags to use. /// a list of the slave ips and ports Task SentinelGetSlaveAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None); diff --git a/src/StackExchange.Redis/RedisLiterals.cs b/src/StackExchange.Redis/RedisLiterals.cs index 3ec0d9c4c..6ac09c1de 100644 --- a/src/StackExchange.Redis/RedisLiterals.cs +++ b/src/StackExchange.Redis/RedisLiterals.cs @@ -118,14 +118,14 @@ public static readonly RedisValue // misc (config, etc) databases = "databases", + master = "master", no = "no", normal = "normal", pubsub = "pubsub", replication = "replication", + sentinel = "sentinel", server = "server", - master = "master", slave = "slave", - sentinel = "sentinel", slave_read_only = "slave-read-only", timeout = "timeout", yes = "yes"; From 3017b81a03f966773eea222da80e786fb1ccedb5 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 11 May 2020 21:05:13 -0500 Subject: [PATCH 17/23] Remove Role from IServer, wait for connect timeout --- .../ConnectionMultiplexer.cs | 7 +++--- src/StackExchange.Redis/Interfaces/IServer.cs | 25 +++++++------------ src/StackExchange.Redis/RedisServer.cs | 12 --------- src/StackExchange.Redis/ResultProcessor.cs | 19 -------------- tests/StackExchange.Redis.Tests/Sentinel.cs | 2 +- 5 files changed, 14 insertions(+), 51 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index a1cee4d10..cc6917066 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -2354,6 +2354,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co bool success = false; ConnectionMultiplexer connection = null; + var sw = Stopwatch.StartNew(); do { attempts++; @@ -2390,14 +2391,14 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co // verify role is master according to: // https://redis.io/topics/sentinel-clients - if (connection.GetServer(newMasterEndPoint).Role() == RedisLiterals.master) + if (connection.GetServer(newMasterEndPoint)?.Role() == RedisLiterals.master) { success = true; break; } Thread.Sleep(100); - } while (attempts < 3); + } while (sw.ElapsedMilliseconds < config.ConnectTimeout); if (!success) { diff --git a/src/StackExchange.Redis/Interfaces/IServer.cs b/src/StackExchange.Redis/Interfaces/IServer.cs index 646f9cd38..435605f39 100644 --- a/src/StackExchange.Redis/Interfaces/IServer.cs +++ b/src/StackExchange.Redis/Interfaces/IServer.cs @@ -638,22 +638,6 @@ public partial interface IServer : IRedis /// https://redis.io/commands/time Task TimeAsync(CommandFlags flags = CommandFlags.None); - /// - /// The ROLE command returns the current server role which can be master, slave or sentinel. - /// - /// The command flags to use. - /// The server's current role. - /// https://redis.io/commands/role - string Role(CommandFlags flags = CommandFlags.None); - - /// - /// The ROLE command returns the current server role which can be master, slave or sentinel. - /// - /// The command flags to use. - /// The server's current role. - /// https://redis.io/commands/role - Task RoleAsync(CommandFlags flags = CommandFlags.None); - /// /// Gets a text-based latency diagnostic /// @@ -1001,5 +985,14 @@ internal static class IServerExtensions /// /// The server to simulate failure on. public static void SimulateConnectionFailure(this IServer server) => (server as RedisServer)?.SimulateConnectionFailure(); + + public static string Role(this IServer server) + { + var result = (RedisResult[])server.Execute("ROLE"); + if (result != null && result.Length > 0) + return result[0].ToString(); + + return null; + } } } diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index 3455c02d2..53a26c4c3 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -543,18 +543,6 @@ public Task TimeAsync(CommandFlags flags = CommandFlags.None) return ExecuteAsync(msg, ResultProcessor.DateTime); } - public string Role(CommandFlags flags = CommandFlags.None) - { - var msg = Message.Create(-1, flags, RedisCommand.ROLE); - return ExecuteSync(msg, ResultProcessor.Role); - } - - public Task RoleAsync(CommandFlags flags = CommandFlags.None) - { - var msg = Message.Create(-1, flags, RedisCommand.ROLE); - return ExecuteAsync(msg, ResultProcessor.Role); - } - internal static Message CreateSlaveOfMessage(EndPoint endpoint, CommandFlags flags = CommandFlags.None) { RedisValue host, port; diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 1cbc98421..6219c1aab 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -85,9 +85,6 @@ public static readonly ResultProcessor public static readonly ResultProcessor ResponseTimer = new TimingProcessor(); - public static readonly ResultProcessor - Role = new RoleProcessor(); - public static readonly ResultProcessor ScriptResult = new ScriptResultProcessor(); @@ -898,22 +895,6 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } - private sealed class RoleProcessor : ResultProcessor - { - protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) - { - switch (result.Type) - { - case ResultType.MultiBulk: - var arr = result.GetItems(); - var role = arr[0].GetString(); - SetResult(message, role); - return true; - } - return false; - } - } - private sealed class ConnectionIdentityProcessor : ResultProcessor { protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index b1041b416..776fe2924 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -817,7 +817,7 @@ private async Task WaitForRoleAsync(IServer server, string role) var sw = Stopwatch.StartNew(); while (sw.Elapsed < TimeSpan.FromSeconds(30)) { - if (await server.RoleAsync() == role) + if (server.Role() == role) return true; await Task.Delay(1000); From 495cd3e585dabb2e117e4279ca78c53e0368e759 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 12 May 2020 18:44:11 -0500 Subject: [PATCH 18/23] Remove role from more spots. Cleanup tests and make them more reliable. Make SentinelMasterConnect private. --- src/StackExchange.Redis/CommandMap.cs | 2 +- .../ConnectionMultiplexer.cs | 8 +- src/StackExchange.Redis/Enums/RedisCommand.cs | 1 - src/StackExchange.Redis/Message.cs | 1 - tests/StackExchange.Redis.Tests/Sentinel.cs | 554 ++++++------------ 5 files changed, 182 insertions(+), 384 deletions(-) diff --git a/src/StackExchange.Redis/CommandMap.cs b/src/StackExchange.Redis/CommandMap.cs index a47d96860..5d2178d48 100644 --- a/src/StackExchange.Redis/CommandMap.cs +++ b/src/StackExchange.Redis/CommandMap.cs @@ -72,7 +72,7 @@ internal CommandMap(CommandBytes[] map) /// https://redis.io/topics/sentinel public static CommandMap Sentinel { get; } = Create(new HashSet { // see https://redis.io/topics/sentinel - "auth", "ping", "info", "role", "sentinel", "subscribe", "shutdown", "psubscribe", "unsubscribe", "punsubscribe" }, true); + "auth", "ping", "info", "sentinel", "subscribe", "shutdown", "psubscribe", "unsubscribe", "punsubscribe" }, true); /// /// Create a new CommandMap, customizing some commands diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index cc6917066..13afd0605 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1086,7 +1086,7 @@ public static Task SentinelConnectAsync(ConfigurationOpti /// /// The string configuration to use for this multiplexer. /// The to log to. - public static ConnectionMultiplexer SentinelMasterConnect(string configuration, TextWriter log = null) + private static ConnectionMultiplexer SentinelMasterConnect(string configuration, TextWriter log = null) { return SentinelMasterConnect(PrepareConfig(configuration, sentinel: true), log); } @@ -1097,7 +1097,7 @@ public static ConnectionMultiplexer SentinelMasterConnect(string configuration, /// /// The configuration options to use for this multiplexer. /// The to log to. - public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions configuration, TextWriter log = null) + private static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions configuration, TextWriter log = null) { var sentinelConnection = SentinelConnect(configuration, log); @@ -1114,7 +1114,7 @@ public static ConnectionMultiplexer SentinelMasterConnect(ConfigurationOptions c /// /// The string configuration to use for this multiplexer. /// The to log to. - public static Task SentinelMasterConnectAsync(string configuration, TextWriter log = null) + private static Task SentinelMasterConnectAsync(string configuration, TextWriter log = null) { return SentinelMasterConnectAsync(PrepareConfig(configuration, sentinel: true), log); } @@ -1125,7 +1125,7 @@ public static Task SentinelMasterConnectAsync(string conf /// /// The configuration options to use for this multiplexer. /// The to log to. - public static async Task SentinelMasterConnectAsync(ConfigurationOptions configuration, TextWriter log = null) + private static async Task SentinelMasterConnectAsync(ConfigurationOptions configuration, TextWriter log = null) { var sentinelConnection = await SentinelConnectAsync(configuration, log).ForAwait(); diff --git a/src/StackExchange.Redis/Enums/RedisCommand.cs b/src/StackExchange.Redis/Enums/RedisCommand.cs index 0ced54933..5f5602517 100644 --- a/src/StackExchange.Redis/Enums/RedisCommand.cs +++ b/src/StackExchange.Redis/Enums/RedisCommand.cs @@ -121,7 +121,6 @@ internal enum RedisCommand RENAME, RENAMENX, RESTORE, - ROLE, RPOP, RPOPLPUSH, RPUSH, diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 77e823a68..e340e71cd 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -571,7 +571,6 @@ internal static bool RequiresDatabase(RedisCommand command) case RedisCommand.QUIT: case RedisCommand.READONLY: case RedisCommand.READWRITE: - case RedisCommand.ROLE: case RedisCommand.SAVE: case RedisCommand.SCRIPT: case RedisCommand.SHUTDOWN: diff --git a/tests/StackExchange.Redis.Tests/Sentinel.cs b/tests/StackExchange.Redis.Tests/Sentinel.cs index 776fe2924..f1d04002f 100644 --- a/tests/StackExchange.Redis.Tests/Sentinel.cs +++ b/tests/StackExchange.Redis.Tests/Sentinel.cs @@ -48,88 +48,107 @@ public Sentinel(ITestOutputHelper output) : base(output) SentinelServerA = Conn.GetServer(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); SentinelServerB = Conn.GetServer(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortB); SentinelServerC = Conn.GetServer(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortC); - SentinelsServers = new IServer[] { SentinelServerA, SentinelServerB, SentinelServerC }; + SentinelsServers = new[] { SentinelServerA, SentinelServerB, SentinelServerC }; + + // wait until we are in a state of a single master and slave + WaitForReady(); } [Fact] - public async Task MasterConnectWithConnectionStringFailoverTest() + public void MasterConnectTest() { - var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName},allowAdmin=true"; + var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},serviceName={ServiceOptions.ServiceName},allowAdmin=true"; var conn = ConnectionMultiplexer.Connect(connectionString); - // should have 1 master and 1 slave + var db = conn.GetDatabase(); + db.Ping(); + var endpoints = conn.GetEndPoints(); Assert.Equal(2, endpoints.Length); var servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); Assert.Equal(2, servers.Length); - var server1 = servers.First(); - var server2 = servers.Last(); - Assert.Equal("master", server1.Role()); - Assert.True(await WaitForRoleAsync(server2, "slave")); - - conn.ConfigurationChanged += (s, e) => { - Log($"Configuration changed: {e.EndPoint}"); - }; + var master = servers.FirstOrDefault(s => !s.IsSlave); + Assert.NotNull(master); + var slave = servers.FirstOrDefault(s => s.IsSlave); + Assert.NotNull(slave); + Assert.NotEqual(master.EndPoint.ToString(), slave.EndPoint.ToString()); - var db = conn.GetDatabase(); - var test = db.Ping(); - Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, - TestConfig.Current.SentinelPortA, test.TotalMilliseconds); - - // set string value on current master - var expected = DateTime.Now.ToShortTimeString(); + var expected = DateTime.Now.Ticks.ToString(); + Log("Tick Key: " + expected); var key = Me(); - Log(string.Concat(key, ":", expected)); db.KeyDelete(key, CommandFlags.FireAndForget); db.StringSet(key, expected); + var value = db.StringGet(key); + Assert.Equal(expected, value); + // force read from slave, replication has some lag - await WaitForReplication(server1); - var value = db.StringGet(key, CommandFlags.DemandSlave); + WaitForReplication(servers.First()); + value = db.StringGet(key, CommandFlags.DemandSlave); Assert.Equal(expected, value); + } - // forces and verifies failover - var sw = Stopwatch.StartNew(); - await DoFailoverAsync(); + [Fact] + public async Task MasterConnectAsyncTest() + { + var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},serviceName={ServiceOptions.ServiceName},allowAdmin=true"; + var conn = await ConnectionMultiplexer.ConnectAsync(connectionString); - endpoints = conn.GetEndPoints(); + var db = conn.GetDatabase(); + await db.PingAsync(); + + var endpoints = conn.GetEndPoints(); Assert.Equal(2, endpoints.Length); - servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); + var servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); Assert.Equal(2, servers.Length); - server1 = servers.First(); - server2 = servers.Last(); + var master = servers.FirstOrDefault(s => !s.IsSlave); + Assert.NotNull(master); + var slave = servers.FirstOrDefault(s => s.IsSlave); + Assert.NotNull(slave); + Assert.NotEqual(master.EndPoint.ToString(), slave.EndPoint.ToString()); - // check to make sure roles have swapped - Assert.Equal("master", server2.Role()); - Assert.True(await WaitForRoleAsync(server1, "slave")); - Log($"Time to swap: {sw.Elapsed}"); - Assert.True(sw.Elapsed < TimeSpan.FromSeconds(60)); + var expected = DateTime.Now.Ticks.ToString(); + Log("Tick Key: " + expected); + var key = Me(); + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + await db.StringSetAsync(key, expected); - value = db.StringGet(key); + var value = await db.StringGetAsync(key); Assert.Equal(expected, value); - db.StringSet(key, expected); - - conn.Dispose(); + // force read from slave, replication has some lag + WaitForReplication(servers.First()); + value = await db.StringGetAsync(key, CommandFlags.DemandSlave); + Assert.Equal(expected, value); } [Fact] - public async Task MasterConnectAsyncWithConnectionStringFailoverTest() + public async Task ManagedMasterConnectionEndToEndWithFailoverTest() { - var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName}"; + var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},serviceName={ServiceOptions.ServiceName},allowAdmin=true"; var conn = await ConnectionMultiplexer.ConnectAsync(connectionString); conn.ConfigurationChanged += (s, e) => { Log($"Configuration changed: {e.EndPoint}"); }; + var db = conn.GetDatabase(); + await db.PingAsync(); - var test = await db.PingAsync(); - Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, - TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + var endpoints = conn.GetEndPoints(); + Assert.Equal(2, endpoints.Length); + + var servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); + Assert.Equal(2, servers.Length); + + var master = servers.FirstOrDefault(s => !s.IsSlave); + Assert.NotNull(master); + var slave = servers.FirstOrDefault(s => s.IsSlave); + Assert.NotNull(slave); + Assert.NotEqual(master.EndPoint.ToString(), slave.EndPoint.ToString()); // set string value on current master var expected = DateTime.Now.Ticks.ToString(); @@ -138,39 +157,38 @@ public async Task MasterConnectAsyncWithConnectionStringFailoverTest() await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); await db.StringSetAsync(key, expected); - // forces and verifies failover - await DoFailoverAsync(); - var value = await db.StringGetAsync(key); Assert.Equal(expected, value); - await db.StringSetAsync(key, expected); - } + // force read from slave, replication has some lag + WaitForReplication(servers.First()); + value = await db.StringGetAsync(key, CommandFlags.DemandSlave); + Assert.Equal(expected, value); - [Fact] - public void MasterConnectWithDefaultPortTest() - { - var options = ServiceOptions.Clone(); - options.EndPoints.Add(TestConfig.Current.SentinelServer); + // forces and verifies failover + DoFailover(); - var conn = ConnectionMultiplexer.SentinelMasterConnect(options); - var db = conn.GetDatabase(); + endpoints = conn.GetEndPoints(); + Assert.Equal(2, endpoints.Length); - var test = db.Ping(); - Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, - TestConfig.Current.SentinelPortA, test.TotalMilliseconds); - } + servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); + Assert.Equal(2, servers.Length); - [Fact] - public void MasterConnectWithStringConfigurationTest() - { - var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},password={ServiceOptions.Password},serviceName={ServiceOptions.ServiceName}"; - var conn = ConnectionMultiplexer.Connect(connectionString); - var db = conn.GetDatabase(); + var newMaster = servers.FirstOrDefault(s => !s.IsSlave); + Assert.NotNull(newMaster); + Assert.Equal(slave.EndPoint.ToString(), newMaster.EndPoint.ToString()); + var newSlave = servers.FirstOrDefault(s => s.IsSlave); + Assert.NotNull(newSlave); + Assert.Equal(master.EndPoint.ToString(), newSlave.EndPoint.ToString()); + Assert.NotEqual(master.EndPoint.ToString(), slave.EndPoint.ToString()); - var test = db.Ping(); - Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, - TestConfig.Current.SentinelPortA, test.TotalMilliseconds); + value = await db.StringGetAsync(key); + Assert.Equal(expected, value); + + // force read from slave, replication has some lag + WaitForReplication(newMaster); + value = await db.StringGetAsync(key, CommandFlags.DemandSlave); + Assert.Equal(expected, value); } [Fact] @@ -201,39 +219,6 @@ public async Task SentinelConnectAsyncTest() TestConfig.Current.SentinelPortA, test.TotalMilliseconds); } - [Fact] - public async Task MasterConnectFailoverTest() - { - var options = ServiceOptions.Clone(); - options.EndPoints.Add(TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA); - - // connection is managed and should switch to current master when failover happens - var conn = ConnectionMultiplexer.SentinelMasterConnect(options); - conn.ConfigurationChanged += (s, e) => { - Log($"Configuration changed: {e.EndPoint}"); - }; - var db = conn.GetDatabase(); - - var test = await db.PingAsync(); - Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, - TestConfig.Current.SentinelPortA, test.TotalMilliseconds); - - // set string value on current master - var expected = DateTime.Now.Ticks.ToString(); - Log("Tick Key: " + expected); - var key = Me(); - await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); - await db.StringSetAsync(key, expected); - - // forces and verifies failover - await DoFailoverAsync(); - - var value = await db.StringGetAsync(key); - Assert.Equal(expected, value); - - await db.StringSetAsync(key, expected); - } - [Fact] public void PingTest() { @@ -279,6 +264,7 @@ public async Task SentinelGetMasterAddressByNameAsyncTest() Log("{0}:{1}", ipEndPoint.Address, ipEndPoint.Port); } } + [Fact] public void SentinelGetMasterAddressByNameNegativeTest() { @@ -342,7 +328,6 @@ private class IpComparer : IEqualityComparer public void SentinelSentinelsTest() { var sentinels = SentinelServerA.SentinelSentinels(ServiceName); - var Server26380Info = SentinelServerB.Info(); var expected = new List { SentinelServerB.EndPoint.ToString(), @@ -510,234 +495,6 @@ public async Task SentinelSlavesAsyncTest() } } - [Fact] - public async Task SentinelFailoverTest() - { - var i = 0; - foreach (var server in SentinelsServers) - { - Log("Failover: " + i++); - var master = server.SentinelGetMasterAddressByName(ServiceName); - var slaves = server.SentinelSlaves(ServiceName); - - await Task.Delay(1000).ForAwait(); - try - { - Log("Failover attempted initiated"); - server.SentinelFailover(ServiceName); - Log(" Success!"); - } - catch (RedisServerException ex) when (ex.Message.Contains("NOGOODSLAVE")) - { - // Retry once - Log(" Retry initiated"); - await Task.Delay(1000).ForAwait(); - server.SentinelFailover(ServiceName); - Log(" Retry complete"); - } - await Task.Delay(2000).ForAwait(); - - var newMaster = server.SentinelGetMasterAddressByName(ServiceName); - var newSlave = server.SentinelSlaves(ServiceName); - - Assert.Equal(slaves[0].ToDictionary()["name"], newMaster.ToString()); - Assert.Equal(master.ToString(), newSlave[0].ToDictionary()["name"]); - } - } - - [Fact] - public async Task SentinelFailoverAsyncTest() - { - var i = 0; - foreach (var server in SentinelsServers) - { - Log("Failover: " + i++); - var master = server.SentinelGetMasterAddressByName(ServiceName); - var slaves = server.SentinelSlaves(ServiceName); - - await Task.Delay(1000).ForAwait(); - try - { - Log("Failover attempted initiated"); - await server.SentinelFailoverAsync(ServiceName).ForAwait(); - Log(" Success!"); - } - catch (RedisServerException ex) when (ex.Message.Contains("NOGOODSLAVE")) - { - // Retry once - Log(" Retry initiated"); - await Task.Delay(1000).ForAwait(); - await server.SentinelFailoverAsync(ServiceName).ForAwait(); - Log(" Retry complete"); - } - await Task.Delay(2000).ForAwait(); - - var newMaster = server.SentinelGetMasterAddressByName(ServiceName); - var newSlave = server.SentinelSlaves(ServiceName); - - Assert.Equal(slaves[0].ToDictionary()["name"], newMaster.ToString()); - Assert.Equal(master.ToString(), newSlave[0].ToDictionary()["name"]); - } - } - -#if DEBUG - [Fact] - public async Task GetSentinelMasterConnectionFailoverTest() - { - var conn = Conn.GetSentinelMasterConnection(ServiceOptions); - var endpoint = conn.currentSentinelMasterEndPoint.ToString(); - - try - { - Log("Failover attempted initiated"); - SentinelServerA.SentinelFailover(ServiceName); - Log(" Success!"); - } - catch (RedisServerException ex) when (ex.Message.Contains("NOGOODSLAVE")) - { - // Retry once - Log(" Retry initiated"); - await Task.Delay(1000).ForAwait(); - SentinelServerA.SentinelFailover(ServiceName); - Log(" Retry complete"); - } - await Task.Delay(2000).ForAwait(); - - // Try and complete ASAP - await UntilCondition(TimeSpan.FromSeconds(10), () => { - var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions); - return endpoint != checkConn.currentSentinelMasterEndPoint.ToString(); - }); - - // Post-check for validity - var conn1 = Conn.GetSentinelMasterConnection(ServiceOptions); - Assert.NotEqual(endpoint, conn1.currentSentinelMasterEndPoint.ToString()); - } - - [Fact] - public async Task GetSentinelMasterConnectionFailoverAsyncTest() - { - var conn = Conn.GetSentinelMasterConnection(ServiceOptions); - var endpoint = conn.currentSentinelMasterEndPoint.ToString(); - - try - { - Log("Failover attempted initiated"); - await SentinelServerA.SentinelFailoverAsync(ServiceName).ForAwait(); - Log(" Success!"); - } - catch (RedisServerException ex) when (ex.Message.Contains("NOGOODSLAVE")) - { - // Retry once - Log(" Retry initiated"); - await Task.Delay(1000).ForAwait(); - await SentinelServerA.SentinelFailoverAsync(ServiceName).ForAwait(); - Log(" Retry complete"); - } - - // Try and complete ASAP - await UntilCondition(TimeSpan.FromSeconds(10), () => { - var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions); - return endpoint != checkConn.currentSentinelMasterEndPoint.ToString(); - }); - - // Post-check for validity - var conn1 = Conn.GetSentinelMasterConnection(ServiceOptions); - Assert.NotEqual(endpoint, conn1.currentSentinelMasterEndPoint.ToString()); - } -#endif - - [Fact] - public async Task GetSentinelMasterConnectionWriteReadFailover() - { - Log("Conn:"); - foreach (var server in Conn.GetServerSnapshot().ToArray()) - { - Log(" Endpoint: " + server.EndPoint); - } - Log("Conn Slaves:"); - foreach (var slaves in SentinelServerA.SentinelSlaves(ServiceName)) - { - foreach(var pair in slaves) - { - Log(" {0}: {1}", pair.Key, pair.Value); - } - } - - var conn = Conn.GetSentinelMasterConnection(ServiceOptions); - var s = conn.currentSentinelMasterEndPoint.ToString(); - Log("Sentinel Master Endpoint: " + s); - foreach (var server in conn.GetServerSnapshot().ToArray()) - { - Log(" Server: " + server.EndPoint); - Log(" Master Endpoint: " + server.MasterEndPoint); - Log(" IsSlave: " + server.IsSlave); - Log(" SlaveReadOnly: " + server.SlaveReadOnly); - var info = conn.GetServer(server.EndPoint).Info("Replication"); - foreach (var section in info) - { - Log(" Section: " + section.Key); - foreach (var pair in section) - { - Log(" " + pair.Key +": " + pair.Value); - } - } - } - - IDatabase db = conn.GetDatabase(); - var expected = DateTime.Now.Ticks.ToString(); - Log("Tick Key: " + expected); - var key = Me(); - db.KeyDelete(key, CommandFlags.FireAndForget); - db.StringSet(key, expected); - - await UntilCondition(TimeSpan.FromSeconds(10), - () => SentinelServerA.SentinelMaster(ServiceName).ToDictionary()["num-slaves"] != "0" - ); - Log("Conditions met"); - - try - { - Log("Failover attempted initiated"); - SentinelServerA.SentinelFailover(ServiceName); - Log(" Success!"); - } - catch (RedisServerException ex) when (ex.Message.Contains("NOGOODSLAVE")) - { - // Retry once - Log(" Retry initiated"); - await Task.Delay(1000).ForAwait(); - SentinelServerA.SentinelFailover(ServiceName); - Log(" Retry complete"); - } - Log("Delaying for failover conditions..."); - await Task.Delay(2000).ForAwait(); - Log("Conditons check..."); - // Spin until complete (with a timeout) - since this can vary - await UntilCondition(TimeSpan.FromSeconds(20), () => - { - var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions); - return s != checkConn.currentSentinelMasterEndPoint.ToString() - && expected == checkConn.GetDatabase().StringGet(key); - }); - Log(" Conditions met."); - - var conn1 = Conn.GetSentinelMasterConnection(ServiceOptions); - var s1 = conn1.currentSentinelMasterEndPoint.ToString(); - Log("New master endpoint: " + s1); - - var actual = conn1.GetDatabase().StringGet(key); - Log("Fetched tick key: " + actual); - - Assert.NotNull(s); - Assert.NotNull(s1); - Assert.NotEmpty(s); - Assert.NotEmpty(s1); - Assert.NotEqual(s, s1); - // TODO: Track this down on the test race - //Assert.Equal(expected, actual); - } - [Fact] public async Task SentinelGetSentinelAddressesTest() { @@ -757,19 +514,15 @@ public async Task SentinelGetSentinelAddressesTest() [Fact] public async Task ReadOnlyConnectionSlavesTest() { - var slaves = SentinelServerA.SentinelSlaves(ServiceName); - var config = new ConfigurationOptions - { - Password = ServiceOptions.Password - }; + var slaves = SentinelServerA.SentinelGetSlaveAddresses(ServiceName); + var config = new ConfigurationOptions(); - foreach (var kv in slaves) + foreach (var slave in slaves) { - Assert.Equal("slave", kv.ToDictionary()["flags"]); - config.EndPoints.Add(kv.ToDictionary()["name"]); + config.EndPoints.Add(slave); } - var readonlyConn = ConnectionMultiplexer.Connect(config); + var readonlyConn = await ConnectionMultiplexer.ConnectAsync(config); await UntilCondition(TimeSpan.FromSeconds(2), () => readonlyConn.IsConnected); Assert.True(readonlyConn.IsConnected); @@ -781,69 +534,116 @@ public async Task ReadOnlyConnectionSlavesTest() } - private async Task DoFailoverAsync() + private void DoFailover() { - // capture current master and slave - var master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); - var slaves = SentinelServerA.SentinelSlaves(ServiceName); + WaitForReady(); - await Task.Delay(1000).ForAwait(); - try - { - Log("Failover attempted initiated"); - SentinelServerA.SentinelFailover(ServiceName); - Log(" Success!"); - } - catch (RedisServerException ex) when (ex.Message.Contains("NOGOODSLAVE")) + // capture current slave + var slaves = SentinelServerA.SentinelGetSlaveAddresses(ServiceName); + + Log("Starting failover..."); + var sw = Stopwatch.StartNew(); + SentinelServerA.SentinelFailover(ServiceName); + + // wait until the slave becomes the master + WaitForReady(expectedMaster: slaves[0]); + Log($"Time to failover: {sw.Elapsed}"); + } + + private void WaitForReady(EndPoint expectedMaster = null, bool waitForReplication = false, TimeSpan? duration = null) + { + duration ??= TimeSpan.FromSeconds(30); + + var sw = Stopwatch.StartNew(); + + // wait until we have 1 master and 1 slave and have verified their roles + var master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); + if (expectedMaster != null && expectedMaster.ToString() != master.ToString()) { - // Retry once - Log(" Retry initiated"); - await Task.Delay(1000).ForAwait(); - SentinelServerA.SentinelFailover(ServiceName); - Log(" Retry complete"); + while (sw.Elapsed < duration.Value) + { + Thread.Sleep(1000); + try + { + master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); + if (expectedMaster.ToString() == master.ToString()) + break; + } + catch (Exception) + { + // ignore + } + } } - await Task.Delay(2000).ForAwait(); + if (expectedMaster != null && expectedMaster.ToString() != master.ToString()) + throw new RedisException($"Master was expected to be {expectedMaster}"); + Log($"Master is {master}"); + + var slaves = SentinelServerA.SentinelGetSlaveAddresses(ServiceName); + var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions); - var newMaster = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); - var newSlave = SentinelServerA.SentinelSlaves(ServiceName); + WaitForRole(checkConn.GetServer(master), "master", duration.Value.Subtract(sw.Elapsed)); + WaitForRole(checkConn.GetServer(slaves[0]), "slave", duration.Value.Subtract(sw.Elapsed)); - // make sure master changed - Assert.Equal(slaves[0].ToDictionary()["name"], newMaster.ToString()); - Assert.Equal(master.ToString(), newSlave[0].ToDictionary()["name"]); + if (waitForReplication) + { + WaitForReplication(checkConn.GetServer(master), duration.Value.Subtract(sw.Elapsed)); + } } - private async Task WaitForRoleAsync(IServer server, string role) + private void WaitForRole(IServer server, string role, TimeSpan? duration = null) { + duration ??= TimeSpan.FromSeconds(30); + + Log($"Waiting for server ({server.EndPoint}) role to be \"{role}\"..."); var sw = Stopwatch.StartNew(); - while (sw.Elapsed < TimeSpan.FromSeconds(30)) + while (sw.Elapsed < duration.Value) { - if (server.Role() == role) - return true; + try + { + if (server.Role() == role) + { + Log($"Done waiting for server ({server.EndPoint}) role to be \"{role}\""); + return; + } + } + catch (Exception) + { + // ignore + } - await Task.Delay(1000); + Thread.Sleep(1000); } - return false; + throw new RedisException("Timeout waiting for server to have expected role assigned"); } - private async Task WaitForReplication(IServer master) + private void WaitForReplication(IServer master, TimeSpan? duration = null) { + duration ??= TimeSpan.FromSeconds(10); + + Log("Waiting for master/slave replication to be in sync..."); var sw = Stopwatch.StartNew(); - while (sw.Elapsed < TimeSpan.FromSeconds(10)) + while (sw.Elapsed < duration.Value) { - var info = await master.InfoAsync("replication"); - var replicationInfo = info.FirstOrDefault(f => f.Key == "Replication").ToArray().ToDictionary(); - var slaveInfo = replicationInfo.FirstOrDefault(i => i.Key.StartsWith("slave")).Value?.Split(',').ToDictionary(i => i.Split('=').First(), i => i.Split('=').Last()); + var info = master.Info("replication"); + var replicationInfo = info.FirstOrDefault(f => f.Key == "Replication")?.ToArray().ToDictionary(); + var slaveInfo = replicationInfo?.FirstOrDefault(i => i.Key.StartsWith("slave")).Value?.Split(',').ToDictionary(i => i.Split('=').First(), i => i.Split('=').Last()); var slaveOffset = slaveInfo?["offset"]; - var masterOffset = replicationInfo["master_repl_offset"]; + var masterOffset = replicationInfo?["master_repl_offset"]; if (slaveOffset == masterOffset) - return true; + { + Log($"Done waiting for master ({masterOffset}) / slave ({slaveOffset}) replication to be in sync"); + return; + } + + Log($"Waiting for master ({masterOffset}) / slave ({slaveOffset}) replication to be in sync..."); - await Task.Delay(200); + Thread.Sleep(250); } - return false; + throw new RedisException("Timeout waiting for test servers master/slave replication to be in sync."); } } } From 6ad3aec6cd0bfe344365f7e415a07a0a063889f5 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 12 May 2020 19:22:07 -0500 Subject: [PATCH 19/23] Remove unused variable --- src/StackExchange.Redis/ConnectionMultiplexer.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 13afd0605..0577459f5 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2350,15 +2350,12 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co return sentinelConnectionChildren[config.ServiceName]; } - int attempts = 0; bool success = false; ConnectionMultiplexer connection = null; var sw = Stopwatch.StartNew(); do { - attempts++; - // Get an initial endpoint - try twice EndPoint newMasterEndPoint = GetConfiguredMasterForService(config.ServiceName) ?? GetConfiguredMasterForService(config.ServiceName); From 4cd8faff91b1f6a3aaf16e7cd93371117bc5c226 Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Mon, 8 Jun 2020 09:42:48 -0400 Subject: [PATCH 20/23] Merge bits --- src/StackExchange.Redis/ConnectionMultiplexer.cs | 2 +- src/StackExchange.Redis/Interfaces/IServer.cs | 12 ++++++------ src/StackExchange.Redis/RedisServer.cs | 4 ++-- src/StackExchange.Redis/ResultProcessor.cs | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index b8814c499..beb49c1c7 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2524,7 +2524,7 @@ internal EndPoint[] GetSlavesForService(string serviceName) => .AsParallel() .Select(s => { - try { return GetServer(s.EndPoint).SentinelGetSlaveAddresses(serviceName); } + try { return GetServer(s.EndPoint).SentinelGetReplicaAddresses(serviceName); } catch { return null; } }) .FirstOrDefault(r => r != null); diff --git a/src/StackExchange.Redis/Interfaces/IServer.cs b/src/StackExchange.Redis/Interfaces/IServer.cs index 5a6e9516a..f2da646ca 100644 --- a/src/StackExchange.Redis/Interfaces/IServer.cs +++ b/src/StackExchange.Redis/Interfaces/IServer.cs @@ -806,22 +806,22 @@ public partial interface IServer : IRedis Task SentinelGetSentinelAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Returns the ip and port numbers of all known Sentinel slaves + /// Returns the ip and port numbers of all known Sentinel replicas /// for the given service name. /// /// the sentinel service name /// The command flags to use. - /// a list of the slave ips and ports - EndPoint[] SentinelGetSlaveAddresses(string serviceName, CommandFlags flags = CommandFlags.None); + /// a list of the replica ips and ports + EndPoint[] SentinelGetReplicaAddresses(string serviceName, CommandFlags flags = CommandFlags.None); /// - /// Returns the ip and port numbers of all known Sentinel slaves + /// Returns the ip and port numbers of all known Sentinel replicas /// for the given service name. /// /// the sentinel service name /// The command flags to use. - /// a list of the slave ips and ports - Task SentinelGetSlaveAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None); + /// a list of the replica ips and ports + Task SentinelGetReplicaAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None); /// /// Show the state and info of the specified master. diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index af25060ed..3ee83a0e4 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -830,13 +830,13 @@ public Task SentinelGetSentinelAddressesAsync(string serviceName, Co return ExecuteAsync(msg, ResultProcessor.SentinelAddressesEndPoints); } - public EndPoint[] SentinelGetSlaveAddresses(string serviceName, CommandFlags flags = CommandFlags.None) + public EndPoint[] SentinelGetReplicaAddresses(string serviceName, CommandFlags flags = CommandFlags.None) { var msg = Message.Create(-1, flags, RedisCommand.SENTINEL, RedisLiterals.SLAVES, (RedisValue)serviceName); return ExecuteSync(msg, ResultProcessor.SentinelAddressesEndPoints); } - public Task SentinelGetSlaveAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None) + public Task SentinelGetReplicaAddressesAsync(string serviceName, CommandFlags flags = CommandFlags.None) { var msg = Message.Create(-1, flags, RedisCommand.SENTINEL, RedisLiterals.SLAVES, (RedisValue)serviceName); return ExecuteAsync(msg, ResultProcessor.SentinelAddressesEndPoints); diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 253122cbb..d9202b56a 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -129,7 +129,7 @@ public static readonly ResultProcessor SentinelAddressesEndPoints = new SentinelGetSentinelAddresses(); public static readonly ResultProcessor - SentinelSlaveEndPoints = new SentinelGetSlaveAddresses(); + SentinelReplicaEndPoints = new SentinelGetReplicaAddresses(); public static readonly ResultProcessor[][]> SentinelArrayOfArrays = new SentinelArrayOfArraysProcessor(); @@ -2094,7 +2094,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } - private sealed class SentinelGetSlaveAddresses : ResultProcessor + private sealed class SentinelGetReplicaAddresses : ResultProcessor { protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) { From b606faece7d7be95d4e4cc11a36fb8b3cb1920a9 Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Mon, 8 Jun 2020 09:46:39 -0400 Subject: [PATCH 21/23] More cleanup --- .../ConnectionMultiplexer.cs | 22 +++++++++++-------- src/StackExchange.Redis/RedisServer.cs | 2 ++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index beb49c1c7..85b7614fa 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2383,8 +2383,8 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co $"Sentinel: Failed connecting to configured master for service: {config.ServiceName}"); } - EndPoint[] slaveEndPoints = GetSlavesForService(config.ServiceName) - ?? GetSlavesForService(config.ServiceName); + EndPoint[] replicaEndPoints = GetReplicasForService(config.ServiceName) + ?? GetReplicasForService(config.ServiceName); // Replace the master endpoint, if we found another one // If not, assume the last state is the best we have and minimize the race @@ -2398,8 +2398,10 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co config.EndPoints.TryAdd(newMasterEndPoint); } - foreach (var slaveEndPoint in slaveEndPoints) - config.EndPoints.TryAdd(slaveEndPoint); + foreach (var replicaEndPoint in replicaEndPoints) + { + config.EndPoints.TryAdd(replicaEndPoint); + } connection = ConnectImpl(config, log); @@ -2517,7 +2519,7 @@ internal EndPoint GetConfiguredMasterForService(string serviceName) => internal EndPoint currentSentinelMasterEndPoint; - internal EndPoint[] GetSlavesForService(string serviceName) => + internal EndPoint[] GetReplicasForService(string serviceName) => GetServerSnapshot() .ToArray() .Where(s => s.ServerType == ServerType.Sentinel) @@ -2557,14 +2559,16 @@ internal void SwitchMaster(EndPoint switchBlame, ConnectionMultiplexer connectio if (!connection.servers.Contains(newMasterEndPoint)) { - EndPoint[] slaveEndPoints = GetSlavesForService(serviceName) - ?? GetSlavesForService(serviceName); + EndPoint[] replicaEndPoints = GetReplicasForService(serviceName) + ?? GetReplicasForService(serviceName); connection.servers.Clear(); connection.RawConfig.EndPoints.Clear(); connection.RawConfig.EndPoints.TryAdd(newMasterEndPoint); - foreach (var slaveEndPoint in slaveEndPoints) - connection.RawConfig.EndPoints.TryAdd(slaveEndPoint); + foreach (var replicaEndPoint in replicaEndPoints) + { + connection.RawConfig.EndPoints.TryAdd(replicaEndPoint); + }| Trace(string.Format("Switching master to {0}", newMasterEndPoint)); // Trigger a reconfigure connection.ReconfigureAsync(false, false, logProxy, switchBlame, diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index 3ee83a0e4..2f787c5da 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -878,6 +878,7 @@ public Task[][]> SentinelMastersAsync(CommandFlags return ExecuteAsync(msg, ResultProcessor.SentinelArrayOfArrays); } + // For previous compat only KeyValuePair[][] IServer.SentinelSlaves(string serviceName, CommandFlags flags) => SentinelReplicas(serviceName, flags); @@ -888,6 +889,7 @@ public KeyValuePair[][] SentinelReplicas(string serviceName, Com return ExecuteSync(msg, ResultProcessor.SentinelArrayOfArrays); } + // For previous compat only Task[][]> IServer.SentinelSlavesAsync(string serviceName, CommandFlags flags) => SentinelReplicasAsync(serviceName, flags); From 91b1dbd060802ceaeb3e9547b305a46a4b1c7e00 Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Mon, 8 Jun 2020 09:53:07 -0400 Subject: [PATCH 22/23] Dammit --- src/StackExchange.Redis/ConnectionMultiplexer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 85b7614fa..626d91306 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2568,7 +2568,7 @@ internal void SwitchMaster(EndPoint switchBlame, ConnectionMultiplexer connectio foreach (var replicaEndPoint in replicaEndPoints) { connection.RawConfig.EndPoints.TryAdd(replicaEndPoint); - }| + } Trace(string.Format("Switching master to {0}", newMasterEndPoint)); // Trigger a reconfigure connection.ReconfigureAsync(false, false, logProxy, switchBlame, From ebdb395c89858858baa01b750abc2086715b772d Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Mon, 8 Jun 2020 09:54:44 -0400 Subject: [PATCH 23/23] Fix processor naming --- src/StackExchange.Redis/ResultProcessor.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index d9202b56a..be80a0ed1 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -126,10 +126,10 @@ public static readonly ResultProcessor SentinelMasterEndpoint = new SentinelGetMasterAddressByNameProcessor(); public static readonly ResultProcessor - SentinelAddressesEndPoints = new SentinelGetSentinelAddresses(); + SentinelAddressesEndPoints = new SentinelGetSentinelAddressesProcessor(); public static readonly ResultProcessor - SentinelReplicaEndPoints = new SentinelGetReplicaAddresses(); + SentinelReplicaEndPoints = new SentinelGetReplicaAddressesProcessor(); public static readonly ResultProcessor[][]> SentinelArrayOfArrays = new SentinelArrayOfArraysProcessor(); @@ -2039,7 +2039,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } - private sealed class SentinelGetSentinelAddresses : ResultProcessor + private sealed class SentinelGetSentinelAddressesProcessor : ResultProcessor { protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) { @@ -2094,7 +2094,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } - private sealed class SentinelGetReplicaAddresses : ResultProcessor + private sealed class SentinelGetReplicaAddressesProcessor : ResultProcessor { protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) {