From 1cc05a10bfb85a4f5a0eb370582bae8c5f692c0d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 10 Feb 2023 23:39:49 +0800 Subject: [PATCH 1/4] domain: fix data race in the ttlJobManager Signed-off-by: Weizhen Wang --- domain/domain.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/domain/domain.go b/domain/domain.go index b4a7a2770afd3..7baa7bf547702 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -902,6 +902,12 @@ func (do *Domain) Close() { do.info.RemoveServerInfo() do.info.RemoveMinStartTS() } + ttlJobManager := do.ttlJobManager.Load() + ttlJobManager.Stop() + err := ttlJobManager.WaitStopped(context.Background(), 30*time.Second) + if err != nil { + logutil.BgLogger().Warn("fail to wait until the ttl job manager stop", zap.Error(err)) + } close(do.exit) if do.etcdClient != nil { terror.Log(errors.Trace(do.etcdClient.Close())) @@ -2542,12 +2548,6 @@ func (do *Domain) StartTTLJobManager() { ttlJobManager.Start() <-do.exit - - ttlJobManager.Stop() - err := ttlJobManager.WaitStopped(context.Background(), 30*time.Second) - if err != nil { - logutil.BgLogger().Warn("fail to wait until the ttl job manager stop", zap.Error(err)) - } }, "ttlJobManager") } From b6d96fdd57b4fba6e9fe329af6e04f2800898162 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 10 Feb 2023 23:48:55 +0800 Subject: [PATCH 2/4] domain: fix data race in the ttlJobManager Signed-off-by: Weizhen Wang --- domain/domain.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/domain/domain.go b/domain/domain.go index 7baa7bf547702..364aac9fe620a 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -903,10 +903,12 @@ func (do *Domain) Close() { do.info.RemoveMinStartTS() } ttlJobManager := do.ttlJobManager.Load() - ttlJobManager.Stop() - err := ttlJobManager.WaitStopped(context.Background(), 30*time.Second) - if err != nil { - logutil.BgLogger().Warn("fail to wait until the ttl job manager stop", zap.Error(err)) + if ttlJobManager != nil { + ttlJobManager.Stop() + err := ttlJobManager.WaitStopped(context.Background(), 30*time.Second) + if err != nil { + logutil.BgLogger().Warn("fail to wait until the ttl job manager stop", zap.Error(err)) + } } close(do.exit) if do.etcdClient != nil { From ab12ba5a97f0be4b176d7424d489af7d901a1905 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 11 Feb 2023 00:41:28 +0800 Subject: [PATCH 3/4] domain: fix data race in the ttlJobManager Signed-off-by: Weizhen Wang --- domain/domain.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/domain/domain.go b/domain/domain.go index 364aac9fe620a..321a319fc39ee 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -69,6 +69,7 @@ import ( "github.com/pingcap/tidb/util/engine" "github.com/pingcap/tidb/util/etcd" "github.com/pingcap/tidb/util/expensivequery" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/memoryusagealarm" @@ -905,7 +906,12 @@ func (do *Domain) Close() { ttlJobManager := do.ttlJobManager.Load() if ttlJobManager != nil { ttlJobManager.Stop() - err := ttlJobManager.WaitStopped(context.Background(), 30*time.Second) + err := ttlJobManager.WaitStopped(context.Background(), func() time.Duration { + if intest.InTest { + return 10 * time.Second + } + return 30 * time.Second + }()) if err != nil { logutil.BgLogger().Warn("fail to wait until the ttl job manager stop", zap.Error(err)) } From af8658c1fbb2ae9abc59faf7c0a622529dc05fbe Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 11 Feb 2023 01:22:39 +0800 Subject: [PATCH 4/4] domain: fix data race in the ttlJobManager Signed-off-by: Weizhen Wang --- domain/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/domain/BUILD.bazel b/domain/BUILD.bazel index ccb70230b9829..6e1618129990e 100644 --- a/domain/BUILD.bazel +++ b/domain/BUILD.bazel @@ -58,6 +58,7 @@ go_library( "//util/etcd", "//util/execdetails", "//util/expensivequery", + "//util/intest", "//util/logutil", "//util/memory", "//util/memoryusagealarm",