From 551a4f0ad1f5d1264b92639008cdb2c39003d087 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 3 Dec 2020 20:20:33 +0800 Subject: [PATCH 01/19] init Signed-off-by: crazycs520 --- cache/cache.go | 154 ++++++++++++++++++++++++++++++++ cache/cache_test.go | 207 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 361 insertions(+) create mode 100644 cache/cache.go create mode 100644 cache/cache_test.go diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..f50b73a --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,154 @@ +package cache + +import ( + "errors" + "fmt" + "os" + "sync" + "time" +) + +var ( + defaultLoLogFileMetaCacheCapacity = 10000 + zeroTime = time.Time{} +) + +// InvalidLogFile indicates the log file format is invalid. +var InvalidLogFile = errors.New("invalid format of log file") + +type LogFileMetaCache struct { + mu sync.RWMutex + cache map[string]*LogFileMeta + capacity int +} + +func NewLogFileMetaCache() *LogFileMetaCache { + return &LogFileMetaCache{ + cache: make(map[string]*LogFileMeta), + capacity: defaultLoLogFileMetaCacheCapacity, + } +} + +func (c *LogFileMetaCache) GetFileMata(stat os.FileInfo) *LogFileMeta { + if stat == nil { + return nil + } + c.mu.RLock() + m := c.cache[stat.Name()] + c.mu.RUnlock() + return m +} + +func (c *LogFileMetaCache) AddFileMataToCache(stat os.FileInfo, meta *LogFileMeta) { + if stat == nil || meta == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + name := stat.Name() + _, ok := c.cache[name] + if ok { + c.cache[name] = meta + } else { + // TODO: Use LRU ? + if len(c.cache) < c.capacity { + c.cache[name] = meta + } + } +} + +func (c *LogFileMetaCache) Len() int { + c.mu.RLock() + l := len(c.cache) + c.mu.RUnlock() + return l +} + +type LogFileMeta struct { + inValid bool + mu sync.Mutex + ModTime time.Time + startTime time.Time + endTime time.Time +} + +func NewLogFileMeta(info os.FileInfo) *LogFileMeta { + return &LogFileMeta{ + ModTime: info.ModTime(), + } +} + +func (l *LogFileMeta) SetEndTime(end time.Time) { + l.endTime = end +} + +func (l *LogFileMeta) GetStartTime(stat os.FileInfo, getStartTime func() (time.Time, error)) (time.Time, error) { + if stat == nil { + return zeroTime, fmt.Errorf("file stat can't be nil") + } + l.mu.Lock() + defer l.mu.Unlock() + t := l.startTime + if l.CheckLogTimeValid(t) && l.CheckFileNotModified(stat) { + return t, nil + } + if getStartTime == nil { + return t, fmt.Errorf("can't get file '%v' start time", stat.Name()) + } + t, err := getStartTime() + if err != nil { + if err == InvalidLogFile { + l.inValid = true + } + return t, err + } + l.inValid = false + l.ModTime = stat.ModTime() + l.startTime = t + return t, nil +} + +func (l *LogFileMeta) GetEndTime(stat os.FileInfo, getEndTime func() (time.Time, error)) (time.Time, error) { + if stat == nil { + return zeroTime, fmt.Errorf("file stat can't be nil") + } + l.mu.Lock() + defer l.mu.Unlock() + t := l.endTime + if l.CheckLogTimeValid(t) && l.CheckFileNotModified(stat) { + return t, nil + } + if getEndTime == nil { + return t, fmt.Errorf("can't get file '%v' end time", stat.Name()) + } + t, err := getEndTime() + if err != nil { + if err == InvalidLogFile { + l.inValid = true + } + return t, err + } + l.inValid = false + l.ModTime = stat.ModTime() + l.endTime = t + return t, nil +} + +// IsInValid returns true if the file is invalid. +func (l *LogFileMeta) IsInValid() bool { + l.mu.Lock() + invalid := l.inValid + l.mu.Unlock() + return invalid +} + +// CheckLogTimeValid returns true if t != zeroTime. +func (l *LogFileMeta) CheckLogTimeValid(t time.Time) bool { + return !t.Equal(zeroTime) +} + +// CheckFileNotModified returns true if the file hasn't been modified. +func (l *LogFileMeta) CheckFileNotModified(info os.FileInfo) bool { + return l.ModTime.Equal(info.ModTime()) +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..a0b0224 --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,207 @@ +package cache_test + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/sysutil/cache" +) + +var _ = Suite(&testCacheSuite{}) + +func TestT(t *testing.T) { + TestingT(t) +} + +type testCacheSuite struct { + tmpDir string +} + +func (s *testCacheSuite) SetUpSuite(c *C) { + tmpDir, err := ioutil.TempDir("", "cache") + c.Assert(err, IsNil) + s.tmpDir = tmpDir +} + +func (s *testCacheSuite) TearDownSuite(c *C) { + c.Assert(os.RemoveAll(s.tmpDir), IsNil, Commentf("remote tmpDir %v failed", s.tmpDir)) +} + +func (s *testCacheSuite) prepareFile(c *C, fileName string) (*os.File, os.FileInfo) { + filePath := path.Join(s.tmpDir, fileName) + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + c.Assert(err, IsNil) + stat, err := file.Stat() + c.Assert(err, IsNil) + return file, stat +} + +func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { + fileName := "tidb.log" + file, stat := s.prepareFile(c, fileName) + defer file.Close() + m := cache.NewLogFileMeta(stat) + c.Assert(m.ModTime, Equals, stat.ModTime()) + + // Test GetStartTime meet error + _, err := m.GetStartTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("get start time meet error") + }) + c.Assert(err.Error(), Equals, "get start time meet error") + + // Test GetStartTime + start := time.Now() + fileStart, err := m.GetStartTime(stat, func() (time.Time, error) { + return start, nil + }) + c.Assert(err, IsNil) + c.Assert(fileStart.Equal(start), IsTrue) + + // Test GetStartTime from cache + fileStart, err = m.GetStartTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("should get from cache") + }) + c.Assert(err, IsNil) + c.Assert(fileStart.Equal(start), IsTrue) + + // Test GetStartTime from cache + fileStart, err = m.GetStartTime(stat, nil) + c.Assert(err, IsNil) + c.Assert(fileStart.Equal(start), IsTrue) + // Test GetStartTime with nil stat + _, err = m.GetStartTime(nil, nil) + c.Assert(err.Error(), Equals, "file stat can't be nil") + + // Test file has been modified. + _, err = file.WriteString("a") + c.Assert(err, IsNil) + stat, err = file.Stat() + c.Assert(err, IsNil) + + // Test GetStartTime meet invalid error + _, err = m.GetStartTime(stat, func() (time.Time, error) { + return time.Now(), cache.InvalidLogFile + }) + c.Assert(err, Equals, cache.InvalidLogFile) + c.Assert(m.IsInValid(), IsTrue) + + // Test GetStartTime meet error + _, err = m.GetStartTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("get start time meet error") + }) + c.Assert(err.Error(), Equals, "get start time meet error") + c.Assert(m.IsInValid(), IsTrue) + + newStartTime := time.Now() + fileStart, err = m.GetStartTime(stat, func() (time.Time, error) { + return newStartTime, nil + }) + c.Assert(err, IsNil) + c.Assert(fileStart.Equal(newStartTime), IsTrue) + c.Assert(m.IsInValid(), IsFalse) + + // Test GetStartTime from cache after file changed + fileStart, err = m.GetStartTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("should get from cache") + }) + c.Assert(err, IsNil) + c.Assert(fileStart.Equal(newStartTime), IsTrue) +} + +func (s *testCacheSuite) TestLogFileMetaGetEndTime(c *C) { + fileName := "tidb.log" + file, stat := s.prepareFile(c, fileName) + defer file.Close() + m := cache.NewLogFileMeta(stat) + c.Assert(m.ModTime, Equals, stat.ModTime()) + + // Test GetEndTime meet error + _, err := m.GetEndTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("get end time meet error") + }) + c.Assert(err.Error(), Equals, "get end time meet error") + + // Test GetEndTime + end := time.Now() + fileEnd, err := m.GetEndTime(stat, func() (time.Time, error) { + return end, nil + }) + c.Assert(err, IsNil) + c.Assert(fileEnd.Equal(end), IsTrue) + c.Assert(m.IsInValid(), IsFalse) + + // Test GetEndTime from cache + fileEnd, err = m.GetEndTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("should get from cache") + }) + c.Assert(err, IsNil) + c.Assert(fileEnd.Equal(end), IsTrue) + + // Test GetEndTime from cache + fileEnd, err = m.GetEndTime(stat, nil) + c.Assert(err, IsNil) + c.Assert(fileEnd.Equal(end), IsTrue) + // Test GetEndTime with nil stat + _, err = m.GetEndTime(nil, nil) + c.Assert(err.Error(), Equals, "file stat can't be nil") + + // Test file has been modified. + _, err = file.WriteString("a") + c.Assert(err, IsNil) + stat, err = file.Stat() + c.Assert(err, IsNil) + + // Test GetEndTime meet invalid error + _, err = m.GetEndTime(stat, func() (time.Time, error) { + return time.Now(), cache.InvalidLogFile + }) + c.Assert(err, Equals, cache.InvalidLogFile) + c.Assert(m.IsInValid(), IsTrue) + + // Test GetEndTime meet error + _, err = m.GetEndTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("get end time meet error") + }) + c.Assert(err.Error(), Equals, "get end time meet error") + + // Test GetEndTime success + newEndTime := time.Now() + fileEnd, err = m.GetEndTime(stat, func() (time.Time, error) { + return newEndTime, nil + }) + c.Assert(err, IsNil) + c.Assert(fileEnd.Equal(newEndTime), IsTrue) + c.Assert(m.IsInValid(), IsFalse) + + // Test GetEndTime from cache after file changed + fileEnd, err = m.GetEndTime(stat, func() (time.Time, error) { + return time.Now(), fmt.Errorf("should get from cache") + }) + c.Assert(err, IsNil) + c.Assert(fileEnd.Equal(newEndTime), IsTrue) + c.Assert(m.IsInValid(), IsFalse) +} + +func (s *testCacheSuite) TestLogFileMetaCache(c *C) { + ca := cache.NewLogFileMetaCache() + ca.AddFileMataToCache(nil, nil) + c.Assert(ca.Len(), Equals, 0) + fileName := "tidb.log" + file, stat := s.prepareFile(c, fileName) + defer file.Close() + m := cache.NewLogFileMeta(stat) + ca.AddFileMataToCache(stat, m) + c.Assert(ca.Len(), Equals, 1) + m2 := cache.NewLogFileMeta(stat) + ca.AddFileMataToCache(stat, m2) + c.Assert(ca.Len(), Equals, 1) + ca.AddFileMataToCache(nil, m) + c.Assert(ca.Len(), Equals, 1) + ca.AddFileMataToCache(stat, nil) + c.Assert(ca.Len(), Equals, 1) +} From c22c2028beb71f454c106f0aef4f24b1e0253c13 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 3 Dec 2020 21:19:59 +0800 Subject: [PATCH 02/19] add file cache Signed-off-by: crazycs520 --- search_log.go | 62 +++++++++++++++++++++++++++++++++++++++------- search_log_test.go | 5 +++- service.go | 9 ++++--- service_test.go | 3 ++- 4 files changed, 65 insertions(+), 14 deletions(-) diff --git a/search_log.go b/search_log.go index 15f2b16..7c8cd2a 100644 --- a/search_log.go +++ b/search_log.go @@ -28,6 +28,7 @@ import ( "time" pb "github.com/pingcap/kvproto/pkg/diagnosticspb" + "github.com/pingcap/sysutil/cache" ) type logFile struct { @@ -43,10 +44,14 @@ func (l *logFile) EndTime() int64 { return l.end } -func resolveFiles(ctx context.Context, logFilePath string, beginTime, endTime int64) ([]logFile, error) { +func resolveFiles(ctx context.Context, logFilePath string, beginTime, endTime int64, ca *cache.LogFileMetaCache) ([]logFile, error) { if logFilePath == "" { return nil, errors.New("empty log file location configuration") } + if ca == nil { + // For test. + ca = cache.NewLogFileMetaCache() + } var logFiles []logFile var skipFiles []*os.File @@ -80,13 +85,39 @@ func resolveFiles(ctx context.Context, logFilePath string, beginTime, endTime in } reader := bufio.NewReader(file) - firstItem, err := readFirstValidLog(ctx, reader, 10) + stat, err := file.Stat() + if err != nil { + return nil + } + meta := ca.GetFileMata(stat) + if meta == nil { + meta = cache.NewLogFileMeta(stat) + defer ca.AddFileMataToCache(stat, meta) + } else { + if meta.CheckFileNotModified(stat) && meta.IsInValid() { + return nil + } + } + + firstTime, err := meta.GetStartTime(stat, func() (time.Time, error) { + firstItem, err := readFirstValidLog(ctx, reader, 10) + if err != nil { + return time.Time{}, err + } + return convertToGoTime(firstItem.Time), nil + }) if err != nil { skipFiles = append(skipFiles, file) return nil } - lastItem, err := readLastValidLog(ctx, file, 10) + lastTime, err := meta.GetEndTime(stat, func() (time.Time, error) { + lastItem, err := readLastValidLog(ctx, file, 10) + if err != nil { + return time.Time{}, err + } + return convertToGoTime(lastItem.Time), nil + }) if err != nil { skipFiles = append(skipFiles, file) return nil @@ -98,13 +129,15 @@ func resolveFiles(ctx context.Context, logFilePath string, beginTime, endTime in return nil } - if beginTime > lastItem.Time || endTime < firstItem.Time { + fileStartTime := convertToLogTime(firstTime) + fileEndTime := convertToLogTime(lastTime) + if beginTime > fileEndTime || endTime < fileStartTime { skipFiles = append(skipFiles, file) } else { logFiles = append(logFiles, logFile{ file: file, - begin: firstItem.Time, - end: lastItem.Time, + begin: fileStartTime, + end: fileEndTime, }) } return nil @@ -143,6 +176,9 @@ func readFirstValidLog(ctx context.Context, reader *bufio.Reader, tryLines int64 for { line, err := readLine(reader) if err != nil { + if err == io.EOF { + return nil, cache.InvalidLogFile + } return nil, err } item, err := parseLogItem(line) @@ -157,7 +193,7 @@ func readFirstValidLog(ctx context.Context, reader *bufio.Reader, tryLines int64 return nil, ctx.Err() } } - return nil, errors.New("not a valid log file") + return nil, cache.InvalidLogFile } func readLastValidLog(ctx context.Context, file *os.File, tryLines int) (*pb.LogMessage, error) { @@ -185,7 +221,7 @@ func readLastValidLog(ctx context.Context, file *os.File, tryLines int) (*pb.Log break } } - return nil, errors.New("not a valid log file") + return nil, cache.InvalidLogFile } // Read a line from a reader. @@ -327,7 +363,15 @@ func parseTimeStamp(s string) (int64, error) { if err != nil { return 0, err } - return t.UnixNano() / int64(time.Millisecond), nil + return convertToLogTime(t), nil +} + +func convertToLogTime(t time.Time) int64 { + return t.UnixNano() / int64(time.Millisecond) +} + +func convertToGoTime(ms int64) time.Time { + return time.Unix(0, ms*int64(time.Millisecond)) } // logIterator implements Iterator and IteratorWithPeek interface. diff --git a/search_log_test.go b/search_log_test.go index 237fd33..4fb984e 100644 --- a/search_log_test.go +++ b/search_log_test.go @@ -28,6 +28,7 @@ import ( . "github.com/pingcap/check" pb "github.com/pingcap/kvproto/pkg/diagnosticspb" "github.com/pingcap/sysutil" + "github.com/pingcap/sysutil/cache" "google.golang.org/grpc" ) @@ -187,12 +188,13 @@ func (s *searchLogSuite) TestResoveFiles(c *C) { }, } + ca := cache.NewLogFileMetaCache() for i, cas := range cases { beginTime, err := sysutil.ParseTimeStamp(cas.search.start) c.Assert(err, IsNil) endTime, err := sysutil.ParseTimeStamp(cas.search.end) c.Assert(err, IsNil) - logFiles, err := sysutil.ResolveFiles(context.Background(), filepath.Join(s.tmpDir, "tidb.log"), beginTime, endTime) + logFiles, err := sysutil.ResolveFiles(context.Background(), filepath.Join(s.tmpDir, "tidb.log"), beginTime, endTime, ca) c.Assert(err, IsNil) c.Assert(len(logFiles), Equals, len(cas.expect), Commentf("search range (index: %d): %+v", i, cas.search)) @@ -205,6 +207,7 @@ func (s *searchLogSuite) TestResoveFiles(c *C) { c.Assert(endTime, Equals, logFiles[j].EndTime(), Commentf("case index: %d, expect index: %v", i, j)) } } + c.Assert(ca.Len(), Equals, 5) } func (s *searchLogSuite) TestLogIterator(c *C) { diff --git a/service.go b/service.go index d9cda8b..8df5df4 100644 --- a/service.go +++ b/service.go @@ -22,15 +22,18 @@ import ( "sort" pb "github.com/pingcap/kvproto/pkg/diagnosticspb" + "github.com/pingcap/sysutil/cache" ) type DiagnosticsServer struct { - logFile string + logFile string + logFileMetaCache *cache.LogFileMetaCache } func NewDiagnosticsServer(logFile string) *DiagnosticsServer { return &DiagnosticsServer{ - logFile: logFile, + logFile: logFile, + logFileMetaCache: cache.NewLogFileMetaCache(), } } @@ -43,7 +46,7 @@ func (d *DiagnosticsServer) SearchLog(req *pb.SearchLogRequest, stream pb.Diagno } ctx := stream.Context() - logFiles, err := resolveFiles(ctx, d.logFile, beginTime, endTime) + logFiles, err := resolveFiles(ctx, d.logFile, beginTime, endTime, d.logFileMetaCache) if err != nil { return err } diff --git a/service_test.go b/service_test.go index 924148c..1a1ff78 100644 --- a/service_test.go +++ b/service_test.go @@ -40,7 +40,8 @@ func TestT(t *testing.T) { func (s *serviceSuite) SetUpSuite(c *C) { server := grpc.NewServer() - pb.RegisterDiagnosticsServer(server, &sysutil.DiagnosticsServer{}) + diagnosticsServer := sysutil.NewDiagnosticsServer("") + pb.RegisterDiagnosticsServer(server, diagnosticsServer) // Find a available port listener, err := net.Listen("tcp", ":0") From 59ae0a3f75236f4660eb833487d57b05a5fee3a8 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 7 Dec 2020 16:18:47 +0800 Subject: [PATCH 03/19] refine test Signed-off-by: crazycs520 --- cache/cache.go | 11 +++++--- cache/cache_test.go | 68 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index f50b73a..52b4299 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -29,6 +29,13 @@ func NewLogFileMetaCache() *LogFileMetaCache { } } +func NewLogFileMetaCacheWithCap(cap int) *LogFileMetaCache { + return &LogFileMetaCache{ + cache: make(map[string]*LogFileMeta), + capacity: cap, + } +} + func (c *LogFileMetaCache) GetFileMata(stat os.FileInfo) *LogFileMeta { if stat == nil { return nil @@ -79,10 +86,6 @@ func NewLogFileMeta(info os.FileInfo) *LogFileMeta { } } -func (l *LogFileMeta) SetEndTime(end time.Time) { - l.endTime = end -} - func (l *LogFileMeta) GetStartTime(stat os.FileInfo, getStartTime func() (time.Time, error)) (time.Time, error) { if stat == nil { return zeroTime, fmt.Errorf("file stat can't be nil") diff --git a/cache/cache_test.go b/cache/cache_test.go index a0b0224..0a05cfa 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -49,7 +49,10 @@ func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { c.Assert(m.ModTime, Equals, stat.ModTime()) // Test GetStartTime meet error - _, err := m.GetStartTime(stat, func() (time.Time, error) { + _, err := m.GetStartTime(stat, nil) + c.Assert(err.Error(), Equals, "can't get file 'tidb.log' start time") + + _, err = m.GetStartTime(stat, func() (time.Time, error) { return time.Now(), fmt.Errorf("get start time meet error") }) c.Assert(err.Error(), Equals, "get start time meet error") @@ -121,7 +124,10 @@ func (s *testCacheSuite) TestLogFileMetaGetEndTime(c *C) { c.Assert(m.ModTime, Equals, stat.ModTime()) // Test GetEndTime meet error - _, err := m.GetEndTime(stat, func() (time.Time, error) { + _, err := m.GetEndTime(stat, nil) + c.Assert(err.Error(), Equals, "can't get file 'tidb.log' end time") + + _, err = m.GetEndTime(stat, func() (time.Time, error) { return time.Now(), fmt.Errorf("get end time meet error") }) c.Assert(err.Error(), Equals, "get end time meet error") @@ -194,14 +200,72 @@ func (s *testCacheSuite) TestLogFileMetaCache(c *C) { fileName := "tidb.log" file, stat := s.prepareFile(c, fileName) defer file.Close() + m := cache.NewLogFileMeta(stat) ca.AddFileMataToCache(stat, m) c.Assert(ca.Len(), Equals, 1) + m = ca.GetFileMata(stat) + c.Assert(m, NotNil) + c.Assert(m.IsInValid(), IsFalse) + c.Assert(m.CheckFileNotModified(stat), IsTrue) + m2 := cache.NewLogFileMeta(stat) ca.AddFileMataToCache(stat, m2) c.Assert(ca.Len(), Equals, 1) + m = ca.GetFileMata(stat) + c.Assert(m, NotNil) + c.Assert(m.IsInValid(), IsFalse) + c.Assert(m.CheckFileNotModified(stat), IsTrue) + ca.AddFileMataToCache(nil, m) c.Assert(ca.Len(), Equals, 1) ca.AddFileMataToCache(stat, nil) c.Assert(ca.Len(), Equals, 1) + + m = ca.GetFileMata(stat) + c.Assert(m, NotNil) + c.Assert(m.IsInValid(), IsFalse) + c.Assert(m.CheckFileNotModified(stat), IsTrue) + + m = ca.GetFileMata(nil) + c.Assert(m, IsNil) +} + +func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { + ca := cache.NewLogFileMetaCacheWithCap(1) + ca.AddFileMataToCache(nil, nil) + c.Assert(ca.Len(), Equals, 0) + fileName := "tidb.log" + file, stat := s.prepareFile(c, fileName) + defer file.Close() + fileName2 := "tidb2.log" + file2, stat2 := s.prepareFile(c, fileName2) + defer file2.Close() + + m := cache.NewLogFileMeta(stat) + ca.AddFileMataToCache(stat, m) + c.Assert(ca.Len(), Equals, 1) + m = ca.GetFileMata(stat) + c.Assert(m, NotNil) + c.Assert(m.IsInValid(), IsFalse) + c.Assert(m.CheckFileNotModified(stat), IsTrue) + + m2 := cache.NewLogFileMeta(stat2) + ca.AddFileMataToCache(stat2, m2) + c.Assert(ca.Len(), Equals, 1) + m = ca.GetFileMata(stat2) + c.Assert(m, IsNil) + + m = ca.GetFileMata(stat) + c.Assert(m, NotNil) + c.Assert(m.IsInValid(), IsFalse) + c.Assert(m.CheckFileNotModified(stat), IsTrue) + + _, err := file.WriteString("abc") + c.Assert(err, IsNil) + stat, err = file.Stat() + c.Assert(err, IsNil) + m = ca.GetFileMata(stat) + c.Assert(m, NotNil) + c.Assert(m.CheckFileNotModified(stat), IsFalse) } From 6c050bb802e8df7b17a8b7455b0b2eed1e6d7f07 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 7 Dec 2020 16:38:28 +0800 Subject: [PATCH 04/19] add bench test Signed-off-by: crazycs520 --- profile | Bin 0 -> 7264 bytes search_log_test.go | 32 +++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 profile diff --git a/profile b/profile new file mode 100644 index 0000000000000000000000000000000000000000..a474d91084ede0807511b1475ed434d84bd79ccf GIT binary patch literal 7264 zcmV-m9G~MKiwFP!00004|D<_&cvMxk?^}>zQwh|n3M7j`JsccJlnN=4!3hKeQ5hsC z&aqN;QYlk)s;pDR;2365WCmpr92#&!P!L2!1RQVx6hT|wZSmGlP|sV@cMFBfdXvfDo|~)V9dG`8~=3Hjq-h4 z^M>OE`soVji!WzrdZ7Y*s6a1NZSXVx(-!BoMW|8=u#gJ1qAI1@U@;ZTS6fu1w01hG zHnimR*(_av|7a)t1?VD|R-&V7ODnbrtq^A`p%r2&l?ttZ<~J{AxguQR7FrRur}kP0 z9aSM6IU=-T{IQMDiqTDO4RusSwDZGRET?dmEvL{!9<77djeu;Ds|eb(EhsF~-KvS{OFL|{x9hIWrK3T*2xNup4&|KJ&I%=Kta@CGjEqs#2O0YsNRHgVc z2uBH)Q@K`ArBo>rk=9u+S1ww+_Ym*X9=`>>Ky|>N@V3V;)J5xDrBny(N?o;Xx?7dd z&kJAE^#bL_ON7adr_pKJ={l-XTC(~xT`yFKj};0H@eDdc({xnr>FCd!_y`ZaXNSat z-Ko2#>!>===bsAAi@R-Cd9ere5Sp9*^SM9qo@IEjNcJqlGwDq2EFBfnuTKcABObNS z>xgI5+1gp&j@T8OcYWFnE?2Q?=PD^}-@(m}_t5vt9%U^$;Y!=_o$wqwN9&}c^3wE~ zPcp3>r`xV7$DY(v>#UN;_9~t4{Ro z{kz#&opG!v-x<%N^Tb)@^p~Ztv0-jp1PI;=e5?&N@1J$t@au1UertLq`MW-p5tpd%w2JmoX{10FIZg|92x*PVU-dc_B zR$XY`7b12VhD7W%^pju2y3(`r#MV;$8zAVYy3w)EX7Wa-<1u@q)A4*dU+d%bU>KWM zF8ZdetGKv=I(O;XO-FSaZCox+J_GmLPCf%KpbJFZ(`m=wBr7$nl5;ieOMSI|x?7z= z@BKwc-LYIq-LXIQ*9PdQGz!10I?B0Qpn6~zN!@x8?tvH4 zMcT#Qa-4|G-)!4f;3_RX?esIW?oKYal}-y@m%2I=10r%JUP6~hUG<xD|k6K#Z}Ck~+@qGl#xOZ6USo^^IcO;qR5uLnh!N<3uCRN_z?DkG4nIFwKwutBvgC$1yZUyGBP< zLl13pF#VTeNpU}I`hFgBpN|V|ls_NG(pYVj*Mn|seq!%%m+IxK?p;$*=+{x5Pk);t z;9i8!xDmV_?1;_NH`X~!=L=IG+V;NK-3O=J8uY!B>WkOWbz)^-dVAY^)~z2-^$1r#98cqg){h?eRA~M2SeelJ25ZVBG=dc)XA--zsb|KzKH)`m0V_R&VwZ4O^sLus``}ODY zyO93)7fFJPaGIS27vW8GlceB9^v~H+niaT@VXPBggUvfvPi^ZeDZcQcEQpn>i|P2U z3vw1;jLkhnr;G7sx>lA{wJ+{H-qp!Df z74*6Il1p_|1L>uY#o3qQ=e9wY;;nS6R;IhvAo});xqS3MoFr!s#5$_eZqrd+M&BP2 z9}U8TUZD-b00rdC!8C8PlO+^Mlb2?qCd3P_XWDTK%BY7p|Amxd@v-eR?^Uj%Ucw%N^u*t6}u$p9G_Z zpdI-k7^0A7=%|L%`V$hAq4=RoG#`o$)F45*f>y5;tuDtUwpN#8n8Jb!BWUv?3D_`v z#&*;&Y@|kMWmnQy`=pf($B*qw9gY!-NGrRF_AGT&z7N4$j%Q%=PhTwtSLv|fSB$vw zDo5vXbv5mOO@@gpa7`OQ2f3K`vVfe5%`^b z_y}yGCMlOubmEz5Tmo0(cHZ+U93p#OiBXCQJY0n_ifN{fYBX)g3}q|1HalcSs=|?@ zMoVFip?yc?q^ohJ?VhVKPI1YVYiP;0k}D%|pd2w06O_=d)KQJ4gU^c+qi}<*;3&L< z?vO)kY2zx%qtWP7H+@my@4KVv_*)R=3=){g7fe}K0f z2V(Q$Ezh=b+08%amTPI&S2EeQ#f5DV@+QPVXyy@dQZ0UA&ug_fnI>y@>Zop}JzolK9L}{1d>r0IcM0tl+V-8$ zuEhs!jJX!?rn|K}b+@{eew-?WRE~Ebcspl|!Rv3h@ur(^xwS0MU3K(t3^Rr5V*Hmb zK;iVZf{}Fs6t~gWdz@%2=V-XG2%F~}=;kUKIOsOUyK)tv1BYg@={7hkJkbHcTY|;d zykk>$)kg(%RQ2@L|CA)UE>mOI;XQPZBvFta_+FA|JdP0GjmLZGUP+=5?b|4JUXROd z-(8RQ(S4Fc2JKlRs@;IcIf-2A25bgu25ppHp&DrQL4m*mlVsVg zxQ(;SrEbL;Kr=wQMt7?yO*t@!on41%gesS+!HAJtj>3@y}&4K9pkfs#ODAioI*4`VY|EDovYa9%9EB*o073q70l1 zG#9i7L3gVO^y(I&HefHIHsCyhr2xja!z|M8ae82E=pAQ(9ujc z{+R>NKBIoYJ$z3_buX>i@9c3HWsgE^{`YSxxUlc#J<8R6w1@BA1=+%Wo|Cr3(Mt7E z_vs$h4D`}LsqEYF7dsJd$HhR4L0Zuipl!#cxSQ~Fy+B2=R#LACmjEpR?b<4risDkB zrJ&uWd(>2*1#dfg?O?$1W^8NM3{zk#uxcJP4d}DI;>iO1zMZHK&P%{Fke2{b(}A|y zJTZKjpEnBdm>rTBE(2NyTD!dVFdf=*N_x}`pf7jI){VG;E5V~?0)6`rCo-!!&K@-j z=((4L%fwF^on6Yrhkzd9PfmYk(1wgN5?i$~=LB0w(u`7KL z=D~be05!K47U>?f5NOuU6+FnK@eR(c5|zd^Kx;sIl=WByv@vtgFWl{Oje8+1%IrgG zG0^G{#Geyzd7)mY0{FO%fD`aBpvOQf@?s{>iWfs%GRUKr03Dbi=TAhrF}c)4d>rU; zkawx2K;OJ3^X4Sp@H9q2f{p)kVf`J%8z9BeqFJ1+FL*S4<0(9iKoSDKO zIi$JzZ+e6yYP(=H(0^{u8#?a6AMAkMgHHlI30eo;qt*aDazx5j;rk3-x!JbQYFNX= zVFov+L0dcJ9tHY){*b4mdJO2$_LW==_aZ+jmZ*F2DWIo7dz#6Q1HH2NMJC^e{C-oS z?!#w*o&oJyCO-kRe~+}D4j4x8cFP!D_9)0a_82@4Pk=qw=fkwE&mVvudBefO7f_t$ zOlZY@F8G6lnbcaK|FYk6+u-MnbZzA!*Nd66ir3ojqj@Ww<$OeEwGL>>i}N+TNHqhV zsMHHpKl~P@OEv>;0NTJKP0n}gQ;Gtp>m;(45&~u=T(!J_Qpe;W? z%nVZjkMXUmgugZdZ3L}@*OT$S8aQY@JPCz<-J_lYdT**^F0If!FqwoX@E5dRi*r~322i8W6uIT{i)>It;hqtw^PPnrO&{# zAnC>~@~90!Z@nTwkj58V9=X9jlKD_Q2Q=ep!Og4q8lZdCMxbT8?`I!O2RznZoHQNq zd7$S(> z&35FdEf5)?YwAUyrC&eDP@VrYRitJCe#OvH%3oW6wt)5``(-Q8i_?~~CbI#LGDiu2 zZ3WtDH@R&<$4^M{2JtO+d2WR4^&)JAZ6L#tSG@!@>#!I-2ky#(|UXxl;8)yqIDzMIV)4**Ur(2LYuz)wLhQV#+iV&zJi@-ooN zEKya;^c_GuoL>(Dz5?_LXgk@o9YFu_lk7GR@B@a`QvP}s=vC&(%QBv5UV`oLGVB1| zqh0}8od3|4m;0{(Pi-Ex6KL0SlGF16V-mmlfV+TpfoAGn^(xR8ze!Cl0Q`}sof7_f z4d^x78M}byo{*Yc2zb=a$Ay5qfp&9E=8~^IxD#H5U7P{k)oVa|7fUb}0ruC6)MCIt zyY(Wq1aJdit4f)$2WStMR#gep_X6z&twU7_GrbP_o@E)Pu4mY*0W1%e{ugQz-R5;TLJhk z(7QZfxif|qy=UWD2~jqCZ*Aa}_cIch)1Q%^qrGst??0igHa6~CJh->JcO6>!hWFrpdH{UegyC!&_S+B-K*XQdUWO- z&a%~D{}Z5;zupIW-{JTGX!{(=vNeE9IU=R}^#RZa%;C-9=K(kf@52YsVu*8EMS%=D zhk!oHzh>mwehB2ISEPH@hd^&mTfsU#3ivYHP{Ln_fDVDopof8;c-HASm$jIsKZL`u z1)Nm(s3Sl>{kDJs`eNMC16f%o@tmAE0+(Fs^=1~^aR+D!^KA>&M?g~XF#DUz~@?~{>OIecUFG{`e^HXHsvwEV;yD8n8MGrCA{z<(1+ZD za^e1?j2RyF3D7S`dnQ7Y5^LGv#j^Or-rKX26VFMlf!MEWbCIHdB$9;R_@UGcdvo8I$~a zb=pc+N8`ajv^pHGwvxf>WLmhYV~s{2;S0z89WruOB4Cbg$nDb*F+-#4Z#RM||C!b4 z-`%G^l8V_od9ynO17`gHEEx{Q0?8)U!JUx|2ZtsD5!2r-w=g(7GLUQ-f(?To=11VYuEx4*wdwPgg7!N$61Jj{m;|9#H7lwqZOJ*#YitK;F? zbR=a|`UV?ju#t0dY`!wX|38(nPYcKW-lWkGHG(PMb%AKw@Ja5p5;upddV+9;Pm&uF zh(`T=Td^+KXat+sEVgEB+BA)1rLQ5LY#PVra>>;A>svP_(G;$Z#Y3%mS9TpVp|`KM zuXith@0tNMHNE@Q)Lby2Pw!sAK!d-2fYCP?Z0KV%vcCUMy0JBmZtb0n_YJdBft24B zw|u5CkuyS4H=Zy|{xK>ZjfeduamzQ_S@C!K-8^P}dbQ#!Yx2oy_utj9)4glNOc_Zt z5UozcqfuYas$oMbeIdh2CF7I*RVN?Zg1x#Sl&z#LEhyQIf=Q#nfA0UqKCyJlkdo}& z+D5jrSpZF@&4!41hrh?k8#}O89cze~fhapD>-ek1@$KW5FUnuGr&|XJre{r4am$fy zJrc~!2YCg7IEa-#u4sn9{58fCUh8_Y);5~SCYdyG{*Fl{{T}Dr*AU_~;XPV$sl$}{ zn+J`p2AR`~(XG$P+O&uO2iZ9_Z_A34Z8ZZi!)ifr47Lf`Ff;_sRMg+C#UZ|->4e01 zw0Zd?Bj~SaNww2*XgZN|h2!A5q`#vRa$h3V$g6ctk!UoRD^?_I2BIVVXP<1RBUl}8 z(Z-xrUm|H3u|#U5zvu5u+M8!HB|959vT@UBEy0`OX}vBt;tp_Zsr1R8&;EJXfh@kW-7;E8Yu?5)@sN(u-aj% zYq}$Cq{XnTQ&^N8-#Sp);O@?=4vu1OHgZh3zBO#k9G)Ai>iYI!OvoBuYZy&_YOOYe zl7FnltoaMVQS9`UoO9Av)G!h`=ftha5hEJ%mpa<`qVZtUsfvf`b(Rqeu%?|eQkhx| z#?4UO#6To3(U{jhG?#nvlwmJrLvE$wiCAvGkP%7;Q?(NfBQYdtgd(ZdrZVU7>R32v zF(+$unzQKZSrrQ!W-1woR{Da0V53o+2$-kb#bM`ND1R3gW`u8GoH6KHaLyjY6 z#A5LYhQIRUI5;{~J1owH)5>LVA`mu4@Owi3>p;MY1;cUaKG_nCM(Tq+*XOS9`Owmk zG;CzaUa)&s#rV;o(kGad+ed!#gpf6`J}hf_$sXiafuYQlo0ps$R5X$@e6jc6)HyskAuDYq3^O$RRQcu%Y;0>baG81&jcam-FcOY3Qe$LD%N-Mohj<2=#$?Jv!Sy=J==k7QHJ%)CIli;nfGck z`!cc*`R`LMkA{4!XV&3ESkg%5k|7*-0+ioNOk<*;NvqEcY5M_C7}ytCV2#C-hOFiS zolKjxfd*qtJYuGX+ldWsb zv}J@^>n<5hV@53Re$~@i_Vuh{U=sAL%qGIvfEhNfjtBWz?UbCOO(PetTpCCu{W%T! uCSuQhoip26jq!9eRGTsaQAtNDXJB3JWGiLF{&xTX0RR734c1Z3Gynh)wjUJ$ literal 0 HcmV?d00001 diff --git a/search_log_test.go b/search_log_test.go index 4fb984e..f4d9ed6 100644 --- a/search_log_test.go +++ b/search_log_test.go @@ -20,11 +20,14 @@ import ( "io/ioutil" "log" "net" + "net/http" "os" "path/filepath" "strings" "time" + _ "net/http/pprof" + . "github.com/pingcap/check" pb "github.com/pingcap/kvproto/pkg/diagnosticspb" "github.com/pingcap/sysutil" @@ -55,6 +58,10 @@ func (s *searchLogSuite) SetUpSuite(c *C) { s.server = server s.address = fmt.Sprintf(":%d", listener.Addr().(*net.TCPAddr).Port) + go func() { + log.Println(http.ListenAndServe("localhost:10091", nil)) + }() + go func() { if err := server.Serve(listener); err != nil { log.Fatalf("failed to serve: %v", err) @@ -72,7 +79,7 @@ func (s *searchLogSuite) writeTmpFile(c *C, filename string, lines []string) { c.Assert(err, IsNil, Commentf("write tmp file %s failed", filename)) } -func (s *searchLogSuite) TestResoveFiles(c *C) { +func (s *searchLogSuite) prepareLogFiles(c *C) { s.writeTmpFile(c, "tidb.log", []string{ `20/08/26 06:19:13.011 -04:00 [INFO] [printer.go:41] ["Welcome to TiDB."]`, `hello TiDB`, @@ -105,6 +112,10 @@ func (s *searchLogSuite) TestResoveFiles(c *C) { `[2019/08/26 06:22:16.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, `[2019/08/26 06:22:17.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, }) +} + +func (s *searchLogSuite) TestResoveFiles(c *C) { + s.prepareLogFiles(c) type timeRange struct{ start, end string } cases := []struct { @@ -646,6 +657,25 @@ func (s *searchLogSuite) BenchmarkReadLastLinesOfHugeLine(c *C) { } } +func (s *searchLogSuite) BenchmarkResolveFiles(c *C) { + s.prepareLogFiles(c) + + ca := cache.NewLogFileMetaCache() + beginTime, err := sysutil.ParseTimeStamp("2019/08/26 06:19:13.011 -04:00") + c.Assert(err, IsNil) + endTime, err := sysutil.ParseTimeStamp("2019/08/26 06:22:17.011 -04:00") + c.Assert(err, IsNil) + path := filepath.Join(s.tmpDir, "tidb.log") + c.ResetTimer() + //for i := 0; i < c.N; i++ { + fmt.Printf("---------start------\n\n") + for i := 0; i < 500000; i++ { + logFiles, err := sysutil.ResolveFiles(context.Background(), path, beginTime, endTime, ca) + c.Assert(err, IsNil) + c.Assert(len(logFiles), Equals, 4) + } +} + // run benchmark by `go test -check.b` // result: // searchLogSuite.BenchmarkReadLastLines 1000000 2008 ns/op From 9dd18ed31826ed8a377de50e5904e01e96711461 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 7 Dec 2020 16:53:59 +0800 Subject: [PATCH 05/19] add bench test Signed-off-by: crazycs520 --- search_log.go | 2 +- search_log_test.go | 30 +++++++++++++----------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/search_log.go b/search_log.go index 7c8cd2a..32117d0 100644 --- a/search_log.go +++ b/search_log.go @@ -83,7 +83,6 @@ func resolveFiles(ctx context.Context, logFilePath string, beginTime, endTime in if err != nil { return nil } - reader := bufio.NewReader(file) stat, err := file.Stat() if err != nil { @@ -100,6 +99,7 @@ func resolveFiles(ctx context.Context, logFilePath string, beginTime, endTime in } firstTime, err := meta.GetStartTime(stat, func() (time.Time, error) { + reader := bufio.NewReader(file) firstItem, err := readFirstValidLog(ctx, reader, 10) if err != nil { return time.Time{}, err diff --git a/search_log_test.go b/search_log_test.go index f4d9ed6..b5ffdd0 100644 --- a/search_log_test.go +++ b/search_log_test.go @@ -20,14 +20,11 @@ import ( "io/ioutil" "log" "net" - "net/http" "os" "path/filepath" "strings" "time" - _ "net/http/pprof" - . "github.com/pingcap/check" pb "github.com/pingcap/kvproto/pkg/diagnosticspb" "github.com/pingcap/sysutil" @@ -58,10 +55,6 @@ func (s *searchLogSuite) SetUpSuite(c *C) { s.server = server s.address = fmt.Sprintf(":%d", listener.Addr().(*net.TCPAddr).Port) - go func() { - log.Println(http.ListenAndServe("localhost:10091", nil)) - }() - go func() { if err := server.Serve(listener); err != nil { log.Fatalf("failed to serve: %v", err) @@ -79,7 +72,7 @@ func (s *searchLogSuite) writeTmpFile(c *C, filename string, lines []string) { c.Assert(err, IsNil, Commentf("write tmp file %s failed", filename)) } -func (s *searchLogSuite) prepareLogFiles(c *C) { +func (s *searchLogSuite) TestResoveFiles(c *C) { s.writeTmpFile(c, "tidb.log", []string{ `20/08/26 06:19:13.011 -04:00 [INFO] [printer.go:41] ["Welcome to TiDB."]`, `hello TiDB`, @@ -112,10 +105,6 @@ func (s *searchLogSuite) prepareLogFiles(c *C) { `[2019/08/26 06:22:16.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, `[2019/08/26 06:22:17.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, }) -} - -func (s *searchLogSuite) TestResoveFiles(c *C) { - s.prepareLogFiles(c) type timeRange struct{ start, end string } cases := []struct { @@ -658,7 +647,16 @@ func (s *searchLogSuite) BenchmarkReadLastLinesOfHugeLine(c *C) { } func (s *searchLogSuite) BenchmarkResolveFiles(c *C) { - s.prepareLogFiles(c) + for i := 0; i < 1000; i++ { + s.writeTmpFile(c, fmt.Sprintf("tidb-%v.log", i), []string{ + `20/08/26 06:19:13.011 -04:00 [INFO] [printer.go:41] ["Welcome to TiDB."]`, + `[2019/08/26 06:19:13.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, + `[2019/08/26 06:19:14.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, + `[2019/08/26 06:19:15.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, + `[2019/08/26 06:19:16.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, + `[2019/08/26 06:19:17.011 -04:00] [INFO] [printer.go:41] ["Welcome to TiDB."]`, + }) + } ca := cache.NewLogFileMetaCache() beginTime, err := sysutil.ParseTimeStamp("2019/08/26 06:19:13.011 -04:00") @@ -667,12 +665,10 @@ func (s *searchLogSuite) BenchmarkResolveFiles(c *C) { c.Assert(err, IsNil) path := filepath.Join(s.tmpDir, "tidb.log") c.ResetTimer() - //for i := 0; i < c.N; i++ { - fmt.Printf("---------start------\n\n") - for i := 0; i < 500000; i++ { + for i := 0; i < c.N; i++ { logFiles, err := sysutil.ResolveFiles(context.Background(), path, beginTime, endTime, ca) c.Assert(err, IsNil) - c.Assert(len(logFiles), Equals, 4) + c.Assert(len(logFiles), Equals, 1000) } } From 0e884a05aa35ce0e45d242d2a3ff5eef140f0715 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 7 Dec 2020 20:09:27 +0800 Subject: [PATCH 06/19] remove redundant file Signed-off-by: crazycs520 --- profile | Bin 7264 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 profile diff --git a/profile b/profile deleted file mode 100644 index a474d91084ede0807511b1475ed434d84bd79ccf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7264 zcmV-m9G~MKiwFP!00004|D<_&cvMxk?^}>zQwh|n3M7j`JsccJlnN=4!3hKeQ5hsC z&aqN;QYlk)s;pDR;2365WCmpr92#&!P!L2!1RQVx6hT|wZSmGlP|sV@cMFBfdXvfDo|~)V9dG`8~=3Hjq-h4 z^M>OE`soVji!WzrdZ7Y*s6a1NZSXVx(-!BoMW|8=u#gJ1qAI1@U@;ZTS6fu1w01hG zHnimR*(_av|7a)t1?VD|R-&V7ODnbrtq^A`p%r2&l?ttZ<~J{AxguQR7FrRur}kP0 z9aSM6IU=-T{IQMDiqTDO4RusSwDZGRET?dmEvL{!9<77djeu;Ds|eb(EhsF~-KvS{OFL|{x9hIWrK3T*2xNup4&|KJ&I%=Kta@CGjEqs#2O0YsNRHgVc z2uBH)Q@K`ArBo>rk=9u+S1ww+_Ym*X9=`>>Ky|>N@V3V;)J5xDrBny(N?o;Xx?7dd z&kJAE^#bL_ON7adr_pKJ={l-XTC(~xT`yFKj};0H@eDdc({xnr>FCd!_y`ZaXNSat z-Ko2#>!>===bsAAi@R-Cd9ere5Sp9*^SM9qo@IEjNcJqlGwDq2EFBfnuTKcABObNS z>xgI5+1gp&j@T8OcYWFnE?2Q?=PD^}-@(m}_t5vt9%U^$;Y!=_o$wqwN9&}c^3wE~ zPcp3>r`xV7$DY(v>#UN;_9~t4{Ro z{kz#&opG!v-x<%N^Tb)@^p~Ztv0-jp1PI;=e5?&N@1J$t@au1UertLq`MW-p5tpd%w2JmoX{10FIZg|92x*PVU-dc_B zR$XY`7b12VhD7W%^pju2y3(`r#MV;$8zAVYy3w)EX7Wa-<1u@q)A4*dU+d%bU>KWM zF8ZdetGKv=I(O;XO-FSaZCox+J_GmLPCf%KpbJFZ(`m=wBr7$nl5;ieOMSI|x?7z= z@BKwc-LYIq-LXIQ*9PdQGz!10I?B0Qpn6~zN!@x8?tvH4 zMcT#Qa-4|G-)!4f;3_RX?esIW?oKYal}-y@m%2I=10r%JUP6~hUG<xD|k6K#Z}Ck~+@qGl#xOZ6USo^^IcO;qR5uLnh!N<3uCRN_z?DkG4nIFwKwutBvgC$1yZUyGBP< zLl13pF#VTeNpU}I`hFgBpN|V|ls_NG(pYVj*Mn|seq!%%m+IxK?p;$*=+{x5Pk);t z;9i8!xDmV_?1;_NH`X~!=L=IG+V;NK-3O=J8uY!B>WkOWbz)^-dVAY^)~z2-^$1r#98cqg){h?eRA~M2SeelJ25ZVBG=dc)XA--zsb|KzKH)`m0V_R&VwZ4O^sLus``}ODY zyO93)7fFJPaGIS27vW8GlceB9^v~H+niaT@VXPBggUvfvPi^ZeDZcQcEQpn>i|P2U z3vw1;jLkhnr;G7sx>lA{wJ+{H-qp!Df z74*6Il1p_|1L>uY#o3qQ=e9wY;;nS6R;IhvAo});xqS3MoFr!s#5$_eZqrd+M&BP2 z9}U8TUZD-b00rdC!8C8PlO+^Mlb2?qCd3P_XWDTK%BY7p|Amxd@v-eR?^Uj%Ucw%N^u*t6}u$p9G_Z zpdI-k7^0A7=%|L%`V$hAq4=RoG#`o$)F45*f>y5;tuDtUwpN#8n8Jb!BWUv?3D_`v z#&*;&Y@|kMWmnQy`=pf($B*qw9gY!-NGrRF_AGT&z7N4$j%Q%=PhTwtSLv|fSB$vw zDo5vXbv5mOO@@gpa7`OQ2f3K`vVfe5%`^b z_y}yGCMlOubmEz5Tmo0(cHZ+U93p#OiBXCQJY0n_ifN{fYBX)g3}q|1HalcSs=|?@ zMoVFip?yc?q^ohJ?VhVKPI1YVYiP;0k}D%|pd2w06O_=d)KQJ4gU^c+qi}<*;3&L< z?vO)kY2zx%qtWP7H+@my@4KVv_*)R=3=){g7fe}K0f z2V(Q$Ezh=b+08%amTPI&S2EeQ#f5DV@+QPVXyy@dQZ0UA&ug_fnI>y@>Zop}JzolK9L}{1d>r0IcM0tl+V-8$ zuEhs!jJX!?rn|K}b+@{eew-?WRE~Ebcspl|!Rv3h@ur(^xwS0MU3K(t3^Rr5V*Hmb zK;iVZf{}Fs6t~gWdz@%2=V-XG2%F~}=;kUKIOsOUyK)tv1BYg@={7hkJkbHcTY|;d zykk>$)kg(%RQ2@L|CA)UE>mOI;XQPZBvFta_+FA|JdP0GjmLZGUP+=5?b|4JUXROd z-(8RQ(S4Fc2JKlRs@;IcIf-2A25bgu25ppHp&DrQL4m*mlVsVg zxQ(;SrEbL;Kr=wQMt7?yO*t@!on41%gesS+!HAJtj>3@y}&4K9pkfs#ODAioI*4`VY|EDovYa9%9EB*o073q70l1 zG#9i7L3gVO^y(I&HefHIHsCyhr2xja!z|M8ae82E=pAQ(9ujc z{+R>NKBIoYJ$z3_buX>i@9c3HWsgE^{`YSxxUlc#J<8R6w1@BA1=+%Wo|Cr3(Mt7E z_vs$h4D`}LsqEYF7dsJd$HhR4L0Zuipl!#cxSQ~Fy+B2=R#LACmjEpR?b<4risDkB zrJ&uWd(>2*1#dfg?O?$1W^8NM3{zk#uxcJP4d}DI;>iO1zMZHK&P%{Fke2{b(}A|y zJTZKjpEnBdm>rTBE(2NyTD!dVFdf=*N_x}`pf7jI){VG;E5V~?0)6`rCo-!!&K@-j z=((4L%fwF^on6Yrhkzd9PfmYk(1wgN5?i$~=LB0w(u`7KL z=D~be05!K47U>?f5NOuU6+FnK@eR(c5|zd^Kx;sIl=WByv@vtgFWl{Oje8+1%IrgG zG0^G{#Geyzd7)mY0{FO%fD`aBpvOQf@?s{>iWfs%GRUKr03Dbi=TAhrF}c)4d>rU; zkawx2K;OJ3^X4Sp@H9q2f{p)kVf`J%8z9BeqFJ1+FL*S4<0(9iKoSDKO zIi$JzZ+e6yYP(=H(0^{u8#?a6AMAkMgHHlI30eo;qt*aDazx5j;rk3-x!JbQYFNX= zVFov+L0dcJ9tHY){*b4mdJO2$_LW==_aZ+jmZ*F2DWIo7dz#6Q1HH2NMJC^e{C-oS z?!#w*o&oJyCO-kRe~+}D4j4x8cFP!D_9)0a_82@4Pk=qw=fkwE&mVvudBefO7f_t$ zOlZY@F8G6lnbcaK|FYk6+u-MnbZzA!*Nd66ir3ojqj@Ww<$OeEwGL>>i}N+TNHqhV zsMHHpKl~P@OEv>;0NTJKP0n}gQ;Gtp>m;(45&~u=T(!J_Qpe;W? z%nVZjkMXUmgugZdZ3L}@*OT$S8aQY@JPCz<-J_lYdT**^F0If!FqwoX@E5dRi*r~322i8W6uIT{i)>It;hqtw^PPnrO&{# zAnC>~@~90!Z@nTwkj58V9=X9jlKD_Q2Q=ep!Og4q8lZdCMxbT8?`I!O2RznZoHQNq zd7$S(> z&35FdEf5)?YwAUyrC&eDP@VrYRitJCe#OvH%3oW6wt)5``(-Q8i_?~~CbI#LGDiu2 zZ3WtDH@R&<$4^M{2JtO+d2WR4^&)JAZ6L#tSG@!@>#!I-2ky#(|UXxl;8)yqIDzMIV)4**Ur(2LYuz)wLhQV#+iV&zJi@-ooN zEKya;^c_GuoL>(Dz5?_LXgk@o9YFu_lk7GR@B@a`QvP}s=vC&(%QBv5UV`oLGVB1| zqh0}8od3|4m;0{(Pi-Ex6KL0SlGF16V-mmlfV+TpfoAGn^(xR8ze!Cl0Q`}sof7_f z4d^x78M}byo{*Yc2zb=a$Ay5qfp&9E=8~^IxD#H5U7P{k)oVa|7fUb}0ruC6)MCIt zyY(Wq1aJdit4f)$2WStMR#gep_X6z&twU7_GrbP_o@E)Pu4mY*0W1%e{ugQz-R5;TLJhk z(7QZfxif|qy=UWD2~jqCZ*Aa}_cIch)1Q%^qrGst??0igHa6~CJh->JcO6>!hWFrpdH{UegyC!&_S+B-K*XQdUWO- z&a%~D{}Z5;zupIW-{JTGX!{(=vNeE9IU=R}^#RZa%;C-9=K(kf@52YsVu*8EMS%=D zhk!oHzh>mwehB2ISEPH@hd^&mTfsU#3ivYHP{Ln_fDVDopof8;c-HASm$jIsKZL`u z1)Nm(s3Sl>{kDJs`eNMC16f%o@tmAE0+(Fs^=1~^aR+D!^KA>&M?g~XF#DUz~@?~{>OIecUFG{`e^HXHsvwEV;yD8n8MGrCA{z<(1+ZD za^e1?j2RyF3D7S`dnQ7Y5^LGv#j^Or-rKX26VFMlf!MEWbCIHdB$9;R_@UGcdvo8I$~a zb=pc+N8`ajv^pHGwvxf>WLmhYV~s{2;S0z89WruOB4Cbg$nDb*F+-#4Z#RM||C!b4 z-`%G^l8V_od9ynO17`gHEEx{Q0?8)U!JUx|2ZtsD5!2r-w=g(7GLUQ-f(?To=11VYuEx4*wdwPgg7!N$61Jj{m;|9#H7lwqZOJ*#YitK;F? zbR=a|`UV?ju#t0dY`!wX|38(nPYcKW-lWkGHG(PMb%AKw@Ja5p5;upddV+9;Pm&uF zh(`T=Td^+KXat+sEVgEB+BA)1rLQ5LY#PVra>>;A>svP_(G;$Z#Y3%mS9TpVp|`KM zuXith@0tNMHNE@Q)Lby2Pw!sAK!d-2fYCP?Z0KV%vcCUMy0JBmZtb0n_YJdBft24B zw|u5CkuyS4H=Zy|{xK>ZjfeduamzQ_S@C!K-8^P}dbQ#!Yx2oy_utj9)4glNOc_Zt z5UozcqfuYas$oMbeIdh2CF7I*RVN?Zg1x#Sl&z#LEhyQIf=Q#nfA0UqKCyJlkdo}& z+D5jrSpZF@&4!41hrh?k8#}O89cze~fhapD>-ek1@$KW5FUnuGr&|XJre{r4am$fy zJrc~!2YCg7IEa-#u4sn9{58fCUh8_Y);5~SCYdyG{*Fl{{T}Dr*AU_~;XPV$sl$}{ zn+J`p2AR`~(XG$P+O&uO2iZ9_Z_A34Z8ZZi!)ifr47Lf`Ff;_sRMg+C#UZ|->4e01 zw0Zd?Bj~SaNww2*XgZN|h2!A5q`#vRa$h3V$g6ctk!UoRD^?_I2BIVVXP<1RBUl}8 z(Z-xrUm|H3u|#U5zvu5u+M8!HB|959vT@UBEy0`OX}vBt;tp_Zsr1R8&;EJXfh@kW-7;E8Yu?5)@sN(u-aj% zYq}$Cq{XnTQ&^N8-#Sp);O@?=4vu1OHgZh3zBO#k9G)Ai>iYI!OvoBuYZy&_YOOYe zl7FnltoaMVQS9`UoO9Av)G!h`=ftha5hEJ%mpa<`qVZtUsfvf`b(Rqeu%?|eQkhx| z#?4UO#6To3(U{jhG?#nvlwmJrLvE$wiCAvGkP%7;Q?(NfBQYdtgd(ZdrZVU7>R32v zF(+$unzQKZSrrQ!W-1woR{Da0V53o+2$-kb#bM`ND1R3gW`u8GoH6KHaLyjY6 z#A5LYhQIRUI5;{~J1owH)5>LVA`mu4@Owi3>p;MY1;cUaKG_nCM(Tq+*XOS9`Owmk zG;CzaUa)&s#rV;o(kGad+ed!#gpf6`J}hf_$sXiafuYQlo0ps$R5X$@e6jc6)HyskAuDYq3^O$RRQcu%Y;0>baG81&jcam-FcOY3Qe$LD%N-Mohj<2=#$?Jv!Sy=J==k7QHJ%)CIli;nfGck z`!cc*`R`LMkA{4!XV&3ESkg%5k|7*-0+ioNOk<*;NvqEcY5M_C7}ytCV2#C-hOFiS zolKjxfd*qtJYuGX+ldWsb zv}J@^>n<5hV@53Re$~@i_Vuh{U=sAL%qGIvfEhNfjtBWz?UbCOO(PetTpCCu{W%T! uCSuQhoip26jq!9eRGTsaQAtNDXJB3JWGiLF{&xTX0RR734c1Z3Gynh)wjUJ$ From ad19aae7b34f257ad61122f3477308998f9bc409 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Mon, 7 Dec 2020 20:13:02 +0800 Subject: [PATCH 07/19] use serial suit Signed-off-by: crazycs520 --- cache/cache_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index 0a05cfa..45298e5 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -12,7 +12,7 @@ import ( "github.com/pingcap/sysutil/cache" ) -var _ = Suite(&testCacheSuite{}) +var _ = SerialSuites(&testCacheSuite{}) func TestT(t *testing.T) { TestingT(t) From 94f4b98c3a66d29b5ecf222f0e39b4911efb5e38 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 11:33:57 +0800 Subject: [PATCH 08/19] update test Signed-off-by: crazycs520 --- cache/cache_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cache/cache_test.go b/cache/cache_test.go index 45298e5..c77bb04 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -83,6 +83,8 @@ func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { // Test file has been modified. _, err = file.WriteString("a") c.Assert(err, IsNil) + err = file.Sync() + c.Assert(err, IsNil) stat, err = file.Stat() c.Assert(err, IsNil) @@ -159,6 +161,8 @@ func (s *testCacheSuite) TestLogFileMetaGetEndTime(c *C) { // Test file has been modified. _, err = file.WriteString("a") c.Assert(err, IsNil) + err = file.Sync() + c.Assert(err, IsNil) stat, err = file.Stat() c.Assert(err, IsNil) @@ -263,6 +267,8 @@ func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { _, err := file.WriteString("abc") c.Assert(err, IsNil) + err = file.Sync() + c.Assert(err, IsNil) stat, err = file.Stat() c.Assert(err, IsNil) m = ca.GetFileMata(stat) From ff136f3b989888ec37f05c2abb6a1008f7deec83 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 11:44:24 +0800 Subject: [PATCH 09/19] update test Signed-off-by: crazycs520 --- cache/cache_test.go | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index c77bb04..2d6ffb4 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -41,6 +41,17 @@ func (s *testCacheSuite) prepareFile(c *C, fileName string) (*os.File, os.FileIn return file, stat } +func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.File, os.FileInfo) { + stat, err := file.Stat() + c.Assert(err, IsNil) + name := stat.Name() + _, err = file.WriteString(data) + c.Assert(err, IsNil) + err = file.Close() + c.Assert(err, IsNil) + return s.prepareFile(c, name) +} + func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { fileName := "tidb.log" file, stat := s.prepareFile(c, fileName) @@ -81,12 +92,7 @@ func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { c.Assert(err.Error(), Equals, "file stat can't be nil") // Test file has been modified. - _, err = file.WriteString("a") - c.Assert(err, IsNil) - err = file.Sync() - c.Assert(err, IsNil) - stat, err = file.Stat() - c.Assert(err, IsNil) + file, stat = s.writeAndReopen(c, file, "a") // Test GetStartTime meet invalid error _, err = m.GetStartTime(stat, func() (time.Time, error) { @@ -159,12 +165,7 @@ func (s *testCacheSuite) TestLogFileMetaGetEndTime(c *C) { c.Assert(err.Error(), Equals, "file stat can't be nil") // Test file has been modified. - _, err = file.WriteString("a") - c.Assert(err, IsNil) - err = file.Sync() - c.Assert(err, IsNil) - stat, err = file.Stat() - c.Assert(err, IsNil) + file, stat = s.writeAndReopen(c, file, "a") // Test GetEndTime meet invalid error _, err = m.GetEndTime(stat, func() (time.Time, error) { @@ -265,12 +266,7 @@ func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { c.Assert(m.IsInValid(), IsFalse) c.Assert(m.CheckFileNotModified(stat), IsTrue) - _, err := file.WriteString("abc") - c.Assert(err, IsNil) - err = file.Sync() - c.Assert(err, IsNil) - stat, err = file.Stat() - c.Assert(err, IsNil) + file, stat = s.writeAndReopen(c, file, "abc") m = ca.GetFileMata(stat) c.Assert(m, NotNil) c.Assert(m.CheckFileNotModified(stat), IsFalse) From 8306ad1644cd95ac2aac064ecea9b2cc836ca723 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 11:53:27 +0800 Subject: [PATCH 10/19] add log Signed-off-by: crazycs520 --- cache/cache_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cache/cache_test.go b/cache/cache_test.go index 2d6ffb4..9bf9d49 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -266,7 +266,9 @@ func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { c.Assert(m.IsInValid(), IsFalse) c.Assert(m.CheckFileNotModified(stat), IsTrue) + fmt.Printf("old mod time: \n%v\n%v ---\n", stat.ModTime(), m.ModTime) file, stat = s.writeAndReopen(c, file, "abc") + fmt.Printf("new mod time: \n%v\n%v ---\n", stat.ModTime(), m.ModTime) m = ca.GetFileMata(stat) c.Assert(m, NotNil) c.Assert(m.CheckFileNotModified(stat), IsFalse) From 6c770f725dc6d1848165efc1e2ccdaff5a10cc25 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 11:58:42 +0800 Subject: [PATCH 11/19] add log Signed-off-by: crazycs520 --- cache/cache_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index 9bf9d49..1ccc1d9 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -266,9 +266,9 @@ func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { c.Assert(m.IsInValid(), IsFalse) c.Assert(m.CheckFileNotModified(stat), IsTrue) - fmt.Printf("old mod time: \n%v\n%v ---\n", stat.ModTime(), m.ModTime) + fmt.Printf("old mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) file, stat = s.writeAndReopen(c, file, "abc") - fmt.Printf("new mod time: \n%v\n%v ---\n", stat.ModTime(), m.ModTime) + fmt.Printf("new mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) m = ca.GetFileMata(stat) c.Assert(m, NotNil) c.Assert(m.CheckFileNotModified(stat), IsFalse) From 3da86e772c20a30d1a306d6fb0d630dec161700f Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 13:01:23 +0800 Subject: [PATCH 12/19] add log Signed-off-by: crazycs520 --- cache/cache_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cache/cache_test.go b/cache/cache_test.go index 1ccc1d9..c485f0d 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -45,6 +45,9 @@ func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.F stat, err := file.Stat() c.Assert(err, IsNil) name := stat.Name() + err = file.Close() + c.Assert(err, IsNil) + file, stat = s.prepareFile(c, name) _, err = file.WriteString(data) c.Assert(err, IsNil) err = file.Close() From 02dcc45caf273127a3f2d206854a6c64496b0e90 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 13:17:26 +0800 Subject: [PATCH 13/19] add sleep Signed-off-by: crazycs520 --- cache/cache_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cache/cache_test.go b/cache/cache_test.go index c485f0d..5377b28 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -47,6 +47,7 @@ func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.F name := stat.Name() err = file.Close() c.Assert(err, IsNil) + time.Sleep(time.Millisecond * 10) file, stat = s.prepareFile(c, name) _, err = file.WriteString(data) c.Assert(err, IsNil) From 81e0b048af3ff60ef32c215085966b271502b6b0 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 13:22:18 +0800 Subject: [PATCH 14/19] remove sleep Signed-off-by: crazycs520 --- cache/cache_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index 5377b28..a01c59f 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -47,10 +47,11 @@ func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.F name := stat.Name() err = file.Close() c.Assert(err, IsNil) - time.Sleep(time.Millisecond * 10) file, stat = s.prepareFile(c, name) _, err = file.WriteString(data) c.Assert(err, IsNil) + err = file.Sync() + c.Assert(err, IsNil) err = file.Close() c.Assert(err, IsNil) return s.prepareFile(c, name) @@ -96,7 +97,7 @@ func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { c.Assert(err.Error(), Equals, "file stat can't be nil") // Test file has been modified. - file, stat = s.writeAndReopen(c, file, "a") + file, stat = s.writeAndReopen(c, file, "[2019/08/26 06:22:14.011 -04:00] [INFO] [printer.go:41] [\"Welcome to TiDB.\"]") // Test GetStartTime meet invalid error _, err = m.GetStartTime(stat, func() (time.Time, error) { @@ -169,7 +170,7 @@ func (s *testCacheSuite) TestLogFileMetaGetEndTime(c *C) { c.Assert(err.Error(), Equals, "file stat can't be nil") // Test file has been modified. - file, stat = s.writeAndReopen(c, file, "a") + file, stat = s.writeAndReopen(c, file, "[2019/08/26 06:22:14.011 -04:00] [INFO] [printer.go:41] [\"Welcome to TiDB.\"]") // Test GetEndTime meet invalid error _, err = m.GetEndTime(stat, func() (time.Time, error) { @@ -271,7 +272,7 @@ func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { c.Assert(m.CheckFileNotModified(stat), IsTrue) fmt.Printf("old mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) - file, stat = s.writeAndReopen(c, file, "abc") + file, stat = s.writeAndReopen(c, file, "[2019/08/26 06:22:14.011 -04:00] [INFO] [printer.go:41] [\"Welcome to TiDB.\"]") fmt.Printf("new mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) m = ca.GetFileMata(stat) c.Assert(m, NotNil) From cca321453dae90d343a9333540afcd251c972b82 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 13:27:01 +0800 Subject: [PATCH 15/19] add sleep in test Signed-off-by: crazycs520 --- cache/cache_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index a01c59f..26d3337 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -42,16 +42,13 @@ func (s *testCacheSuite) prepareFile(c *C, fileName string) (*os.File, os.FileIn } func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.File, os.FileInfo) { + // mock delay. + time.Sleep(time.Microsecond) stat, err := file.Stat() c.Assert(err, IsNil) name := stat.Name() - err = file.Close() - c.Assert(err, IsNil) - file, stat = s.prepareFile(c, name) _, err = file.WriteString(data) c.Assert(err, IsNil) - err = file.Sync() - c.Assert(err, IsNil) err = file.Close() c.Assert(err, IsNil) return s.prepareFile(c, name) From 2c5a418403d32441285dff59245b81e5e64d244f Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 13:29:09 +0800 Subject: [PATCH 16/19] remove log Signed-off-by: crazycs520 --- cache/cache_test.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index 26d3337..198a90c 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -44,14 +44,11 @@ func (s *testCacheSuite) prepareFile(c *C, fileName string) (*os.File, os.FileIn func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.File, os.FileInfo) { // mock delay. time.Sleep(time.Microsecond) - stat, err := file.Stat() - c.Assert(err, IsNil) - name := stat.Name() - _, err = file.WriteString(data) + _, err := file.WriteString(data) c.Assert(err, IsNil) - err = file.Close() + stat, err := file.Stat() c.Assert(err, IsNil) - return s.prepareFile(c, name) + return file, stat } func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { @@ -268,9 +265,7 @@ func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { c.Assert(m.IsInValid(), IsFalse) c.Assert(m.CheckFileNotModified(stat), IsTrue) - fmt.Printf("old mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) file, stat = s.writeAndReopen(c, file, "[2019/08/26 06:22:14.011 -04:00] [INFO] [printer.go:41] [\"Welcome to TiDB.\"]") - fmt.Printf("new mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) m = ca.GetFileMata(stat) c.Assert(m, NotNil) c.Assert(m.CheckFileNotModified(stat), IsFalse) From 0698780c9aeaf207224167053c17ea99627e0444 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 14:15:47 +0800 Subject: [PATCH 17/19] add sync Signed-off-by: crazycs520 --- cache/cache_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index 198a90c..e0dcee4 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -43,9 +43,11 @@ func (s *testCacheSuite) prepareFile(c *C, fileName string) (*os.File, os.FileIn func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.File, os.FileInfo) { // mock delay. - time.Sleep(time.Microsecond) + time.Sleep(time.Millisecond) _, err := file.WriteString(data) c.Assert(err, IsNil) + err = file.Sync() + c.Assert(err, IsNil) stat, err := file.Stat() c.Assert(err, IsNil) return file, stat From 11d7119227b8424fa8a40268ae661ebeebb89118 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 14:19:27 +0800 Subject: [PATCH 18/19] make test stable Signed-off-by: crazycs520 --- cache/cache_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cache/cache_test.go b/cache/cache_test.go index e0dcee4..9256520 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -46,11 +46,12 @@ func (s *testCacheSuite) writeAndReopen(c *C, file *os.File, data string) (*os.F time.Sleep(time.Millisecond) _, err := file.WriteString(data) c.Assert(err, IsNil) - err = file.Sync() - c.Assert(err, IsNil) stat, err := file.Stat() c.Assert(err, IsNil) - return file, stat + name := stat.Name() + err = file.Close() + c.Assert(err, IsNil) + return s.prepareFile(c, name) } func (s *testCacheSuite) TestLogFileMetaGetStartTime(c *C) { From 87f4c930a12364ef7254d51fcf8a20bfc920a2c2 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Tue, 8 Dec 2020 14:33:44 +0800 Subject: [PATCH 19/19] Revert "remove log" This reverts commit 2c5a418403d32441285dff59245b81e5e64d244f. --- cache/cache_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cache/cache_test.go b/cache/cache_test.go index 9256520..26dd47c 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -268,7 +268,9 @@ func (s *testCacheSuite) TestLogFileMetaCacheWithCap(c *C) { c.Assert(m.IsInValid(), IsFalse) c.Assert(m.CheckFileNotModified(stat), IsTrue) + fmt.Printf("old mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) file, stat = s.writeAndReopen(c, file, "[2019/08/26 06:22:14.011 -04:00] [INFO] [printer.go:41] [\"Welcome to TiDB.\"]") + fmt.Printf("new mod time: \n%v\n%v -%v--\n", stat.ModTime(), m.ModTime, stat.Size()) m = ca.GetFileMata(stat) c.Assert(m, NotNil) c.Assert(m.CheckFileNotModified(stat), IsFalse)