From 1bdfc807e5f8fa57e76b49fedbd75e6b43fcf81d Mon Sep 17 00:00:00 2001 From: Zhou Kunqin <25057648+time-and-fate@users.noreply.github.com> Date: Wed, 10 May 2023 13:12:07 +0800 Subject: [PATCH] domain, util/replayer: increase GC duration for plan replayer capture file to 7 days (#43664) close pingcap/tidb#43663 --- domain/BUILD.bazel | 2 +- domain/domain.go | 2 +- domain/plan_replayer.go | 18 ++++++++---- domain/plan_replayer_test.go | 55 +++++++++++++++++++++++++++++++++--- util/replayer/BUILD.bazel | 1 + util/replayer/replayer.go | 10 ++++++- 6 files changed, 76 insertions(+), 12 deletions(-) diff --git a/domain/BUILD.bazel b/domain/BUILD.bazel index 4f9cfbaaf9..003da86fe9 100644 --- a/domain/BUILD.bazel +++ b/domain/BUILD.bazel @@ -117,7 +117,7 @@ go_test( ], embed = [":domain"], flaky = True, - shard_count = 22, + shard_count = 23, deps = [ "//config", "//ddl", diff --git a/domain/domain.go b/domain/domain.go index d2bdf875c0..01ae37908d 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -1979,7 +1979,7 @@ func (do *Domain) DumpFileGcCheckerLoop() { case <-do.exit: return case <-gcTicker.C: - do.dumpFileGcChecker.gcDumpFiles(time.Hour) + do.dumpFileGcChecker.gcDumpFiles(time.Hour, time.Hour*24*7) } } }, "dumpFileGcChecker") diff --git a/domain/plan_replayer.go b/domain/plan_replayer.go index 97d5e1673f..e0ad326e1b 100644 --- a/domain/plan_replayer.go +++ b/domain/plan_replayer.go @@ -74,11 +74,11 @@ func parseTime(s string) (time.Time, error) { return time.Unix(0, i), nil } -func (p *dumpFileGcChecker) gcDumpFiles(t time.Duration) { +func (p *dumpFileGcChecker) gcDumpFiles(gcDurationDefault, gcDurationForCapture time.Duration) { p.Lock() defer p.Unlock() for _, path := range p.paths { - p.gcDumpFilesByPath(path, t) + p.gcDumpFilesByPath(path, gcDurationDefault, gcDurationForCapture) } } @@ -86,7 +86,7 @@ func (p *dumpFileGcChecker) setupSctx(sctx sessionctx.Context) { p.sctx = sctx } -func (p *dumpFileGcChecker) gcDumpFilesByPath(path string, t time.Duration) { +func (p *dumpFileGcChecker) gcDumpFilesByPath(path string, gcDurationDefault, gcDurationForCapture time.Duration) { files, err := ioutil.ReadDir(path) if err != nil { if !os.IsNotExist(err) { @@ -94,7 +94,8 @@ func (p *dumpFileGcChecker) gcDumpFilesByPath(path string, t time.Duration) { } } - gcTime := time.Now().Add(-t) + gcTargetTimeDefault := time.Now().Add(-gcDurationDefault) + gcTargetTimeForCapture := time.Now().Add(-gcDurationForCapture) for _, f := range files { fileName := f.Name() createTime, err := parseTime(fileName) @@ -103,7 +104,14 @@ func (p *dumpFileGcChecker) gcDumpFilesByPath(path string, t time.Duration) { continue } isPlanReplayer := strings.Contains(fileName, "replayer") - if !createTime.After(gcTime) { + isPlanReplayerCapture := strings.Contains(fileName, "capture") + canGC := false + if isPlanReplayer && isPlanReplayerCapture { + canGC = !createTime.After(gcTargetTimeForCapture) + } else { + canGC = !createTime.After(gcTargetTimeDefault) + } + if canGC { err := os.Remove(filepath.Join(path, f.Name())) if err != nil { logutil.BgLogger().Warn("[dumpFileGcChecker] remove file failed", zap.Error(err), zap.String("filename", fileName)) diff --git a/domain/plan_replayer_test.go b/domain/plan_replayer_test.go index f2e002b29d..d93e3ea1e8 100644 --- a/domain/plan_replayer_test.go +++ b/domain/plan_replayer_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/util/replayer" "github.com/stretchr/testify/require" ) @@ -39,11 +40,57 @@ func TestPlanReplayerGC(t *testing.T) { handler := &dumpFileGcChecker{ paths: []string{replayer.GetPlanReplayerDirName()}, } - handler.gcDumpFiles(0) + handler.gcDumpFiles(0, 0) + require.NoFileExists(t, path) +} - _, err = os.Stat(path) - require.NotNil(t, err) - require.True(t, os.IsNotExist(err)) +func TestPlanReplayerDifferentGC(t *testing.T) { + dirName := replayer.GetPlanReplayerDirName() + + time1 := time.Now().Add(-7 * 25 * time.Hour).UnixNano() + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time1))) + file1, fileName1, err := replayer.GeneratePlanReplayerFile(true, false, false) + require.NoError(t, err) + require.NoError(t, file1.Close()) + filePath1 := filepath.Join(dirName, fileName1) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField")) + + time2 := time.Now().Add(-7 * 23 * time.Hour).UnixNano() + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time2))) + file2, fileName2, err := replayer.GeneratePlanReplayerFile(true, false, false) + require.NoError(t, err) + require.NoError(t, file2.Close()) + filePath2 := filepath.Join(dirName, fileName2) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField")) + + time3 := time.Now().Add(-2 * time.Hour).UnixNano() + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time3))) + file3, fileName3, err := replayer.GeneratePlanReplayerFile(false, false, false) + require.NoError(t, err) + require.NoError(t, file3.Close()) + filePath3 := filepath.Join(dirName, fileName3) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField")) + + time4 := time.Now().UnixNano() + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time4))) + file4, fileName4, err := replayer.GeneratePlanReplayerFile(false, false, false) + require.NoError(t, err) + require.NoError(t, file4.Close()) + filePath4 := filepath.Join(dirName, fileName4) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/replayer/InjectPlanReplayerFileNameTimeField")) + + handler := &dumpFileGcChecker{ + paths: []string{dirName}, + } + handler.gcDumpFiles(time.Hour, time.Hour*24*7) + require.NoFileExists(t, filePath1) + require.FileExists(t, filePath2) + require.NoFileExists(t, filePath3) + require.FileExists(t, filePath4) + + handler.gcDumpFiles(0, 0) + require.NoFileExists(t, filePath2) + require.NoFileExists(t, filePath4) } func TestDumpGCFileParseTime(t *testing.T) { diff --git a/util/replayer/BUILD.bazel b/util/replayer/BUILD.bazel index bd04735f32..3dee9377ff 100644 --- a/util/replayer/BUILD.bazel +++ b/util/replayer/BUILD.bazel @@ -8,5 +8,6 @@ go_library( deps = [ "//config", "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", ], ) diff --git a/util/replayer/replayer.go b/util/replayer/replayer.go index 226154be31..489747f5f2 100644 --- a/util/replayer/replayer.go +++ b/util/replayer/replayer.go @@ -15,14 +15,15 @@ package replayer import ( + "crypto/rand" "encoding/base64" "fmt" - "math/rand" "os" "path/filepath" "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" ) @@ -58,6 +59,9 @@ func GeneratePlanReplayerFileName(isCapture, isContinuesCapture, enableHistorica func generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture bool) (string, error) { // Generate key and create zip file time := time.Now().UnixNano() + failpoint.Inject("InjectPlanReplayerFileNameTimeField", func(val failpoint.Value) { + time = int64(val.(int)) + }) b := make([]byte, 16) //nolint: gosec _, err := rand.Read(b) @@ -65,9 +69,13 @@ func generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistorica return "", err } key := base64.URLEncoding.EncodeToString(b) + // "capture_replayer" in filename has special meaning for the /plan_replayer/dump/ HTTP handler if isContinuesCapture || isCapture && enableHistoricalStatsForCapture { return fmt.Sprintf("capture_replayer_%v_%v.zip", key, time), nil } + if isCapture && !enableHistoricalStatsForCapture { + return fmt.Sprintf("capture_normal_replayer_%v_%v.zip", key, time), nil + } return fmt.Sprintf("replayer_%v_%v.zip", key, time), nil }