database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ddl/ingest: fix ingest failure due to memory used up (#47863)

Browse Source 
close pingcap/tidb#47862
This commit is contained in:
GMHDBJD
2023-11-06 20:09:41 +08:00
committed by GitHub
parent f04ed92907
commit 5fe83fcc8e
2 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/ddl/ingest/BUILD.bazel
View File

@ -40,6 +40,7 @@ go_library(
"//pkg/util/dbterror",
"//pkg/util/generic",
"//pkg/util/logutil",
"//pkg/util/memory",
"//pkg/util/size",
"@com_github_google_uuid//:uuid",
"@com_github_pingcap_errors//:errors",

14
pkg/ddl/ingest/env.go
View File

@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/memory"
"github.com/pingcap/tidb/pkg/util/size"
"go.uber.org/zap"
)
@ -44,7 +45,7 @@ var (
LitInitialized bool
)
const maxMemoryQuota = 2 * size.GB
const defaultMemoryQuota = 2 * size.GB
// InitGlobalLightningEnv initialize Lightning backfill environment.
func InitGlobalLightningEnv(ctx context.Context, sctx sessionctx.Context) {
@ -66,12 +67,19 @@ func InitGlobalLightningEnv(ctx context.Context, sctx sessionctx.Context) {
return
}
LitSortPath = sPath
LitBackCtxMgr = newLitBackendCtxMgr(ctx, sctx, LitSortPath, maxMemoryQuota)
memTotal, err := memory.MemTotal()
if err != nil {
logutil.BgLogger().Warn("get total memory fail", zap.Error(err))
memTotal = defaultMemoryQuota
} else {
memTotal = memTotal / 2
}
LitBackCtxMgr = newLitBackendCtxMgr(ctx, sctx, LitSortPath, memTotal)
LitRLimit = util.GenRLimit("ddl-ingest")
LitInitialized = true
logutil.BgLogger().Info(LitInfoEnvInitSucc,
zap.String("category", "ddl-ingest"),
zap.Uint64("memory limitation", maxMemoryQuota),
zap.Uint64("memory limitation", memTotal),
zap.String("disk usage info", LitDiskRoot.UsageInfo()),
zap.Uint64("max open file number", LitRLimit),
zap.Bool("lightning is initialized", LitInitialized))