From 5fe83fcc8ee476ac904e868042eea56529febc81 Mon Sep 17 00:00:00 2001
From: GMHDBJD <35025882+GMHDBJD@users.noreply.github.com>
Date: Mon, 6 Nov 2023 20:09:41 +0800
Subject: [PATCH] ddl/ingest: fix ingest failure due to memory used up (#47863)

close pingcap/tidb#47862
---
 pkg/ddl/ingest/BUILD.bazel |  1 +
 pkg/ddl/ingest/env.go      | 14 +++++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/pkg/ddl/ingest/BUILD.bazel b/pkg/ddl/ingest/BUILD.bazel
index 2e51d87b36..b1423b7411 100644
--- a/pkg/ddl/ingest/BUILD.bazel
+++ b/pkg/ddl/ingest/BUILD.bazel
@@ -40,6 +40,7 @@ go_library(
         "//pkg/util/dbterror",
         "//pkg/util/generic",
         "//pkg/util/logutil",
+        "//pkg/util/memory",
         "//pkg/util/size",
         "@com_github_google_uuid//:uuid",
         "@com_github_pingcap_errors//:errors",
diff --git a/pkg/ddl/ingest/env.go b/pkg/ddl/ingest/env.go
index ab440ca6fe..bb4703d7cb 100644
--- a/pkg/ddl/ingest/env.go
+++ b/pkg/ddl/ingest/env.go
@@ -25,6 +25,7 @@ import (
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/util"
 	"github.com/pingcap/tidb/pkg/util/logutil"
+	"github.com/pingcap/tidb/pkg/util/memory"
 	"github.com/pingcap/tidb/pkg/util/size"
 	"go.uber.org/zap"
 )
@@ -44,7 +45,7 @@ var (
 	LitInitialized bool
 )
 
-const maxMemoryQuota = 2 * size.GB
+const defaultMemoryQuota = 2 * size.GB
 
 // InitGlobalLightningEnv initialize Lightning backfill environment.
 func InitGlobalLightningEnv(ctx context.Context, sctx sessionctx.Context) {
@@ -66,12 +67,19 @@ func InitGlobalLightningEnv(ctx context.Context, sctx sessionctx.Context) {
 		return
 	}
 	LitSortPath = sPath
-	LitBackCtxMgr = newLitBackendCtxMgr(ctx, sctx, LitSortPath, maxMemoryQuota)
+	memTotal, err := memory.MemTotal()
+	if err != nil {
+		logutil.BgLogger().Warn("get total memory fail", zap.Error(err))
+		memTotal = defaultMemoryQuota
+	} else {
+		memTotal = memTotal / 2
+	}
+	LitBackCtxMgr = newLitBackendCtxMgr(ctx, sctx, LitSortPath, memTotal)
 	LitRLimit = util.GenRLimit("ddl-ingest")
 	LitInitialized = true
 	logutil.BgLogger().Info(LitInfoEnvInitSucc,
 		zap.String("category", "ddl-ingest"),
-		zap.Uint64("memory limitation", maxMemoryQuota),
+		zap.Uint64("memory limitation", memTotal),
 		zap.String("disk usage info", LitDiskRoot.UsageInfo()),
 		zap.Uint64("max open file number", LitRLimit),
 		zap.Bool("lightning is initialized", LitInitialized))