backend: ensure engine cleanup to be executed (#64105)

close pingcap/tidb#64101

pkg/ddl/ingest/engine.go

@@ -54,6 +54,7 @@ type engineInfo struct {
 	openedEngine *backend.OpenedEngine
 
 	uuid        uuid.UUID
+	backend     backend.Backend
 	writerCache generic.SyncMap[int, backend.EngineWriter]
 	memRoot     MemRoot
 	flushLock   *sync.RWMutex
@@ -66,6 +67,7 @@ func newEngineInfo(
 	unique bool,
 	en *backend.OpenedEngine,
 	uuid uuid.UUID,
+	bk backend.Backend,
 	memRoot MemRoot,
 ) *engineInfo {
 	return &engineInfo{
@@ -75,6 +77,7 @@ func newEngineInfo(
 		unique:       unique,
 		openedEngine: en,
 		uuid:         uuid,
+		backend:      bk,
 		writerCache:  generic.NewSyncMap[int, backend.EngineWriter](4),
 		memRoot:      memRoot,
 		flushLock:    &sync.RWMutex{},
@@ -104,26 +107,25 @@ func (ei *engineInfo) Close(cleanup bool) {
 	}
 	err := ei.closeWriters()
 	if err != nil {
-		logutil.Logger(ei.ctx).Error(LitErrCloseWriterErr, zap.Error(err),
+		logutil.Logger(ei.ctx).Warn(LitErrCloseWriterErr, zap.Error(err),
 			zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
 	}
-
-	indexEngine := ei.openedEngine
-	closedEngine, err := indexEngine.Close(ei.ctx)
+	if cleanup {
+		defer func() {
+			err = ei.backend.CleanupEngine(ei.ctx, ei.uuid)
+			if err != nil {
+				logutil.Logger(ei.ctx).Warn(LitErrCleanEngineErr, zap.Error(err),
+					zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
+			}
+		}()
+	}
+	_, err = ei.openedEngine.Close(ei.ctx)
 	if err != nil {
-		logutil.Logger(ei.ctx).Error(LitErrCloseEngineErr, zap.Error(err),
+		logutil.Logger(ei.ctx).Warn(LitErrCloseEngineErr, zap.Error(err),
 			zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
 		return
 	}
 	ei.openedEngine = nil
-	if cleanup {
-		// local intermediate files will be removed.
-		err = closedEngine.Cleanup(ei.ctx)
-		if err != nil {
-			logutil.Logger(ei.ctx).Error(LitErrCleanEngineErr, zap.Error(err),
-				zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
-		}
-	}
 }
 
 // writerContext is used to keep a lightning local writer for each backfill worker.

pkg/ddl/ingest/engine_mgr.go

@@ -79,6 +79,7 @@ func (bc *litBackendCtx) Register(indexIDs []int64, uniques []bool, tbl table.Ta
 			uniques[i],
 			openedEngine,
 			openedEngine.GetEngineUUID(),
+			bc.backend,
 			bc.memRoot,
 		)
 	}

pkg/lightning/backend/local/engine.go

@@ -1568,6 +1568,13 @@ type dbSSTIngester struct {
 }
 
 func (i dbSSTIngester) mergeSSTs(metas []*sstMeta, dir string, blockSize int) (*sstMeta, error) {
+	failpoint.InjectCall("beforeMergeSSTs")
+	failpoint.Inject("mockErrInMergeSSTs", func(val failpoint.Value) {
+		if val.(bool) {
+			failpoint.Return(nil, errors.New("mocked error in mergeSSTs"))
+		}
+	})
+
 	if len(metas) == 0 {
 		return nil, errors.New("sst metas is empty")
 	} else if len(metas) == 1 {