store/tikv: Add metrics for async secondary lock cleanup (#5575)

store/tikv/2pc.go

@@ -227,6 +227,7 @@ func (c *twoPhaseCommitter) doActionOnKeys(bo *Backoffer, action twoPhaseCommitA
 			e := c.doActionOnBatches(bo, action, batches)
 			if e != nil {
 				log.Debugf("2PC async doActionOnBatches %s err: %v", action, e)
+				secondaryLockCleanupFailureCounter.WithLabelValues("commit").Inc()
 			}
 		})
 	} else {
@@ -572,6 +573,7 @@ func (c *twoPhaseCommitter) execute(ctx goctx.Context) error {
 			twoPhaseCommitGP.Go(func() {
 				err := c.cleanupKeys(NewBackoffer(cleanupMaxBackoff, goctx.Background()), writtenKeys)
 				if err != nil {
+					secondaryLockCleanupFailureCounter.WithLabelValues("rollback").Inc()
 					log.Infof("2PC cleanup err: %v, tid: %d", err, c.startTS)
 				} else {
 					log.Infof("2PC clean up done, tid: %d", c.startTS)

store/tikv/metrics.go

@@ -172,6 +172,14 @@ var (
 			Name:      "load_safepoint_total",
 			Help:      "Counter of load safepoint.",
 		}, []string{"type"})
+
+	secondaryLockCleanupFailureCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "tidb",
+			Subsystem: "tikvclient",
+			Name:      "lock_cleanup_task_total",
+			Help:      "failure statistic of secondary lock cleanup task.",
+		}, []string{"type"})
 )
 
 func reportRegionError(e *errorpb.Error) {
@@ -213,4 +221,5 @@ func init() {
 	prometheus.MustRegister(rawkvSizeHistogram)
 	prometheus.MustRegister(txnRegionsNumHistogram)
 	prometheus.MustRegister(loadSafepointCounter)
+	prometheus.MustRegister(secondaryLockCleanupFailureCounter)
 }