add metrics for gc failure count (#5694)

store/tikv/gcworker/gc_worker.go

@@ -304,16 +304,19 @@ func (w *GCWorker) runGCJob(ctx goctx.Context, safePoint uint64) {
 	gcWorkerCounter.WithLabelValues("run_job").Inc()
 	err := resolveLocks(ctx, w.store, safePoint, w.uuid)
 	if err != nil {
+		gcFailureCounter.WithLabelValues("resolve_lock").Inc()
 		w.done <- errors.Trace(err)
 		return
 	}
 	err = w.deleteRanges(ctx, safePoint)
 	if err != nil {
+		gcFailureCounter.WithLabelValues("delete_range").Inc()
 		w.done <- errors.Trace(err)
 		return
 	}
 	err = doGC(ctx, w.store, safePoint, w.uuid)
 	if err != nil {
+		gcFailureCounter.WithLabelValues("gc").Inc()
 		log.Error("do GC returns an error", err)
 		w.gcIsRunning = false
 		w.done <- errors.Trace(err)

store/tikv/gcworker/misc.go

@@ -41,12 +41,20 @@ var (
 			Subsystem: "tikvclient",
 			Name:      "gc_config",
 			Help:      "Gauge of GC configs.",
-		}, []string{"type"},
-	)
+		}, []string{"type"})
+
+	gcFailureCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "tidb",
+			Subsystem: "tikvclient",
+			Name:      "gc_failure",
+			Help:      "Counter of gc job failure.",
+		}, []string{"type"})
 )
 
 func init() {
 	prometheus.MustRegister(gcWorkerCounter)
 	prometheus.MustRegister(gcConfigGauge)
 	prometheus.MustRegister(gcHistogram)
+	prometheus.MustRegister(gcFailureCounter)
 }