util/intest: support enable internal check by failpoint (#60141)

close pingcap/tidb#60142

cmd/tidb-server/BUILD.bazel

@@ -41,6 +41,7 @@ go_library(
         "//pkg/util/deadlockhistory",
         "//pkg/util/disk",
         "//pkg/util/domainutil",
+        "//pkg/util/intest",
         "//pkg/util/kvcache",
         "//pkg/util/logutil",
         "//pkg/util/memory",

cmd/tidb-server/main.go

@@ -66,6 +66,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/deadlockhistory"
 	"github.com/pingcap/tidb/pkg/util/disk"
 	"github.com/pingcap/tidb/pkg/util/domainutil"
+	"github.com/pingcap/tidb/pkg/util/intest"
 	"github.com/pingcap/tidb/pkg/util/kvcache"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/memory"
@@ -310,6 +311,9 @@ func main() {
 		logutil.BgLogger().Warn(warnMsg)
 		tikv.EnableFailpoints()
 	}
+	if intest.EnableInternalCheck {
+		logutil.BgLogger().Warn("internal check is enabled, this should NOT happen in the production environment")
+	}
 	setGlobalVars()
 	setCPUAffinity()
 	cgmon.StartCgroupMonitor()

pkg/util/intest/BUILD.bazel

@@ -4,12 +4,14 @@ go_library(
     name = "intest",
     srcs = [
         "assert.go",  #keep
+        "assert_common.go",
         "in_unittest.go",  #keep
         "no_assert.go",
         "not_in_unittest.go",
     ],
     importpath = "github.com/pingcap/tidb/pkg/util/intest",
     visibility = ["//visibility:public"],
+    deps = ["@com_github_pingcap_failpoint//:failpoint"],
 )
 
 go_test(

pkg/util/intest/assert.go

@@ -16,74 +16,33 @@
 
 package intest
 
-import (
-	"fmt"
-	"reflect"
-)
-
 // EnableAssert checks if the assert function should work.
-const EnableAssert = true
+var EnableAssert = true
 
 // Assert asserts a condition is true
 func Assert(cond bool, msgAndArgs ...any) {
-	if EnableAssert && !cond {
-		doPanic("", msgAndArgs...)
+	if EnableAssert || EnableInternalCheck {
+		doAssert(cond, msgAndArgs...)
 	}
 }
 
 // AssertNoError asserts an error is nil
 func AssertNoError(err error, msgAndArgs ...any) {
-	if EnableAssert && err != nil {
-		doPanic(fmt.Sprintf("error is not nil: %+v", err), msgAndArgs...)
+	if EnableAssert || EnableInternalCheck {
+		doAssertNoError(err, msgAndArgs...)
 	}
 }
 
 // AssertNotNil asserts an object is not nil
 func AssertNotNil(obj any, msgAndArgs ...any) {
-	if EnableAssert {
-		Assert(obj != nil, msgAndArgs...)
-		value := reflect.ValueOf(obj)
-		switch value.Kind() {
-		case reflect.Func, reflect.Chan, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer:
-			Assert(!value.IsNil(), msgAndArgs...)
-		}
+	if EnableAssert || EnableInternalCheck {
+		doAssertNotNil(obj, msgAndArgs...)
 	}
 }
 
 // AssertFunc asserts a function condition
 func AssertFunc(fn func() bool, msgAndArgs ...any) {
-	if EnableAssert {
-		Assert(fn != nil, msgAndArgs...)
-		Assert(fn(), msgAndArgs...)
+	if EnableAssert || EnableInternalCheck {
+		doAssertFunc(fn, msgAndArgs...)
 	}
 }
-
-func doPanic(extraMsg string, userMsgAndArgs ...any) {
-	panic(assertionFailedMsg(extraMsg, userMsgAndArgs...))
-}
-
-func assertionFailedMsg(extraMsg string, userMsgAndArgs ...any) string {
-	msg := "assert failed"
-	if len(userMsgAndArgs) == 0 {
-		if extraMsg != "" {
-			msg = fmt.Sprintf("%s, %s", msg, extraMsg)
-		}
-		return msg
-	}
-
-	if len(userMsgAndArgs) == 0 {
-		return fmt.Sprintf("assert failed, %s", extraMsg)
-	}
-
-	userMsg, ok := userMsgAndArgs[0].(string)
-	if !ok {
-		userMsg = fmt.Sprintf("%+v", userMsgAndArgs[0])
-	}
-
-	msg = fmt.Sprintf("%s, %s", msg, userMsg)
-	if extraMsg != "" {
-		msg = fmt.Sprintf("%s, %s", msg, extraMsg)
-	}
-
-	return fmt.Sprintf(msg, userMsgAndArgs[1:]...)
-}

pkg/util/intest/assert_common.go

@@ -0,0 +1,99 @@
+// Copyright 2025 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package intest
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/pingcap/failpoint"
+)
+
+// EnableInternalCheck is a general switch to enable internal check.
+var EnableInternalCheck = false
+
+// Assert asserts a condition is true
+func doAssert(cond bool, msgAndArgs ...any) {
+	if !cond {
+		doPanic("", msgAndArgs...)
+	}
+}
+
+// AssertNoError asserts an error is nil
+func doAssertNoError(err error, msgAndArgs ...any) {
+	if err != nil {
+		doPanic(fmt.Sprintf("error is not nil: %+v", err), msgAndArgs...)
+	}
+}
+
+// AssertNotNil asserts an object is not nil
+func doAssertNotNil(obj any, msgAndArgs ...any) {
+	doAssert(obj != nil, msgAndArgs...)
+	value := reflect.ValueOf(obj)
+	switch value.Kind() {
+	case reflect.Func, reflect.Chan, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer:
+		doAssert(!value.IsNil(), msgAndArgs...)
+	}
+}
+
+// AssertFunc asserts a function condition
+func doAssertFunc(fn func() bool, msgAndArgs ...any) {
+	doAssert(fn != nil, msgAndArgs...)
+	doAssert(fn(), msgAndArgs...)
+}
+
+func doPanic(extraMsg string, userMsgAndArgs ...any) {
+	panic(assertionFailedMsg(extraMsg, userMsgAndArgs...))
+}
+
+func assertionFailedMsg(extraMsg string, userMsgAndArgs ...any) string {
+	msg := "assert failed"
+	if len(userMsgAndArgs) == 0 {
+		if extraMsg != "" {
+			msg = fmt.Sprintf("%s, %s", msg, extraMsg)
+		}
+		return msg
+	}
+
+	if len(userMsgAndArgs) == 0 {
+		return fmt.Sprintf("assert failed, %s", extraMsg)
+	}
+
+	userMsg, ok := userMsgAndArgs[0].(string)
+	if !ok {
+		userMsg = fmt.Sprintf("%+v", userMsgAndArgs[0])
+	}
+
+	msg = fmt.Sprintf("%s, %s", msg, userMsg)
+	if extraMsg != "" {
+		msg = fmt.Sprintf("%s, %s", msg, extraMsg)
+	}
+
+	return fmt.Sprintf(msg, userMsgAndArgs[1:]...)
+}
+
+func init() {
+	if InTest || EnableAssert {
+		EnableInternalCheck = true
+	}
+	// Use `export GO_FAILPOINTS="/enableInternalCheck=return(true)"` to enable internal check.
+	// The path is "/" instead of "pingcap/tidb/pkg/intest/enableInternalCheck" because of the init().
+	failpoint.Inject("enableInternalCheck", func(val failpoint.Value) {
+		if val.(bool) {
+			EnableInternalCheck = true
+			EnableAssert = true
+		}
+	})
+}

pkg/util/intest/in_unittest.go

@@ -17,4 +17,4 @@
 package intest
 
 // InTest checks if the code is running in test.
-const InTest = true
+var InTest = true

pkg/util/intest/no_assert.go

@@ -17,20 +17,32 @@
 package intest
 
 // EnableAssert checks if the code is running in integration test.
-const EnableAssert = false
+var EnableAssert = false
 
-// Assert is a stub function in release build.
-// See the same function in `util/intest/assert.go` for the real implement in test.
-func Assert(_ bool, _ ...any) {}
+// Assert asserts a condition is true
+func Assert(cond bool, msgAndArgs ...any) {
+	if EnableInternalCheck {
+		doAssert(cond, msgAndArgs...)
+	}
+}
 
-// AssertNotNil is a stub function in release build.
-// See the same function in `util/intest/assert.go` for the real implement in test.
-func AssertNotNil(_ any, _ ...any) {}
+// AssertNoError asserts an error is nil
+func AssertNoError(err error, msgAndArgs ...any) {
+	if EnableInternalCheck {
+		doAssertNoError(err, msgAndArgs...)
+	}
+}
 
-// AssertNoError is a stub function in release build.
-// See the same function in `util/intest/assert.go` for the real implement in test.
-func AssertNoError(_ error, _ ...any) {}
+// AssertNotNil asserts an object is not nil
+func AssertNotNil(obj any, msgAndArgs ...any) {
+	if EnableInternalCheck {
+		doAssertNotNil(obj, msgAndArgs...)
+	}
+}
 
-// AssertFunc is a stub function in release build.
-// See the same function `util/intest/assert.go` for the real implement in test.
-func AssertFunc(_ func() bool, _ ...any) {}
+// AssertFunc asserts a function condition
+func AssertFunc(fn func() bool, msgAndArgs ...any) {
+	if EnableInternalCheck {
+		doAssertFunc(fn, msgAndArgs...)
+	}
+}

pkg/util/intest/not_in_unittest.go

@@ -17,4 +17,4 @@
 package intest
 
 // InTest checks if the code is running in test.
-const InTest = false
+var InTest = false