database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

*: move config file options prepared-plan-cache.* to sysvars (#34790)

Browse Source 
close pingcap/tidb#30168, ref pingcap/tidb#33769
This commit is contained in:
Yuanjia Zhang
2022-05-19 16:18:39 +08:00
committed by GitHub
parent 2f4b11dc26
commit 0cbccb74f3
15 changed files with 127 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
config/config.go
View File

@ -956,7 +956,10 @@ var deprecatedConfig = map[string]struct{}{
"performance.committer-concurrency": {},
"experimental.enable-global-kill": {},
"performance.run-auto-analyze": {}, //use tidb_enable_auto_analyze
// use tidb_enable_prepared_plan_cache, tidb_prepared_plan_cache_size and tidb_prepared_plan_cache_memory_guard_ratio
"prepared-plan-cache.enabled": {},
"prepared-plan-cache.capacity": {},
"prepared-plan-cache.memory-guard-ratio": {},
}
func isAllDeprecatedConfigItems(items []string) bool {
@ -1123,12 +1126,6 @@ func (c *Config) Valid() error {
return fmt.Errorf("tidb_memory_usage_alarm_ratio in [Instance] must be greater than or equal to 0 and less than or equal to 1")
}
if c.PreparedPlanCache.Capacity < 1 {
return fmt.Errorf("capacity in [prepared-plan-cache] should be at least 1")
}
if c.PreparedPlanCache.MemoryGuardRatio < 0 || c.PreparedPlanCache.MemoryGuardRatio > 1 {
return fmt.Errorf("memory-guard-ratio in [prepared-plan-cache] must be NOT less than 0 and more than 1")
}
if len(c.IsolationRead.Engines) < 1 {
return fmt.Errorf("the number of [isolation-read]engines for isolation read should be at least 1")
}

5
config/config.toml.example
View File

@ -275,11 +275,6 @@ networks = ""
# PROXY protocol header read timeout, unit is second
header-timeout = 5
[prepared-plan-cache]
enabled = false
capacity = 1000
memory-guard-ratio = 0.1
[opentracing]
# Enable opentracing.
enable = false

15
config/config_test.go
View File

@ -492,21 +492,6 @@ func TestTxnTotalSizeLimitValid(t *testing.T) {
}
}
func TestPreparePlanCacheValid(t *testing.T) {
conf := NewConfig()
tests := map[PreparedPlanCache]bool{
{Enabled: true, Capacity: 0}: false,
{Enabled: true, Capacity: 2}: true,
{Enabled: true, MemoryGuardRatio: -0.1}: false,
{Enabled: true, MemoryGuardRatio: 2.2}: false,
{Enabled: true, Capacity: 2, MemoryGuardRatio: 0.5}: true,
}
for testCase, res := range tests {
conf.PreparedPlanCache = testCase
require.Equal(t, res, conf.Valid() == nil)
}
}
func TestConflictInstanceConfig(t *testing.T) {
var expectedNewName string
conf := new(Config)

1
config/config_util.go
View File

@ -54,7 +54,6 @@ var (
"CompatibleKillQuery": {},
"TreatOldVersionUTF8AsUTF8MB4": {},
"OpenTracing.Enable": {},
"PreparedPlanCache.Enabled": {},
}
)

9
executor/explainfor_test.go
View File

@ -474,14 +474,11 @@ func TestExplainTiFlashSystemTables(t *testing.T) {
}
func TestPointGetUserVarPlanCache(t *testing.T) {
orgEnable := core.PreparedPlanCacheEnabled()
defer func() {
core.SetPreparedPlanCache(orgEnable)
}()
core.SetPreparedPlanCache(true)
store, clean := testkit.CreateMockStore(t)
defer clean()
tmp := testkit.NewTestKit(t, store)
defer tmp.MustExec("set global tidb_enable_prepared_plan_cache=" + variable.BoolToOnOff(variable.EnablePreparedPlanCache.Load()))
tmp.MustExec("set global tidb_enable_prepared_plan_cache=ON")
tk := testkit.NewTestKit(t, store)
tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost", CurrentUser: true, AuthUsername: "root", AuthHostname: "%"}, nil, []byte("012345678901234567890"))

38
executor/set_test.go
View File

@ -1642,6 +1642,44 @@ func TestSetTopSQLVariables(t *testing.T) {
tk.MustQuery("show global variables like '%top_sql%'").Check(testkit.Rows("tidb_enable_top_sql OFF", "tidb_top_sql_max_meta_count 5000", "tidb_top_sql_max_time_series_count 20"))
}
func TestPreparePlanCacheValid(t *testing.T) {
store, clean := testkit.CreateMockStore(t)
defer clean()
tk := testkit.NewTestKit(t, store)
tk.MustExec("SET GLOBAL tidb_prepared_plan_cache_size = 0")
tk.MustQuery("show warnings").Check(testkit.Rows(
"Warning 1292 Truncated incorrect tidb_prepared_plan_cache_size value: '0'"))
tk.MustQuery("select @@global.tidb_prepared_plan_cache_size").Check(testkit.Rows("1"))
tk.MustExec("SET GLOBAL tidb_prepared_plan_cache_size = 2")
tk.MustQuery("select @@global.tidb_prepared_plan_cache_size").Check(testkit.Rows("2"))
tk.MustExec("SET GLOBAL tidb_prepared_plan_cache_memory_guard_ratio = -0.1")
tk.MustQuery("show warnings").Check(testkit.Rows(
"Warning 1292 Truncated incorrect tidb_prepared_plan_cache_memory_guard_ratio value: '-0.1'"))
tk.MustQuery("select @@global.tidb_prepared_plan_cache_memory_guard_ratio").Check(testkit.Rows("0"))
tk.MustExec("SET GLOBAL tidb_prepared_plan_cache_memory_guard_ratio = 2.2")
tk.MustQuery("show warnings").Check(testkit.Rows(
"Warning 1292 Truncated incorrect tidb_prepared_plan_cache_memory_guard_ratio value: '2.2'"))
tk.MustQuery("select @@global.tidb_prepared_plan_cache_memory_guard_ratio").Check(testkit.Rows("1"))
tk.MustExec("SET GLOBAL tidb_prepared_plan_cache_memory_guard_ratio = 0.5")
tk.MustQuery("select @@global.tidb_prepared_plan_cache_memory_guard_ratio").Check(testkit.Rows("0.5"))
orgEnable := variable.EnablePreparedPlanCache.Load()
defer func() {
variable.EnablePreparedPlanCache.Store(orgEnable)
}()
tk.MustExec("SET GLOBAL tidb_enable_prepared_plan_cache = 0")
tk.MustQuery("select @@global.tidb_enable_prepared_plan_cache").Check(testkit.Rows("0"))
require.False(t, variable.EnablePreparedPlanCache.Load())
tk.MustExec("SET GLOBAL tidb_enable_prepared_plan_cache = 1")
tk.MustQuery("select @@global.tidb_enable_prepared_plan_cache").Check(testkit.Rows("1"))
require.True(t, variable.EnablePreparedPlanCache.Load())
tk.MustExec("SET GLOBAL tidb_enable_prepared_plan_cache = 0")
tk.MustQuery("select @@global.tidb_enable_prepared_plan_cache").Check(testkit.Rows("0"))
require.False(t, variable.EnablePreparedPlanCache.Load())
}
func TestInstanceScopeSwitching(t *testing.T) {
store, clean := testkit.CreateMockStore(t)
defer clean()

10
infoschema/tables_test.go
View File

@ -35,7 +35,6 @@ import (
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/session/txninfo"
"github.com/pingcap/tidb/sessionctx/variable"
@ -1362,12 +1361,9 @@ func TestStmtSummaryHistoryTableOther(t *testing.T) {
func TestPerformanceSchemaforPlanCache(t *testing.T) {
store, clean := testkit.CreateMockStore(t)
defer clean()
orgEnable := plannercore.PreparedPlanCacheEnabled()
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
}()
plannercore.SetPreparedPlanCache(true)
tmp := testkit.NewTestKit(t, store)
defer tmp.MustExec("set global tidb_enable_prepared_plan_cache=" + variable.BoolToOnOff(variable.EnablePreparedPlanCache.Load()))
tmp.MustExec("set global tidb_enable_prepared_plan_cache=ON")
tk := newTestKitWithPlanCache(t, store)

24
planner/core/cache.go
View File

@ -17,7 +17,6 @@ package core
import (
"bytes"
"math"
"sync/atomic"
"time"
"github.com/pingcap/errors"
@ -36,35 +35,20 @@ import (
)
var (
// preparedPlanCacheEnabledValue stores the global config "prepared-plan-cache-enabled".
// The value is false unless "prepared-plan-cache-enabled" is true in configuration.
preparedPlanCacheEnabledValue int32 = 0
// PreparedPlanCacheCapacity stores the global config "prepared-plan-cache-capacity".
PreparedPlanCacheCapacity uint = 1000
// PreparedPlanCacheMemoryGuardRatio stores the global config "prepared-plan-cache-memory-guard-ratio".
PreparedPlanCacheMemoryGuardRatio = 0.1
// PreparedPlanCacheMaxMemory stores the max memory size defined in the global config "performance-server-memory-quota".
PreparedPlanCacheMaxMemory = *atomic2.NewUint64(math.MaxUint64)
)
const (
preparedPlanCacheEnabled = 1
preparedPlanCacheUnable = 0
)
// SetPreparedPlanCache sets isEnabled to true, then prepared plan cache is enabled.
// FIXME: leave it for test, remove it after implementing session-level plan-cache variables.
func SetPreparedPlanCache(isEnabled bool) {
if isEnabled {
atomic.StoreInt32(&preparedPlanCacheEnabledValue, preparedPlanCacheEnabled)
} else {
atomic.StoreInt32(&preparedPlanCacheEnabledValue, preparedPlanCacheUnable)
}
variable.EnablePreparedPlanCache.Store(isEnabled) // only for test
}
// PreparedPlanCacheEnabled returns whether the prepared plan cache is enabled.
// FIXME: leave it for test, remove it after implementing session-level plan-cache variables.
func PreparedPlanCacheEnabled() bool {
isEnabled := atomic.LoadInt32(&preparedPlanCacheEnabledValue)
return isEnabled == preparedPlanCacheEnabled
return variable.EnablePreparedPlanCache.Load()
}
// planCacheKey is used to access Plan Cache. We put some variables that do not affect the plan into planCacheKey, such as the sql text.

19
session/bootstrap.go
View File

@ -617,11 +617,13 @@ const (
version89 = 89
// version90 converts enable-batch-dml, mem-quota-query, query-log-max-len, committer-concurrency, run-auto-analyze, and oom-action to a sysvar
version90 = 90
// version91 converts prepared-plan-cache to sysvars
version91 = 91
)
// currentBootstrapVersion is defined as a variable, so we can modify its value for testing.
// please make sure this is the largest version
var currentBootstrapVersion int64 = version90
var currentBootstrapVersion int64 = version91
var (
bootstrapVersion = []func(Session, int64){
@ -715,6 +717,7 @@ var (
upgradeToVer88,
upgradeToVer89,
upgradeToVer90,
upgradeToVer91,
}
)
@ -1858,6 +1861,20 @@ func upgradeToVer90(s Session, ver int64) {
importConfigOption(s, "oom-action", variable.TiDBMemOOMAction, valStr)
}
func upgradeToVer91(s Session, ver int64) {
if ver >= version91 {
return
}
valStr := variable.BoolToOnOff(config.GetGlobalConfig().PreparedPlanCache.Enabled)
importConfigOption(s, "prepared-plan-cache.enable", variable.TiDBEnablePrepPlanCache, valStr)
valStr = strconv.Itoa(int(config.GetGlobalConfig().PreparedPlanCache.Capacity))
importConfigOption(s, "prepared-plan-cache.capacity", variable.TiDBPrepPlanCacheSize, valStr)
valStr = strconv.FormatFloat(config.GetGlobalConfig().PreparedPlanCache.MemoryGuardRatio, 'f', -1, 64)
importConfigOption(s, "prepared-plan-cache.memory-guard-ratio", variable.TiDBPrepPlanCacheMemoryGuardRatio, valStr)
}
func writeOOMAction(s Session) {
comment := "oom-action is `log` by default in v3.0.x, `cancel` by default in v4.0.11+"
mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`,

8
session/session.go
View File

@ -2982,8 +2982,8 @@ func createSessionWithOpt(store kv.Storage, opt *Opt) (*session, error) {
if opt != nil && opt.PreparedPlanCache != nil {
s.preparedPlanCache = opt.PreparedPlanCache
} else {
s.preparedPlanCache = kvcache.NewSimpleLRUCache(plannercore.PreparedPlanCacheCapacity,
plannercore.PreparedPlanCacheMemoryGuardRatio, plannercore.PreparedPlanCacheMaxMemory.Load())
s.preparedPlanCache = kvcache.NewSimpleLRUCache(uint(variable.PreparedPlanCacheSize.Load()),
variable.PreparedPlanCacheMemoryGuardRatio.Load(), plannercore.PreparedPlanCacheMaxMemory.Load())
}
}
s.mu.values = make(map[fmt.Stringer]interface{})
@ -3015,8 +3015,8 @@ func CreateSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, er
}
s.functionUsageMu.builtinFunctionUsage = make(telemetry.BuiltinFunctionsUsage)
if plannercore.PreparedPlanCacheEnabled() {
s.preparedPlanCache = kvcache.NewSimpleLRUCache(plannercore.PreparedPlanCacheCapacity,
plannercore.PreparedPlanCacheMemoryGuardRatio, plannercore.PreparedPlanCacheMaxMemory.Load())
s.preparedPlanCache = kvcache.NewSimpleLRUCache(uint(variable.PreparedPlanCacheSize.Load()),
variable.PreparedPlanCacheMemoryGuardRatio.Load(), plannercore.PreparedPlanCacheMaxMemory.Load())
}
s.mu.values = make(map[fmt.Stringer]interface{})
s.lockedTables = make(map[int64]model.TableLockTpInfo)

24
sessionctx/variable/sysvar.go
View File

@ -754,6 +754,30 @@ var defaultSysVars = []*SysVar{
return nil
},
},
{Scope: ScopeGlobal, Name: TiDBEnablePrepPlanCache, Value: BoolToOnOff(DefTiDBEnablePrepPlanCache), Type: TypeBool, SetGlobal: func(s *SessionVars, val string) error {
EnablePreparedPlanCache.Store(TiDBOptOn(val))
return nil
}, GetGlobal: func(s *SessionVars) (string, error) {
return BoolToOnOff(EnablePreparedPlanCache.Load()), nil
}},
{Scope: ScopeGlobal, Name: TiDBPrepPlanCacheSize, Value: strconv.FormatUint(uint64(DefTiDBPrepPlanCacheSize), 10), Type: TypeUnsigned, MinValue: 1, MaxValue: 100000, SetGlobal: func(s *SessionVars, val string) error {
uVal, err := strconv.ParseUint(val, 10, 64)
if err == nil {
PreparedPlanCacheSize.Store(uVal)
}
return err
}, GetGlobal: func(s *SessionVars) (string, error) {
return strconv.FormatUint(PreparedPlanCacheSize.Load(), 10), nil
}},
{Scope: ScopeGlobal, Name: TiDBPrepPlanCacheMemoryGuardRatio, Value: strconv.FormatFloat(DefTiDBPrepPlanCacheMemoryGuardRatio, 'f', -1, 64), Type: TypeFloat, MinValue: 0.0, MaxValue: 1.0, SetGlobal: func(s *SessionVars, val string) error {
f, err := strconv.ParseFloat(val, 64)
if err == nil {
PreparedPlanCacheMemoryGuardRatio.Store(f)
}
return err
}, GetGlobal: func(s *SessionVars) (string, error) {
return strconv.FormatFloat(PreparedPlanCacheMemoryGuardRatio.Load(), 'f', -1, 64), nil
}},
{Scope: ScopeGlobal, Name: TiDBMemOOMAction, Value: DefTiDBMemOOMAction, PossibleValues: []string{"CANCEL", "LOG"}, Type: TypeEnum,
GetGlobal: func(s *SessionVars) (string, error) {
return OOMAction.Load(), nil

13
sessionctx/variable/tidb_vars.go
View File

@ -693,6 +693,12 @@ const (
//TiDBMemOOMAction indicates what operation TiDB perform when a single SQL statement exceeds
// the memory quota specified by tidb_mem_quota_query and cannot be spilled to disk.
TiDBMemOOMAction = "tidb_mem_oom_action"
// TiDBEnablePrepPlanCache indicates whether to enable prepared plan cache
TiDBEnablePrepPlanCache = "tidb_enable_prepared_plan_cache"
// TiDBPrepPlanCacheSize indicates the number of cached statements.
TiDBPrepPlanCacheSize = "tidb_prepared_plan_cache_size"
// TiDBPrepPlanCacheMemoryGuardRatio is used to prevent [performance.max-memory] from being exceeded
TiDBPrepPlanCacheMemoryGuardRatio = "tidb_prepared_plan_cache_memory_guard_ratio"
)
// TiDB intentional limits
@ -876,6 +882,9 @@ const (
DefTiDBMemQuotaAnalyze = -1
DefTiDBEnableAutoAnalyze = true
DefTiDBMemOOMAction = "CANCEL"
DefTiDBEnablePrepPlanCache = false
DefTiDBPrepPlanCacheSize = 1000
DefTiDBPrepPlanCacheMemoryGuardRatio = 0.1
)
// Process global variables.
@ -916,6 +925,10 @@ var (
GCMaxWaitTime = atomic.NewInt64(DefTiDBGCMaxWaitTime)
StatsCacheMemQuota = atomic.NewInt64(DefTiDBStatsCacheMemQuota)
OOMAction = atomic.NewString(DefTiDBMemOOMAction)
// variables for plan cache
EnablePreparedPlanCache = atomic.NewBool(DefTiDBEnablePrepPlanCache)
PreparedPlanCacheSize = atomic.NewUint64(DefTiDBPrepPlanCacheSize)
PreparedPlanCacheMemoryGuardRatio = atomic.NewFloat64(DefTiDBPrepPlanCacheMemoryGuardRatio)
)
var (

10
statistics/handle/update_test.go
View File

@ -25,7 +25,6 @@ import (
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/statistics"
@ -2308,14 +2307,11 @@ func TestDumpColumnStatsUsage(t *testing.T) {
func TestCollectPredicateColumnsFromExecute(t *testing.T) {
for _, val := range []bool{false, true} {
func(planCache bool) {
originalVal1 := plannercore.PreparedPlanCacheEnabled()
defer func() {
plannercore.SetPreparedPlanCache(originalVal1)
}()
plannercore.SetPreparedPlanCache(planCache)
store, dom, clean := testkit.CreateMockStoreAndDomain(t)
defer clean()
tmp := testkit.NewTestKit(t, store)
defer tmp.MustExec("set global tidb_enable_prepared_plan_cache=" + variable.BoolToOnOff(variable.EnablePreparedPlanCache.Load()))
tmp.MustExec("set global tidb_enable_prepared_plan_cache=" + variable.BoolToOnOff(planCache))
tk := testkit.NewTestKit(t, store)
originalVal2 := tk.MustQuery("select @@tidb_enable_column_tracking").Rows()[0][0].(string)

10
tests/realtikvtest/pessimistictest/pessimistic_test.go
View File

@ -34,7 +34,6 @@ import (
"github.com/pingcap/tidb/parser/model"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tidb/parser/terror"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx/variable"
storeerr "github.com/pingcap/tidb/store/driver/error"
@ -2761,14 +2760,11 @@ func TestIssue21498(t *testing.T) {
}
func TestPlanCacheSchemaChange(t *testing.T) {
orgEnable := plannercore.PreparedPlanCacheEnabled()
defer func() {
plannercore.SetPreparedPlanCache(orgEnable)
}()
plannercore.SetPreparedPlanCache(true)
store, clean := realtikvtest.CreateMockStoreAndSetup(t)
defer clean()
tmp := testkit.NewTestKit(t, store)
defer tmp.MustExec("set global tidb_enable_prepared_plan_cache=" + variable.BoolToOnOff(variable.EnablePreparedPlanCache.Load()))
tmp.MustExec("set global tidb_enable_prepared_plan_cache=ON")
tk := testkit.NewTestKit(t, store)
tk2 := testkit.NewTestKit(t, store)

23
tidb-server/main.go
View File

@ -661,19 +661,16 @@ func setGlobalVars() {
}
// For CI environment we default enable prepare-plan-cache.
plannercore.SetPreparedPlanCache(config.CheckTableBeforeDrop || cfg.PreparedPlanCache.Enabled)
if plannercore.PreparedPlanCacheEnabled() {
plannercore.PreparedPlanCacheCapacity = cfg.PreparedPlanCache.Capacity
plannercore.PreparedPlanCacheMemoryGuardRatio = cfg.PreparedPlanCache.MemoryGuardRatio
if plannercore.PreparedPlanCacheMemoryGuardRatio < 0.0 || plannercore.PreparedPlanCacheMemoryGuardRatio > 1.0 {
plannercore.PreparedPlanCacheMemoryGuardRatio = 0.1
}
plannercore.PreparedPlanCacheMaxMemory.Store(cfg.Performance.ServerMemoryQuota)
total, err := memory.MemTotal()
terror.MustNil(err)
if plannercore.PreparedPlanCacheMaxMemory.Load() > total || plannercore.PreparedPlanCacheMaxMemory.Load() <= 0 {
plannercore.PreparedPlanCacheMaxMemory.Store(total)
}
if config.CheckTableBeforeDrop { // only for test
plannercore.SetPreparedPlanCache(true)
}
// use server-memory-quota as max-plan-cache-memory
plannercore.PreparedPlanCacheMaxMemory.Store(cfg.Performance.ServerMemoryQuota)
total, err := memory.MemTotal()
terror.MustNil(err)
// if server-memory-quota is larger than max-system-memory or not set, use max-system-memory as max-plan-cache-memory
if plannercore.PreparedPlanCacheMaxMemory.Load() > total || plannercore.PreparedPlanCacheMaxMemory.Load() <= 0 {
plannercore.PreparedPlanCacheMaxMemory.Store(total)
}
atomic.StoreUint64(&transaction.CommitMaxBackoff, uint64(parseDuration(cfg.TiKVClient.CommitTimeout).Seconds()*1000))