database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

domain: change groupSize in splitForConcurrentFetch (#55518)

Browse Source 
ref pingcap/tidb#50959
This commit is contained in:
Ruihao Chen
2024-08-20 17:13:49 +08:00
committed by GitHub
parent 6d95459631
commit 85235b3672
2 changed files with 16 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/domain/BUILD.bazel
View File

@ -84,6 +84,7 @@ go_library(
"//pkg/util/globalconn",
"//pkg/util/intest",
"//pkg/util/logutil",
"//pkg/util/mathutil",
"//pkg/util/memory",
"//pkg/util/memoryusagealarm",
"//pkg/util/printer",

27
pkg/domain/domain.go
View File

@ -87,6 +87,7 @@ import (
"github.com/pingcap/tidb/pkg/util/globalconn"
"github.com/pingcap/tidb/pkg/util/intest"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/mathutil"
"github.com/pingcap/tidb/pkg/util/memory"
"github.com/pingcap/tidb/pkg/util/memoryusagealarm"
"github.com/pingcap/tidb/pkg/util/replayer"
@ -462,20 +463,22 @@ func (do *Domain) fetchAllSchemasWithTables(m *meta.Meta) ([]*model.DBInfo, erro
const fetchSchemaConcurrency = 1
func (*Domain) splitForConcurrentFetch(schemas []*model.DBInfo) [][]*model.DBInfo {
groupSize := (len(schemas) + fetchSchemaConcurrency - 1) / fetchSchemaConcurrency
if variable.SchemaCacheSize.Load() > 0 && len(schemas) > 1000 {
// TODO: Temporary solution to speed up when too many databases, will refactor it later.
groupSize = 8
}
splitted := make([][]*model.DBInfo, 0, fetchSchemaConcurrency)
groupCnt := fetchSchemaConcurrency
schemaCnt := len(schemas)
for i := 0; i < schemaCnt; i += groupSize {
end := i + groupSize
if end > schemaCnt {
end = schemaCnt
}
splitted = append(splitted, schemas[i:end])
if variable.SchemaCacheSize.Load() > 0 && schemaCnt > 1000 {
// TODO: Temporary solution to speed up when too many databases, will refactor it later.
groupCnt = 8
}
splitted := make([][]*model.DBInfo, 0, groupCnt)
groupSizes := mathutil.Divide2Batches(schemaCnt, groupCnt)
start := 0
for _, groupSize := range groupSizes {
splitted = append(splitted, schemas[start:start+groupSize])
start += groupSize
}
return splitted
}