* 1. init ddl tables create tidb_ddl_job, tidb_ddl_reorg, tidb_ddl_history tables with raw meta write, these 3 tables is use to replace the ddl job queue and reorg and history hash table * 2. setup concurrent ddl env and add ddl worker pool adds the ddl worker pool definition, the ddl job manager will find a job and ship it to a worker in the worker pool. Also, this commit provides a sessionctx wrapper, only use in ddl relate. it just wraps begin, commit and execute * 3. add ddl manager to handle ddl job * 4. reorg handler for concurrent ddl just implements the partner of the reorg information. * 5. manage ddl jobs for concurrent ddl add the partner of add job, delete job and many others related to history job because many of the functions need a session now, we just change the caller * 6. add metrics for concurrent ddl add metrics * 7. support multiple tables * 8. fix test * 9. migrate ddl between table and queue support switch between the old and new ddl framework, migrate the existing ddl job between queue and table * 10. check tikv version and set reorg worker count according cpu count * *: add featuretag on tests Signed-off-by: Weizhen Wang <wangweizhen@pingcap.com> * use a determined table id for 3 tables * remove ctx value * add GetSchemaVersionWithNonEmptyDiff function * address tangenta and zimulala comment * use only one etcd path * make ActionRenameTable support multi-schema * reset sql digest to make top sql work correct * add comment * fix test * remove 0 for schema version lock Co-authored-by: xiongjiwei <xiongjiwei1996@outlook.com> Co-authored-by: Ti Chi Robot <ti-community-prow-bot@tidb.io> Co-authored-by: wjHuang <huangwenjun1997@gmail.com>
148 lines
4.2 KiB
Go
148 lines
4.2 KiB
Go
// Copyright 2015 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package ddl
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/pingcap/tidb/infoschema"
|
|
"github.com/pingcap/tidb/parser/model"
|
|
"github.com/pingcap/tidb/sessionctx"
|
|
"github.com/pingcap/tidb/util/logutil"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type TestInterceptor struct {
|
|
*BaseInterceptor
|
|
|
|
OnGetInfoSchemaExported func(ctx sessionctx.Context, is infoschema.InfoSchema) infoschema.InfoSchema
|
|
}
|
|
|
|
func (ti *TestInterceptor) OnGetInfoSchema(ctx sessionctx.Context, is infoschema.InfoSchema) infoschema.InfoSchema {
|
|
if ti.OnGetInfoSchemaExported != nil {
|
|
return ti.OnGetInfoSchemaExported(ctx, is)
|
|
}
|
|
|
|
return ti.BaseInterceptor.OnGetInfoSchema(ctx, is)
|
|
}
|
|
|
|
// TestDDLCallback is used to customize user callback themselves.
|
|
type TestDDLCallback struct {
|
|
*BaseCallback
|
|
// We recommended to pass the domain parameter to the test ddl callback, it will ensure
|
|
// domain to reload schema before your ddl stepping into the next state change.
|
|
Do DomainReloader
|
|
|
|
onJobRunBefore func(*model.Job)
|
|
OnJobRunBeforeExported func(*model.Job)
|
|
onJobUpdated func(*model.Job)
|
|
OnJobUpdatedExported func(*model.Job)
|
|
onWatched func(ctx context.Context)
|
|
OnGetJobBeforeExported func(string)
|
|
OnGetJobAfterExported func(string, *model.Job)
|
|
}
|
|
|
|
// OnChanged mock the same behavior with the main DDL hook.
|
|
func (tc *TestDDLCallback) OnChanged(err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
logutil.BgLogger().Info("performing DDL change, must reload")
|
|
if tc.Do != nil {
|
|
err = tc.Do.Reload()
|
|
if err != nil {
|
|
logutil.BgLogger().Error("performing DDL change failed", zap.Error(err))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// OnSchemaStateChanged mock the same behavior with the main ddl hook.
|
|
func (tc *TestDDLCallback) OnSchemaStateChanged() {
|
|
if tc.Do != nil {
|
|
if err := tc.Do.Reload(); err != nil {
|
|
logutil.BgLogger().Warn("reload failed on schema state changed", zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// OnJobRunBefore is used to run the user customized logic of `onJobRunBefore` first.
|
|
func (tc *TestDDLCallback) OnJobRunBefore(job *model.Job) {
|
|
logutil.BgLogger().Info("on job run before", zap.String("job", job.String()))
|
|
if tc.OnJobRunBeforeExported != nil {
|
|
tc.OnJobRunBeforeExported(job)
|
|
return
|
|
}
|
|
if tc.onJobRunBefore != nil {
|
|
tc.onJobRunBefore(job)
|
|
return
|
|
}
|
|
|
|
tc.BaseCallback.OnJobRunBefore(job)
|
|
}
|
|
|
|
// OnJobUpdated is used to run the user customized logic of `OnJobUpdated` first.
|
|
func (tc *TestDDLCallback) OnJobUpdated(job *model.Job) {
|
|
logutil.BgLogger().Info("on job updated", zap.String("job", job.String()))
|
|
if tc.OnJobUpdatedExported != nil {
|
|
tc.OnJobUpdatedExported(job)
|
|
return
|
|
}
|
|
if tc.onJobUpdated != nil {
|
|
tc.onJobUpdated(job)
|
|
return
|
|
}
|
|
|
|
tc.BaseCallback.OnJobUpdated(job)
|
|
}
|
|
|
|
// OnWatched is used to run the user customized logic of `OnWatched` first.
|
|
func (tc *TestDDLCallback) OnWatched(ctx context.Context) {
|
|
if tc.onWatched != nil {
|
|
tc.onWatched(ctx)
|
|
return
|
|
}
|
|
|
|
tc.BaseCallback.OnWatched(ctx)
|
|
}
|
|
|
|
// OnGetJobBefore implements Callback.OnGetJobBefore interface.
|
|
func (tc *TestDDLCallback) OnGetJobBefore(jobType string) {
|
|
if tc.OnGetJobBeforeExported != nil {
|
|
tc.OnGetJobBeforeExported(jobType)
|
|
return
|
|
}
|
|
tc.BaseCallback.OnGetJobBefore(jobType)
|
|
}
|
|
|
|
// OnGetJobAfter implements Callback.OnGetJobAfter interface.
|
|
func (tc *TestDDLCallback) OnGetJobAfter(jobType string, job *model.Job) {
|
|
if tc.OnGetJobAfterExported != nil {
|
|
tc.OnGetJobAfterExported(jobType, job)
|
|
return
|
|
}
|
|
tc.BaseCallback.OnGetJobAfter(jobType, job)
|
|
}
|
|
|
|
func TestCallback(t *testing.T) {
|
|
cb := &BaseCallback{}
|
|
require.Nil(t, cb.OnChanged(nil))
|
|
cb.OnJobRunBefore(nil)
|
|
cb.OnJobUpdated(nil)
|
|
cb.OnWatched(context.TODO())
|
|
}
|