434 lines
15 KiB
Go
434 lines
15 KiB
Go
// Copyright 2019 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package ddl_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
. "github.com/pingcap/check"
|
|
"github.com/pingcap/errors"
|
|
"github.com/pingcap/failpoint"
|
|
"github.com/pingcap/parser/model"
|
|
"github.com/pingcap/parser/mysql"
|
|
"github.com/pingcap/tidb/config"
|
|
"github.com/pingcap/tidb/ddl"
|
|
"github.com/pingcap/tidb/domain"
|
|
"github.com/pingcap/tidb/infoschema"
|
|
"github.com/pingcap/tidb/kv"
|
|
"github.com/pingcap/tidb/meta"
|
|
"github.com/pingcap/tidb/session"
|
|
"github.com/pingcap/tidb/store/mockstore"
|
|
"github.com/pingcap/tidb/util/admin"
|
|
"github.com/pingcap/tidb/util/gcutil"
|
|
"github.com/pingcap/tidb/util/mock"
|
|
"github.com/pingcap/tidb/util/testkit"
|
|
)
|
|
|
|
var _ = SerialSuites(&testSerialSuite{})
|
|
|
|
type testSerialSuite struct {
|
|
store kv.Storage
|
|
dom *domain.Domain
|
|
}
|
|
|
|
func (s *testSerialSuite) SetUpSuite(c *C) {
|
|
session.SetSchemaLease(200 * time.Millisecond)
|
|
session.DisableStats4Test()
|
|
|
|
ddl.WaitTimeWhenErrorOccured = 1 * time.Microsecond
|
|
var err error
|
|
s.store, err = mockstore.NewMockTikvStore()
|
|
c.Assert(err, IsNil)
|
|
|
|
s.dom, err = session.BootstrapSession(s.store)
|
|
c.Assert(err, IsNil)
|
|
}
|
|
|
|
func (s *testSerialSuite) TearDownSuite(c *C) {
|
|
if s.dom != nil {
|
|
s.dom.Close()
|
|
}
|
|
if s.store != nil {
|
|
s.store.Close()
|
|
}
|
|
}
|
|
|
|
// TestCancelAddIndex1 tests canceling ddl job when the add index worker is not started.
|
|
func (s *testSerialSuite) TestCancelAddIndexPanic(c *C) {
|
|
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/errorMockPanic", `return(true)`), IsNil)
|
|
defer func() {
|
|
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/errorMockPanic"), IsNil)
|
|
}()
|
|
tk := testkit.NewTestKit(c, s.store)
|
|
tk.MustExec("use test")
|
|
tk.MustExec("drop table if exists t")
|
|
tk.MustExec("create table t(c1 int, c2 int)")
|
|
defer tk.MustExec("drop table t;")
|
|
for i := 0; i < 5; i++ {
|
|
tk.MustExec("insert into t values (?, ?)", i, i)
|
|
}
|
|
var checkErr error
|
|
oldReorgWaitTimeout := ddl.ReorgWaitTimeout
|
|
ddl.ReorgWaitTimeout = 50 * time.Millisecond
|
|
defer func() { ddl.ReorgWaitTimeout = oldReorgWaitTimeout }()
|
|
hook := &ddl.TestDDLCallback{}
|
|
hook.OnJobRunBeforeExported = func(job *model.Job) {
|
|
if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning && job.SchemaState == model.StateWriteReorganization && job.SnapshotVer != 0 {
|
|
jobIDs := []int64{job.ID}
|
|
hookCtx := mock.NewContext()
|
|
hookCtx.Store = s.store
|
|
err := hookCtx.NewTxn(context.Background())
|
|
if err != nil {
|
|
checkErr = errors.Trace(err)
|
|
return
|
|
}
|
|
txn, err := hookCtx.Txn(true)
|
|
if err != nil {
|
|
checkErr = errors.Trace(err)
|
|
return
|
|
}
|
|
errs, err := admin.CancelJobs(txn, jobIDs)
|
|
if err != nil {
|
|
checkErr = errors.Trace(err)
|
|
return
|
|
}
|
|
if errs[0] != nil {
|
|
checkErr = errors.Trace(errs[0])
|
|
return
|
|
}
|
|
txn, err = hookCtx.Txn(true)
|
|
if err != nil {
|
|
checkErr = errors.Trace(err)
|
|
return
|
|
}
|
|
checkErr = txn.Commit(context.Background())
|
|
}
|
|
}
|
|
origHook := s.dom.DDL().GetHook()
|
|
defer s.dom.DDL().(ddl.DDLForTest).SetHook(origHook)
|
|
s.dom.DDL().(ddl.DDLForTest).SetHook(hook)
|
|
rs, err := tk.Exec("alter table t add index idx_c2(c2)")
|
|
if rs != nil {
|
|
rs.Close()
|
|
}
|
|
c.Assert(checkErr, IsNil)
|
|
c.Assert(err, NotNil)
|
|
c.Assert(err.Error(), Equals, "[ddl:12]cancelled DDL job")
|
|
}
|
|
|
|
func (s *testSerialSuite) TestRecoverTableByJobID(c *C) {
|
|
tk := testkit.NewTestKit(c, s.store)
|
|
tk.MustExec("create database if not exists test_recover")
|
|
tk.MustExec("use test_recover")
|
|
tk.MustExec("drop table if exists t_recover")
|
|
tk.MustExec("create table t_recover (a int);")
|
|
defer func(originGC bool) {
|
|
if originGC {
|
|
ddl.EmulatorGCEnable()
|
|
} else {
|
|
ddl.EmulatorGCDisable()
|
|
}
|
|
}(ddl.IsEmulatorGCEnable())
|
|
|
|
// disable emulator GC.
|
|
// Otherwise emulator GC will delete table record as soon as possible after execute drop table ddl.
|
|
ddl.EmulatorGCDisable()
|
|
gcTimeFormat := "20060102-15:04:05 -0700 MST"
|
|
timeBeforeDrop := time.Now().Add(0 - time.Duration(48*60*60*time.Second)).Format(gcTimeFormat)
|
|
timeAfterDrop := time.Now().Add(time.Duration(48 * 60 * 60 * time.Second)).Format(gcTimeFormat)
|
|
safePointSQL := `INSERT HIGH_PRIORITY INTO mysql.tidb VALUES ('tikv_gc_safe_point', '%[1]s', '')
|
|
ON DUPLICATE KEY
|
|
UPDATE variable_value = '%[1]s'`
|
|
// clear GC variables first.
|
|
tk.MustExec("delete from mysql.tidb where variable_name in ( 'tikv_gc_safe_point','tikv_gc_enable' )")
|
|
|
|
tk.MustExec("insert into t_recover values (1),(2),(3)")
|
|
tk.MustExec("drop table t_recover")
|
|
|
|
rs, err := tk.Exec("admin show ddl jobs")
|
|
c.Assert(err, IsNil)
|
|
rows, err := session.GetRows4Test(context.Background(), tk.Se, rs)
|
|
c.Assert(err, IsNil)
|
|
row := rows[0]
|
|
c.Assert(row.GetString(1), Equals, "test_recover")
|
|
c.Assert(row.GetString(3), Equals, "drop table")
|
|
jobID := row.GetInt64(0)
|
|
|
|
// if GC safe point is not exists in mysql.tidb
|
|
_, err = tk.Exec(fmt.Sprintf("recover table by job %d", jobID))
|
|
c.Assert(err, NotNil)
|
|
c.Assert(err.Error(), Equals, "can not get 'tikv_gc_safe_point'")
|
|
// set GC safe point
|
|
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
|
|
|
|
// if GC enable is not exists in mysql.tidb
|
|
_, err = tk.Exec(fmt.Sprintf("recover table by job %d", jobID))
|
|
c.Assert(err, NotNil)
|
|
c.Assert(err.Error(), Equals, "[ddl:-1]can not get 'tikv_gc_enable'")
|
|
|
|
err = gcutil.EnableGC(tk.Se)
|
|
c.Assert(err, IsNil)
|
|
|
|
// recover job is before GC safe point
|
|
tk.MustExec(fmt.Sprintf(safePointSQL, timeAfterDrop))
|
|
_, err = tk.Exec(fmt.Sprintf("recover table by job %d", jobID))
|
|
c.Assert(err, NotNil)
|
|
c.Assert(strings.Contains(err.Error(), "snapshot is older than GC safe point"), Equals, true)
|
|
|
|
// set GC safe point
|
|
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
|
|
// if there is a new table with the same name, should return failed.
|
|
tk.MustExec("create table t_recover (a int);")
|
|
_, err = tk.Exec(fmt.Sprintf("recover table by job %d", jobID))
|
|
c.Assert(err.Error(), Equals, infoschema.ErrTableExists.GenWithStackByArgs("t_recover").Error())
|
|
|
|
// drop the new table with the same name, then recover table.
|
|
tk.MustExec("drop table t_recover")
|
|
|
|
// do recover table.
|
|
tk.MustExec(fmt.Sprintf("recover table by job %d", jobID))
|
|
|
|
// check recover table meta and data record.
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1", "2", "3"))
|
|
// check recover table autoID.
|
|
tk.MustExec("insert into t_recover values (4),(5),(6)")
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1", "2", "3", "4", "5", "6"))
|
|
|
|
// recover table by none exits job.
|
|
_, err = tk.Exec(fmt.Sprintf("recover table by job %d", 10000000))
|
|
c.Assert(err, NotNil)
|
|
|
|
// Disable GC by manual first, then after recover table, the GC enable status should also be disabled.
|
|
err = gcutil.DisableGC(tk.Se)
|
|
c.Assert(err, IsNil)
|
|
|
|
tk.MustExec("delete from t_recover where a > 1")
|
|
tk.MustExec("drop table t_recover")
|
|
rs, err = tk.Exec("admin show ddl jobs")
|
|
c.Assert(err, IsNil)
|
|
rows, err = session.GetRows4Test(context.Background(), tk.Se, rs)
|
|
c.Assert(err, IsNil)
|
|
row = rows[0]
|
|
c.Assert(row.GetString(1), Equals, "test_recover")
|
|
c.Assert(row.GetString(3), Equals, "drop table")
|
|
jobID = row.GetInt64(0)
|
|
|
|
tk.MustExec(fmt.Sprintf("recover table by job %d", jobID))
|
|
|
|
// check recover table meta and data record.
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1"))
|
|
// check recover table autoID.
|
|
tk.MustExec("insert into t_recover values (7),(8),(9)")
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1", "7", "8", "9"))
|
|
|
|
gcEnable, err := gcutil.CheckGCEnable(tk.Se)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(gcEnable, Equals, false)
|
|
}
|
|
|
|
func (s *testSerialSuite) TestRecoverTableByJobIDFail(c *C) {
|
|
tk := testkit.NewTestKit(c, s.store)
|
|
tk.MustExec("create database if not exists test_recover")
|
|
tk.MustExec("use test_recover")
|
|
tk.MustExec("drop table if exists t_recover")
|
|
tk.MustExec("create table t_recover (a int);")
|
|
defer func(originGC bool) {
|
|
if originGC {
|
|
ddl.EmulatorGCEnable()
|
|
} else {
|
|
ddl.EmulatorGCDisable()
|
|
}
|
|
}(ddl.IsEmulatorGCEnable())
|
|
|
|
// disable emulator GC.
|
|
// Otherwise emulator GC will delete table record as soon as possible after execute drop table ddl.
|
|
ddl.EmulatorGCDisable()
|
|
gcTimeFormat := "20060102-15:04:05 -0700 MST"
|
|
timeBeforeDrop := time.Now().Add(0 - time.Duration(48*60*60*time.Second)).Format(gcTimeFormat)
|
|
safePointSQL := `INSERT HIGH_PRIORITY INTO mysql.tidb VALUES ('tikv_gc_safe_point', '%[1]s', '')
|
|
ON DUPLICATE KEY
|
|
UPDATE variable_value = '%[1]s'`
|
|
|
|
tk.MustExec("insert into t_recover values (1),(2),(3)")
|
|
tk.MustExec("drop table t_recover")
|
|
|
|
rs, err := tk.Exec("admin show ddl jobs")
|
|
c.Assert(err, IsNil)
|
|
rows, err := session.GetRows4Test(context.Background(), tk.Se, rs)
|
|
c.Assert(err, IsNil)
|
|
row := rows[0]
|
|
c.Assert(row.GetString(1), Equals, "test_recover")
|
|
c.Assert(row.GetString(3), Equals, "drop table")
|
|
jobID := row.GetInt64(0)
|
|
|
|
// enableGC first
|
|
err = gcutil.EnableGC(tk.Se)
|
|
c.Assert(err, IsNil)
|
|
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
|
|
|
|
// set hook
|
|
hook := &ddl.TestDDLCallback{}
|
|
hook.OnJobRunBeforeExported = func(job *model.Job) {
|
|
if job.Type == model.ActionRecoverTable {
|
|
c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockCommitError", `return(true)`), IsNil)
|
|
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/mockRecoverTableCommitErr", `return(true)`), IsNil)
|
|
}
|
|
}
|
|
origHook := s.dom.DDL().GetHook()
|
|
defer s.dom.DDL().(ddl.DDLForTest).SetHook(origHook)
|
|
s.dom.DDL().(ddl.DDLForTest).SetHook(hook)
|
|
|
|
// do recover table.
|
|
tk.MustExec(fmt.Sprintf("recover table by job %d", jobID))
|
|
c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/mockCommitError"), IsNil)
|
|
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/mockRecoverTableCommitErr"), IsNil)
|
|
|
|
// make sure enable GC after recover table.
|
|
enable, err := gcutil.CheckGCEnable(tk.Se)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(enable, Equals, true)
|
|
|
|
// check recover table meta and data record.
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1", "2", "3"))
|
|
// check recover table autoID.
|
|
tk.MustExec("insert into t_recover values (4),(5),(6)")
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1", "2", "3", "4", "5", "6"))
|
|
}
|
|
|
|
func (s *testSerialSuite) TestRecoverTableByTableNameFail(c *C) {
|
|
tk := testkit.NewTestKit(c, s.store)
|
|
tk.MustExec("create database if not exists test_recover")
|
|
tk.MustExec("use test_recover")
|
|
tk.MustExec("drop table if exists t_recover")
|
|
tk.MustExec("create table t_recover (a int);")
|
|
defer func(originGC bool) {
|
|
if originGC {
|
|
ddl.EmulatorGCEnable()
|
|
} else {
|
|
ddl.EmulatorGCDisable()
|
|
}
|
|
}(ddl.IsEmulatorGCEnable())
|
|
|
|
// disable emulator GC.
|
|
// Otherwise emulator GC will delete table record as soon as possible after execute drop table ddl.
|
|
ddl.EmulatorGCDisable()
|
|
gcTimeFormat := "20060102-15:04:05 -0700 MST"
|
|
timeBeforeDrop := time.Now().Add(0 - time.Duration(48*60*60*time.Second)).Format(gcTimeFormat)
|
|
safePointSQL := `INSERT HIGH_PRIORITY INTO mysql.tidb VALUES ('tikv_gc_safe_point', '%[1]s', '')
|
|
ON DUPLICATE KEY
|
|
UPDATE variable_value = '%[1]s'`
|
|
|
|
tk.MustExec("insert into t_recover values (1),(2),(3)")
|
|
tk.MustExec("drop table t_recover")
|
|
|
|
// enableGC first
|
|
err := gcutil.EnableGC(tk.Se)
|
|
c.Assert(err, IsNil)
|
|
tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop))
|
|
|
|
// set hook
|
|
hook := &ddl.TestDDLCallback{}
|
|
hook.OnJobRunBeforeExported = func(job *model.Job) {
|
|
if job.Type == model.ActionRecoverTable {
|
|
c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockCommitError", `return(true)`), IsNil)
|
|
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/mockRecoverTableCommitErr", `return(true)`), IsNil)
|
|
}
|
|
}
|
|
origHook := s.dom.DDL().GetHook()
|
|
defer s.dom.DDL().(ddl.DDLForTest).SetHook(origHook)
|
|
s.dom.DDL().(ddl.DDLForTest).SetHook(hook)
|
|
|
|
// do recover table.
|
|
tk.MustExec("recover table t_recover")
|
|
c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/mockCommitError"), IsNil)
|
|
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/mockRecoverTableCommitErr"), IsNil)
|
|
|
|
// make sure enable GC after recover table.
|
|
enable, err := gcutil.CheckGCEnable(tk.Se)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(enable, Equals, true)
|
|
|
|
// check recover table meta and data record.
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1", "2", "3"))
|
|
// check recover table autoID.
|
|
tk.MustExec("insert into t_recover values (4),(5),(6)")
|
|
tk.MustQuery("select * from t_recover;").Check(testkit.Rows("1", "2", "3", "4", "5", "6"))
|
|
}
|
|
|
|
func (s *testSerialSuite) TestCancelJobByErrorCountLimit(c *C) {
|
|
tk := testkit.NewTestKit(c, s.store)
|
|
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/mockExceedErrorLimit", `return(true)`), IsNil)
|
|
defer func() {
|
|
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/mockExceedErrorLimit"), IsNil)
|
|
}()
|
|
tk.MustExec("use test")
|
|
tk.MustExec("drop table if exists t")
|
|
_, err := tk.Exec("create table t (a int)")
|
|
c.Assert(err, NotNil)
|
|
c.Assert(err.Error(), Equals, "[ddl:12]cancelled DDL job")
|
|
}
|
|
|
|
func (s *testSerialSuite) TestCanceledJobTakeTime(c *C) {
|
|
tk := testkit.NewTestKit(c, s.store)
|
|
tk.MustExec("use test")
|
|
tk.MustExec("create table t_cjtt(a int)")
|
|
|
|
hook := &ddl.TestDDLCallback{}
|
|
once := sync.Once{}
|
|
hook.OnJobUpdatedExported = func(job *model.Job) {
|
|
once.Do(func() {
|
|
err := kv.RunInNewTxn(s.store, false, func(txn kv.Transaction) error {
|
|
t := meta.NewMeta(txn)
|
|
return t.DropTableOrView(job.SchemaID, job.TableID, true)
|
|
})
|
|
c.Assert(err, IsNil)
|
|
})
|
|
}
|
|
origHook := s.dom.DDL().GetHook()
|
|
s.dom.DDL().(ddl.DDLForTest).SetHook(hook)
|
|
defer s.dom.DDL().(ddl.DDLForTest).SetHook(origHook)
|
|
|
|
originalWT := ddl.WaitTimeWhenErrorOccured
|
|
ddl.WaitTimeWhenErrorOccured = 1 * time.Second
|
|
defer func() { ddl.WaitTimeWhenErrorOccured = originalWT }()
|
|
startTime := time.Now()
|
|
assertErrorCode(c, tk, "alter table t_cjtt add column b int", mysql.ErrNoSuchTable)
|
|
sub := time.Since(startTime)
|
|
c.Assert(sub, Less, ddl.WaitTimeWhenErrorOccured)
|
|
}
|
|
|
|
func (s *testSerialSuite) TestTableLocksEnable(c *C) {
|
|
tk := testkit.NewTestKit(c, s.store)
|
|
tk.MustExec("use test")
|
|
tk.MustExec("drop table if exists t1")
|
|
defer tk.MustExec("drop table if exists t1")
|
|
tk.MustExec("create table t1 (a int)")
|
|
|
|
// Test for enable table lock config.
|
|
cfg := config.GetGlobalConfig()
|
|
newCfg := *cfg
|
|
newCfg.EnableTableLock = false
|
|
config.StoreGlobalConfig(&newCfg)
|
|
defer func() {
|
|
config.StoreGlobalConfig(cfg)
|
|
}()
|
|
|
|
tk.MustExec("lock tables t1 write")
|
|
checkTableLock(c, tk.Se, "test", "t1", model.TableLockNone)
|
|
}
|