From ee7d367b4361af2cb142f247ffb0e73b9f77fde2 Mon Sep 17 00:00:00 2001 From: Lynn Date: Wed, 20 Nov 2019 11:14:13 +0800 Subject: [PATCH] ddl: add a binlog when dropping column (#13536) --- ddl/ddl_worker.go | 15 ++++++++++++--- go.mod | 2 +- go.sum | 5 +++-- sessionctx/binloginfo/binloginfo.go | 9 +++++---- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go index 4c4ad79a7e..2cfd37b2c4 100644 --- a/ddl/ddl_worker.go +++ b/ddl/ddl_worker.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" "github.com/pingcap/parser/terror" + pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client" "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" @@ -426,9 +427,7 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error { if err = w.handleUpdateJobError(t, job, err); err != nil { return errors.Trace(err) } - if job.IsDone() || job.IsRollbackDone() { - binloginfo.SetDDLBinlog(d.binlogCli, txn, job.ID, job.Query) - } + writeBinlog(d.binlogCli, txn, job) return nil }) @@ -462,6 +461,16 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error { } } +func writeBinlog(binlogCli *pumpcli.PumpsClient, txn kv.Transaction, job *model.Job) { + if job.IsDone() || job.IsRollbackDone() || + // When this column is in the "delete only" and "delete reorg" states, the binlog of "drop column" has not been written yet, + // but the column has been removed from the binlog of the write operation. + // So we add this binlog to enable downstream components to handle DML correctly in this schema state. + (job.Type == model.ActionDropColumn && job.SchemaState == model.StateDeleteOnly) { + binloginfo.SetDDLBinlog(binlogCli, txn, job.ID, int32(job.SchemaState), job.Query) + } +} + // waitDependencyJobFinished waits for the dependency-job to be finished. // If the dependency job isn't finished yet, we'd better wait a moment. func (w *worker) waitDependencyJobFinished(job *model.Job, cnt *int) { diff --git a/go.mod b/go.mod index ac464b952e..edfa824c20 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/parser v0.0.0-20191112053614-3b43b46331d5 github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible - github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e + github.com/pingcap/tipb v0.0.0-20191120020146-6161b015e21e github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 // indirect diff --git a/go.sum b/go.sum index c985ea5210..a32d47088e 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mo github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -191,8 +192,8 @@ github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bc github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e h1:TWwzCfLrj9GH5uaT0VcvdSnrHuwEntUfoHDTYdOzNNI= -github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20191120020146-6161b015e21e h1:OWgXsJ2Zwa+q+sqi87fGuNda+ChJrclVd7wiGP5Epps= +github.com/pingcap/tipb v0.0.0-20191120020146-6161b015e21e/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/sessionctx/binloginfo/binloginfo.go b/sessionctx/binloginfo/binloginfo.go index 26de12c950..e20c0822bd 100644 --- a/sessionctx/binloginfo/binloginfo.go +++ b/sessionctx/binloginfo/binloginfo.go @@ -191,7 +191,7 @@ func (info *BinlogInfo) WriteBinlog(clusterID uint64) error { } // SetDDLBinlog sets DDL binlog in the kv.Transaction. -func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery string) { +func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlSchemaState int32, ddlQuery string) { if client == nil { return } @@ -199,9 +199,10 @@ func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery = AddSpecialComment(ddlQuery) info := &BinlogInfo{ Data: &binlog.Binlog{ - Tp: binlog.BinlogType_Prewrite, - DdlJobId: jobID, - DdlQuery: []byte(ddlQuery), + Tp: binlog.BinlogType_Prewrite, + DdlJobId: jobID, + DdlSchemaState: ddlSchemaState, + DdlQuery: []byte(ddlQuery), }, Client: client, }