// Copyright 2020 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cdclog import ( "time" "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/tidb/br/pkg/kv" "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" "go.uber.org/zap" ) // TableBuffer represents the kv buffer of this table. // we restore one tableBuffer in one goroutine. // this is the concurrent unit of log restore. type TableBuffer struct { KvPairs []kv.Row count int size int64 KvEncoder kv.Encoder tableInfo table.Table allocator autoid.Allocators flushKVSize int64 flushKVPairs int colNames []string colPerm []int } func newKVEncoder(allocators autoid.Allocators, tbl table.Table) (kv.Encoder, error) { encTable, err := table.TableFromMeta(allocators, tbl.Meta()) if err != nil { return nil, errors.Trace(err) } return kv.NewTableKVEncoder(encTable, &kv.SessionOptions{ Timestamp: time.Now().Unix(), // TODO get the version from TiDB cluster // currently TiDB only support v1 and v2, and since 4.0 // the default RowFormatVersion is 2, so I think // we can implement the row version retrieve from cluster in the future // when TiDB decide to support v3 RowFormatVersion. RowFormatVersion: "2", }), nil } // NewTableBuffer creates TableBuffer. func NewTableBuffer(tbl table.Table, allocators autoid.Allocators, flushKVPairs int, flushKVSize int64) *TableBuffer { tb := &TableBuffer{ KvPairs: make([]kv.Row, 0, flushKVPairs), flushKVPairs: flushKVPairs, flushKVSize: flushKVSize, } if tbl != nil { tb.ReloadMeta(tbl, allocators) } return tb } // ResetTableInfo set tableInfo to nil for next reload. func (t *TableBuffer) ResetTableInfo() { t.tableInfo = nil } // TableInfo returns the table info of this buffer. func (t *TableBuffer) TableInfo() table.Table { return t.tableInfo } // TableID returns the table id of this buffer. func (t *TableBuffer) TableID() int64 { if t.tableInfo != nil { return t.tableInfo.Meta().ID } return 0 } // ReloadMeta reload columns after // 1. table buffer created. // 2. every ddl executed. func (t *TableBuffer) ReloadMeta(tbl table.Table, allocator autoid.Allocators) { columns := tbl.Meta().Cols() colNames := make([]string, 0, len(columns)) colPerm := make([]int, 0, len(columns)+1) for i, col := range columns { colNames = append(colNames, col.Name.String()) colPerm = append(colPerm, i) } if kv.TableHasAutoRowID(tbl.Meta()) { colPerm = append(colPerm, -1) } if t.allocator == nil { t.allocator = allocator } t.tableInfo = tbl t.colNames = colNames t.colPerm = colPerm // reset kv encoder after meta changed t.KvEncoder = nil } func (t *TableBuffer) translateToDatum(row map[string]Column) ([]types.Datum, error) { cols := make([]types.Datum, 0, len(row)) for _, col := range t.colNames { val, err := row[col].ToDatum() if err != nil { return nil, errors.Trace(err) } cols = append(cols, val) } return cols, nil } func (t *TableBuffer) appendRow( row map[string]Column, item *SortItem, encodeFn func(row []types.Datum, rowID int64, columnPermutation []int) (kv.Row, int, error), ) error { cols, err := t.translateToDatum(row) if err != nil { return errors.Trace(err) } pair, size, err := encodeFn(cols, item.RowID, t.colPerm) if err != nil { return errors.Trace(err) } t.KvPairs = append(t.KvPairs, pair) t.size += int64(size) t.count++ return nil } // Append appends the item to this buffer. func (t *TableBuffer) Append(item *SortItem) error { var err error log.Debug("Append item to buffer", zap.Stringer("table", t.tableInfo.Meta().Name), zap.Any("item", item), ) row := item.Data.(*MessageRow) if t.KvEncoder == nil { // lazy create kv encoder log.Debug("create kv encoder lazily", zap.Any("alloc", t.allocator), zap.Any("tbl", t.tableInfo)) t.KvEncoder, err = newKVEncoder(t.allocator, t.tableInfo) if err != nil { return errors.Trace(err) } } if row.PreColumns != nil { // remove old keys log.Debug("process update event", zap.Any("row", row)) err := t.appendRow(row.PreColumns, item, t.KvEncoder.RemoveRecord) if err != nil { return errors.Trace(err) } } if row.Update != nil { // Add new columns if row.PreColumns == nil { log.Debug("process insert event", zap.Any("row", row)) } err := t.appendRow(row.Update, item, t.KvEncoder.AddRecord) if err != nil { return errors.Trace(err) } } if row.Delete != nil { // Remove current columns log.Debug("process delete event", zap.Any("row", row)) err := t.appendRow(row.Delete, item, t.KvEncoder.RemoveRecord) if err != nil { return errors.Trace(err) } } return nil } // ShouldApply tells whether we should flush memory kv buffer to storage. func (t *TableBuffer) ShouldApply() bool { // flush when reached flush kv len or flush size return t.size >= t.flushKVSize || t.count >= t.flushKVPairs } // IsEmpty tells buffer is empty. func (t *TableBuffer) IsEmpty() bool { return t.size == 0 } // Clear reset the buffer. func (t *TableBuffer) Clear() { t.KvPairs = t.KvPairs[:0] t.count = 0 t.size = 0 }