// Copyright 2023 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package runtime import ( "context" "sync" "time" "github.com/pingcap/errors" "github.com/pingcap/tidb/timer/api" "github.com/pingcap/tidb/util/logutil" "go.uber.org/zap" ) const ( workerRecvChanCap = 8 workerRespChanCap = 128 workerEventDefaultRetryInterval = 10 * time.Second chanBlockInterval = time.Second ) type triggerEventRequest struct { eventID string timer *api.TimerRecord store *api.TimerStore resp chan<- *triggerEventResponse } type triggerEventResponse struct { success bool timerID string eventID string newTimerRecord api.OptionalVal[*api.TimerRecord] retryAfter api.OptionalVal[time.Duration] } type timerEvent struct { eventID string record *api.TimerRecord } func (e *timerEvent) EventID() string { return e.eventID } func (e *timerEvent) Timer() *api.TimerRecord { return e.record } type hookWorker struct { ctx context.Context wg *sync.WaitGroup hookClass string hook api.Hook ch chan *triggerEventRequest logger *zap.Logger nowFunc func() time.Time } func newHookWorker(ctx context.Context, wg *sync.WaitGroup, groupID string, hookClass string, hook api.Hook, nowFunc func() time.Time) *hookWorker { if nowFunc == nil { nowFunc = time.Now } w := &hookWorker{ ctx: ctx, wg: wg, hookClass: hookClass, hook: hook, ch: make(chan *triggerEventRequest, workerRecvChanCap), logger: logutil.BgLogger().With( zap.String("groupID", groupID), zap.String("hookClass", hookClass), ), nowFunc: nowFunc, } wg.Add(1) go w.loop() return w } func (w *hookWorker) loop() { w.logger.Info("timer hookWorker loop started") if w.hook != nil { w.hook.Start() } defer func() { if w.hook != nil { w.hook.Stop() } w.logger.Info("timer hookWorker loop exited") w.wg.Done() }() for { select { case <-w.ctx.Done(): return case req := <-w.ch: logger := w.logger.With( zap.String("timerID", req.timer.ID), zap.String("timerNamespace", req.timer.Namespace), zap.String("timerKey", req.timer.Key), zap.String("eventID", req.eventID), ) resp := w.triggerEvent(req, logger) if !w.responseChan(req.resp, resp, logger) { return } } } } func (w *hookWorker) triggerEvent(req *triggerEventRequest, logger *zap.Logger) *triggerEventResponse { timer := req.timer resp := &triggerEventResponse{ timerID: timer.ID, eventID: req.eventID, } if timer.EventStatus == api.SchedEventIdle { var preResult api.PreSchedEventResult if w.hook != nil { logger.Debug("call OnPreSchedEvent") result, err := w.hook.OnPreSchedEvent(w.ctx, &timerEvent{ eventID: req.eventID, record: timer, }) if err != nil { logger.Error( "error occurs when invoking hook.OnPreSchedEvent", zap.Error(err), zap.Duration("retryAfter", workerEventDefaultRetryInterval), ) resp.retryAfter.Set(workerEventDefaultRetryInterval) return resp } if result.Delay > 0 { resp.retryAfter.Set(result.Delay) return resp } preResult = result } update := buildEventUpdate(req, preResult, w.nowFunc) if err := req.store.Update(w.ctx, timer.ID, update); err != nil { if errors.ErrorEqual(err, api.ErrVersionNotMatch) { newTimer, getErr := req.store.GetByID(w.ctx, timer.ID) if getErr != nil { err = getErr } else { resp.newTimerRecord.Set(newTimer) } } if errors.ErrorEqual(err, api.ErrTimerNotExist) { logger.Info("cannot change timer to trigger state, timer deleted") resp.newTimerRecord.Set(nil) } else if errors.ErrorEqual(err, api.ErrVersionNotMatch) { logger.Info("cannot change timer to trigger state, timer version not match", zap.Uint64("timerVersion", timer.Version), ) if newTimer, err := req.store.GetByID(w.ctx, timer.ID); err == nil { resp.newTimerRecord.Set(newTimer) } } else { logger.Error("error occurs to change timer to trigger state,", zap.Error(err), zap.Duration("retryAfter", workerEventDefaultRetryInterval), ) resp.retryAfter.Set(workerEventDefaultRetryInterval) } return resp } } timer, err := req.store.GetByID(w.ctx, timer.ID) if errors.ErrorEqual(err, api.ErrTimerNotExist) { logger.Info("cannot trigger timer event, timer deleted") resp.newTimerRecord.Set(nil) return resp } if err != nil { logger.Error( "error occurs when getting timer record to trigger timer event", zap.Duration("retryAfter", workerEventDefaultRetryInterval), ) resp.retryAfter.Set(workerEventDefaultRetryInterval) return resp } resp.newTimerRecord.Set(timer) if timer.EventID != req.eventID { logger.Info("cannot trigger timer event, timer event closed") return resp } if w.hook != nil { logger.Debug("call OnSchedEvent") err = w.hook.OnSchedEvent(w.ctx, &timerEvent{ eventID: req.eventID, record: timer, }) if err != nil { logger.Error( "error occurs when invoking hook OnTimerEvent", zap.Error(err), zap.Duration("retryAfter", workerEventDefaultRetryInterval), ) resp.retryAfter.Set(workerEventDefaultRetryInterval) return resp } } resp.success = true return resp } func (w *hookWorker) responseChan(ch chan<- *triggerEventResponse, resp *triggerEventResponse, logger *zap.Logger) bool { sendStart := time.Now() timeout := time.NewTimer(chanBlockInterval) defer timeout.Stop() for { select { case <-w.ctx.Done(): logger.Info("sending resp to chan aborted for context cancelled") zap.Duration("totalBlock", time.Since(sendStart)) return false case ch <- resp: return true case <-timeout.C: logger.Warn( "sending resp to chan is blocked for a long time", zap.Duration("totalBlock", time.Since(sendStart)), ) } } } func buildEventUpdate(req *triggerEventRequest, result api.PreSchedEventResult, nowFunc func() time.Time) *api.TimerUpdate { var update api.TimerUpdate update.EventStatus.Set(api.SchedEventTrigger) update.EventID.Set(req.eventID) update.EventStart.Set(nowFunc()) update.EventData.Set(result.EventData) update.CheckVersion.Set(req.timer.Version) return &update }