526 lines
17 KiB
Go
526 lines
17 KiB
Go
// Copyright 2023 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/pingcap/errors"
|
|
"github.com/pingcap/tidb/timer/api"
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/atomic"
|
|
)
|
|
|
|
func TestWorkerStartStop(t *testing.T) {
|
|
var wg sync.WaitGroup
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
hook := newMockHook()
|
|
hook.On("Start").Return().Once()
|
|
newHookWorker(ctx, &wg, "g1", "h1", hook, nil)
|
|
waitDone(hook.started, time.Second)
|
|
hook.AssertExpectations(t)
|
|
|
|
checkNotDone(hook.stopped, 100*time.Millisecond)
|
|
hook.On("Stop").Return().Once()
|
|
cancel()
|
|
waitDone(hook.stopped, time.Second)
|
|
waitDone(&wg, time.Second)
|
|
hook.AssertExpectations(t)
|
|
}
|
|
|
|
func prepareTimer(t *testing.T, cli api.TimerClient) *api.TimerRecord {
|
|
now := time.Now()
|
|
timer, err := cli.CreateTimer(context.TODO(), api.TimerSpec{
|
|
Key: "key1",
|
|
Data: []byte("data1"),
|
|
SchedPolicyType: api.SchedEventInterval,
|
|
SchedPolicyExpr: "1m",
|
|
HookClass: "h1",
|
|
Enable: true,
|
|
})
|
|
require.NoError(t, err)
|
|
watermark := now.Add(-time.Minute)
|
|
err = cli.UpdateTimer(
|
|
context.TODO(), timer.ID,
|
|
api.WithSetWatermark(watermark),
|
|
api.WithSetSummaryData([]byte("summary1")),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
timer, err = cli.GetTimerByID(context.TODO(), timer.ID)
|
|
require.NoError(t, err)
|
|
|
|
require.NotEmpty(t, timer.ID)
|
|
require.Equal(t, []byte("data1"), timer.Data)
|
|
require.Equal(t, api.SchedEventInterval, timer.SchedPolicyType)
|
|
require.Equal(t, "1m", timer.SchedPolicyExpr)
|
|
require.Equal(t, "h1", timer.HookClass)
|
|
require.True(t, timer.Enable)
|
|
require.Equal(t, watermark, timer.Watermark)
|
|
require.Equal(t, []byte("summary1"), timer.SummaryData)
|
|
require.True(t, !timer.CreateTime.Before(now))
|
|
require.True(t, !timer.CreateTime.After(time.Now()))
|
|
require.Greater(t, timer.Version, uint64(0))
|
|
require.Empty(t, timer.EventID)
|
|
require.Equal(t, api.SchedEventIdle, timer.EventStatus)
|
|
require.Equal(t, 0, len(timer.EventData))
|
|
require.True(t, timer.EventStart.IsZero())
|
|
return timer
|
|
}
|
|
|
|
func getAndCheckTriggeredTimer(
|
|
t *testing.T, cli api.TimerClient, oldTimer *api.TimerRecord,
|
|
eventID string, eventData []byte, eventStartAfter time.Time, eventStartBefore time.Time,
|
|
) *api.TimerRecord {
|
|
timer, err := cli.GetTimerByID(context.Background(), oldTimer.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, api.SchedEventTrigger, timer.EventStatus)
|
|
require.Equal(t, eventID, timer.EventID)
|
|
require.Equal(t, eventData, timer.EventData)
|
|
require.True(t, !timer.EventStart.Before(eventStartAfter))
|
|
require.True(t, !timer.EventStart.After(eventStartBefore))
|
|
require.Greater(t, timer.Version, oldTimer.Version)
|
|
oldTimer = oldTimer.Clone()
|
|
oldTimer.EventID = timer.EventID
|
|
oldTimer.EventData = timer.EventData
|
|
oldTimer.EventStatus = timer.EventStatus
|
|
oldTimer.EventStart = timer.EventStart
|
|
oldTimer.Version = timer.Version
|
|
require.Equal(t, oldTimer, timer)
|
|
return timer
|
|
}
|
|
|
|
func sendWorkerRequestAndCheckResp(
|
|
t *testing.T, w *hookWorker, req *triggerEventRequest,
|
|
respCh <-chan *triggerEventResponse, fn func(response *triggerEventResponse),
|
|
) {
|
|
select {
|
|
case w.ch <- req:
|
|
timeout := time.NewTimer(time.Second)
|
|
defer timeout.Stop()
|
|
select {
|
|
case resp, ok := <-respCh:
|
|
require.True(t, ok)
|
|
fn(resp)
|
|
case <-timeout.C:
|
|
t.FailNow()
|
|
}
|
|
default:
|
|
t.FailNow()
|
|
}
|
|
|
|
select {
|
|
case <-respCh:
|
|
t.FailNow()
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestWorkerProcessIdleTimerSuccess(t *testing.T) {
|
|
var wg sync.WaitGroup
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
store := api.NewMemoryTimerStore()
|
|
cli := api.NewDefaultTimerClient(store)
|
|
respChan := make(chan *triggerEventResponse)
|
|
timer := prepareTimer(t, cli)
|
|
|
|
hook := newMockHook()
|
|
hook.On("Start").Return().Once()
|
|
hook.On("Stop").Return().Once()
|
|
|
|
w := newHookWorker(ctx, &wg, "g1", "h1", hook, nil)
|
|
eventID := uuid.NewString()
|
|
var eventStartRef atomic.Pointer[time.Time]
|
|
var finalTimerRef atomic.Pointer[api.TimerRecord]
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{EventData: []byte("eventdata")}, nil).Once().
|
|
Run(func(args mock.Arguments) {
|
|
funcCtx := args[0].(context.Context)
|
|
event := args[1].(api.TimerShedEvent)
|
|
require.NotNil(t, funcCtx)
|
|
require.Equal(t, eventID, event.EventID())
|
|
require.NotNil(t, event.Timer())
|
|
require.Equal(t, timer, event.Timer())
|
|
now := time.Now()
|
|
eventStartRef.Store(&now)
|
|
})
|
|
|
|
hook.On("OnSchedEvent", mock.Anything, mock.Anything).
|
|
Return(nil).Once().
|
|
Run(func(args mock.Arguments) {
|
|
funcCtx := args[0].(context.Context)
|
|
event := args[1].(api.TimerShedEvent)
|
|
require.NotNil(t, funcCtx)
|
|
require.Equal(t, eventID, event.EventID())
|
|
funcTimer := event.Timer()
|
|
require.NotNil(t, funcTimer)
|
|
tm := getAndCheckTriggeredTimer(t, cli, timer, event.EventID(),
|
|
[]byte("eventdata"), *eventStartRef.Load(), time.Now())
|
|
require.Equal(t, tm, funcTimer)
|
|
finalTimerRef.Store(tm)
|
|
})
|
|
|
|
request := &triggerEventRequest{
|
|
eventID: eventID,
|
|
timer: timer,
|
|
store: store,
|
|
resp: respChan,
|
|
}
|
|
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.True(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newTimer, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, finalTimerRef.Load(), newTimer)
|
|
})
|
|
|
|
cancel()
|
|
waitDone(hook.stopped, time.Second)
|
|
hook.AssertExpectations(t)
|
|
}
|
|
|
|
func TestWorkerProcessTriggeredTimerSuccess(t *testing.T) {
|
|
var wg sync.WaitGroup
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
store := api.NewMemoryTimerStore()
|
|
cli := api.NewDefaultTimerClient(store)
|
|
respChan := make(chan *triggerEventResponse)
|
|
timer := prepareTimer(t, cli)
|
|
eventStart := time.Now()
|
|
eventID := uuid.NewString()
|
|
err := store.Update(ctx, timer.ID, &api.TimerUpdate{
|
|
EventID: api.NewOptionalVal(eventID),
|
|
EventStatus: api.NewOptionalVal(api.SchedEventTrigger),
|
|
EventData: api.NewOptionalVal([]byte("eventdata")),
|
|
EventStart: api.NewOptionalVal(eventStart),
|
|
})
|
|
require.NoError(t, err)
|
|
timer = getAndCheckTriggeredTimer(t, cli, timer, eventID, []byte("eventdata"), eventStart, eventStart)
|
|
|
|
hook := newMockHook()
|
|
hook.On("Start").Return().Once()
|
|
hook.On("Stop").Return().Once()
|
|
|
|
w := newHookWorker(ctx, &wg, "g1", "h1", hook, nil)
|
|
hook.On("OnSchedEvent", mock.Anything, mock.Anything).
|
|
Return(nil).Once().
|
|
Run(func(args mock.Arguments) {
|
|
funcCtx := args[0].(context.Context)
|
|
event := args[1].(api.TimerShedEvent)
|
|
require.NotNil(t, funcCtx)
|
|
require.Equal(t, eventID, event.EventID())
|
|
funcTimer := event.Timer()
|
|
require.NotNil(t, funcTimer)
|
|
require.Equal(t, timer, funcTimer)
|
|
})
|
|
|
|
request := &triggerEventRequest{
|
|
eventID: eventID,
|
|
timer: timer,
|
|
store: store,
|
|
resp: respChan,
|
|
}
|
|
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.True(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newTimer, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, timer, newTimer)
|
|
})
|
|
|
|
cancel()
|
|
waitDone(hook.stopped, time.Second)
|
|
hook.AssertExpectations(t)
|
|
}
|
|
|
|
func TestWorkerProcessDelayOrErr(t *testing.T) {
|
|
var wg sync.WaitGroup
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
store := api.NewMemoryTimerStore()
|
|
cli := api.NewDefaultTimerClient(store)
|
|
respChan := make(chan *triggerEventResponse)
|
|
timer := prepareTimer(t, cli)
|
|
|
|
hook := newMockHook()
|
|
hook.On("Start").Return().Once()
|
|
hook.On("Stop").Return().Once()
|
|
|
|
w := newHookWorker(ctx, &wg, "g1", "h1", hook, nil)
|
|
eventID := uuid.NewString()
|
|
request := &triggerEventRequest{
|
|
eventID: eventID,
|
|
timer: timer,
|
|
store: store,
|
|
resp: respChan,
|
|
}
|
|
|
|
// Delay 5 seconds
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{Delay: 5 * time.Second}, nil).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
retryAfter, ok := resp.retryAfter.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, 5*time.Second, retryAfter)
|
|
_, ok = resp.newTimerRecord.Get()
|
|
require.False(t, ok)
|
|
})
|
|
|
|
// OnPreSchedEvent error
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{}, errors.New("mockErr")).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
delay, ok := resp.retryAfter.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, workerEventDefaultRetryInterval, delay)
|
|
_, ok = resp.newTimerRecord.Get()
|
|
require.False(t, ok)
|
|
})
|
|
|
|
tm, err := cli.GetTimerByID(ctx, timer.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, timer, tm)
|
|
|
|
// update timer unknown error
|
|
mockCore, mockStore := newMockStore()
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{}, nil).Once()
|
|
mockCore.On("Update", mock.Anything, mock.Anything, mock.Anything).
|
|
Return(errors.New("mockErr")).Once()
|
|
request.store = mockStore
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
delay, ok := resp.retryAfter.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, workerEventDefaultRetryInterval, delay)
|
|
_, ok = resp.newTimerRecord.Get()
|
|
require.False(t, ok)
|
|
})
|
|
|
|
// timer meta changed then get record error
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{}, nil).Once()
|
|
mockCore.On("Update", mock.Anything, mock.Anything, mock.Anything).
|
|
Return(api.ErrVersionNotMatch).Once()
|
|
mockCore.On("List", mock.Anything, mock.Anything).
|
|
Return([]*api.TimerRecord(nil), errors.New("mockErr")).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
delay, ok := resp.retryAfter.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, workerEventDefaultRetryInterval, delay)
|
|
_, ok = resp.newTimerRecord.Get()
|
|
require.False(t, ok)
|
|
})
|
|
|
|
// timer event updated then get record error
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{}, nil).Once()
|
|
mockCore.On("Update", mock.Anything, mock.Anything, mock.Anything).
|
|
Return(nil).Once()
|
|
mockCore.On("List", mock.Anything, mock.Anything).
|
|
Return([]*api.TimerRecord(nil), errors.New("mockErr")).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
delay, ok := resp.retryAfter.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, workerEventDefaultRetryInterval, delay)
|
|
_, ok = resp.newTimerRecord.Get()
|
|
require.False(t, ok)
|
|
})
|
|
|
|
// timer event updated then get record return nil
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{}, nil).Once()
|
|
mockCore.On("Update", mock.Anything, mock.Anything, mock.Anything).
|
|
Return(nil).Once()
|
|
mockCore.On("List", mock.Anything, mock.Anything).
|
|
Return([]*api.TimerRecord(nil), nil).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newRecord, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
require.Nil(t, newRecord)
|
|
})
|
|
|
|
// timer event updated then get record return different eventID
|
|
anotherEventIDTimer := timer.Clone()
|
|
anotherEventIDTimer.Version += 2
|
|
anotherEventIDTimer.EventStatus = api.SchedEventTrigger
|
|
anotherEventIDTimer.EventID = "anothereventid"
|
|
anotherEventIDTimer.EventStart = time.Now()
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{}, nil).Once()
|
|
mockCore.On("Update", mock.Anything, mock.Anything, mock.Anything).
|
|
Return(nil).Once()
|
|
mockCore.On("List", mock.Anything, mock.Anything).
|
|
Return([]*api.TimerRecord{anotherEventIDTimer}, nil).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newRecord, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, anotherEventIDTimer, newRecord)
|
|
})
|
|
request.store = store
|
|
|
|
// timer meta changed
|
|
err = cli.UpdateTimer(ctx, timer.ID, api.WithSetSchedExpr(api.SchedEventInterval, "2m"))
|
|
require.NoError(t, err)
|
|
|
|
tm, err = cli.GetTimerByID(ctx, timer.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "2m", tm.SchedPolicyExpr)
|
|
require.Greater(t, tm.Version, timer.Version)
|
|
timer = tm
|
|
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{}, nil).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newTimer, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, timer, newTimer)
|
|
})
|
|
|
|
// OnSchedEvent error
|
|
now := time.Now()
|
|
request.timer = timer
|
|
var finalTimerRef atomic.Pointer[api.TimerRecord]
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{EventData: []byte("eventdata")}, nil).Once()
|
|
hook.On("OnSchedEvent", mock.Anything, mock.Anything).
|
|
Return(errors.New("mockErr")).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
delay, ok := resp.retryAfter.Get()
|
|
require.True(t, ok)
|
|
require.Equal(t, workerEventDefaultRetryInterval, delay)
|
|
newTimer, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
finalTimerRef.Store(newTimer)
|
|
})
|
|
timer = getAndCheckTriggeredTimer(t, cli, timer, eventID, []byte("eventdata"), now, time.Now())
|
|
require.Equal(t, timer, finalTimerRef.Load())
|
|
request.timer = timer
|
|
|
|
// Event closed before trigger
|
|
err = cli.CloseTimerEvent(ctx, timer.ID, eventID)
|
|
require.NoError(t, err)
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newTimer, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
finalTimerRef.Store(newTimer)
|
|
})
|
|
timer, err = cli.GetTimerByID(ctx, timer.ID)
|
|
require.Nil(t, err)
|
|
require.Empty(t, timer.EventID)
|
|
require.Equal(t, timer, finalTimerRef.Load())
|
|
|
|
// Timer deleted
|
|
exist, err := cli.DeleteTimer(ctx, timer.ID)
|
|
require.NoError(t, err)
|
|
require.True(t, exist)
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newTimer, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
require.Nil(t, newTimer)
|
|
})
|
|
|
|
// Timer deleted after OnPreSchedEvent
|
|
timer = prepareTimer(t, cli)
|
|
eventID = uuid.NewString()
|
|
request = &triggerEventRequest{
|
|
eventID: eventID,
|
|
timer: timer,
|
|
store: store,
|
|
resp: respChan,
|
|
}
|
|
exist, err = cli.DeleteTimer(ctx, timer.ID)
|
|
require.NoError(t, err)
|
|
require.True(t, exist)
|
|
hook.On("OnPreSchedEvent", mock.Anything, mock.Anything).
|
|
Return(api.PreSchedEventResult{EventData: []byte("eventdata")}, nil).Once()
|
|
sendWorkerRequestAndCheckResp(t, w, request, respChan, func(resp *triggerEventResponse) {
|
|
require.False(t, resp.success)
|
|
require.Equal(t, timer.ID, resp.timerID)
|
|
require.Equal(t, eventID, resp.eventID)
|
|
_, ok := resp.retryAfter.Get()
|
|
require.False(t, ok)
|
|
newTimer, ok := resp.newTimerRecord.Get()
|
|
require.True(t, ok)
|
|
require.Nil(t, newTimer)
|
|
})
|
|
|
|
cancel()
|
|
waitDone(hook.stopped, time.Second)
|
|
hook.AssertExpectations(t)
|
|
}
|