Add unit tests for runner recovery.
This commit is contained in:

committed by
Sebastian Serth

parent
160a097d07
commit
f259d65aa4
@ -235,3 +235,30 @@ func (s *MainTestSuite) TestNomadEnvironment_DeleteLocally() {
|
|||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
apiMock.AssertExpectations(s.T())
|
apiMock.AssertExpectations(s.T())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MainTestSuite) TestNomadEnvironment_AddRunner() {
|
||||||
|
s.Run("Destroys runner before replacing it", func() {
|
||||||
|
apiMock := &nomad.ExecutorAPIMock{}
|
||||||
|
environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
r := &runner.RunnerMock{}
|
||||||
|
r.On("ID").Return(tests.DefaultRunnerID)
|
||||||
|
r.On("Destroy", mock.Anything).Run(func(args mock.Arguments) {
|
||||||
|
err, ok := args[0].(error)
|
||||||
|
s.Require().True(ok)
|
||||||
|
s.ErrorIs(err, runner.ErrLocalDestruction)
|
||||||
|
}).Return(nil).Once()
|
||||||
|
r2 := &runner.RunnerMock{}
|
||||||
|
r2.On("ID").Return(tests.DefaultRunnerID)
|
||||||
|
|
||||||
|
environment.AddRunner(r)
|
||||||
|
environment.AddRunner(r2)
|
||||||
|
r.AssertExpectations(s.T())
|
||||||
|
|
||||||
|
// Teardown test case
|
||||||
|
r2.On("Destroy", mock.Anything).Return(nil)
|
||||||
|
apiMock.On("LoadRunnerIDs", mock.Anything).Return([]string{}, nil)
|
||||||
|
apiMock.On("DeleteJob", mock.Anything).Return(nil)
|
||||||
|
s.NoError(environment.Delete(tests.ErrCleanupDestroyReason))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CreateOrUpdateTestSuite struct {
|
type CreateOrUpdateTestSuite struct {
|
||||||
@ -315,6 +316,26 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_Load() {
|
|||||||
|
|
||||||
runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx)
|
runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx)
|
||||||
|
|
||||||
|
s.Run("deletes local environments before loading Nomad environments", func() {
|
||||||
|
call.Return([]*nomadApi.Job{}, nil)
|
||||||
|
environment := &runner.ExecutionEnvironmentMock{}
|
||||||
|
environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger))
|
||||||
|
environment.On("Image").Return("")
|
||||||
|
environment.On("CPULimit").Return(uint(0))
|
||||||
|
environment.On("MemoryLimit").Return(uint(0))
|
||||||
|
environment.On("NetworkAccess").Return(false, nil)
|
||||||
|
environment.On("Delete", mock.Anything).Return(nil)
|
||||||
|
runnerManager.StoreEnvironment(environment)
|
||||||
|
|
||||||
|
m, err := NewNomadEnvironmentManager(runnerManager, apiMock, "")
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
err = m.load()
|
||||||
|
s.Require().NoError(err)
|
||||||
|
environment.AssertExpectations(s.T())
|
||||||
|
})
|
||||||
|
runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger)
|
||||||
|
|
||||||
s.Run("Stores fetched environments", func() {
|
s.Run("Stores fetched environments", func() {
|
||||||
_, job := helpers.CreateTemplateJob()
|
_, job := helpers.CreateTemplateJob()
|
||||||
call.Return([]*nomadApi.Job{job}, nil)
|
call.Return([]*nomadApi.Job{job}, nil)
|
||||||
@ -354,6 +375,66 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_Load() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MainTestSuite) TestNomadEnvironmentManager_KeepEnvironmentsSynced() {
|
||||||
|
apiMock := &nomad.ExecutorAPIMock{}
|
||||||
|
runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx)
|
||||||
|
m, err := NewNomadEnvironmentManager(runnerManager, apiMock, "")
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
s.Run("stops when context is done", func() {
|
||||||
|
apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, context.DeadlineExceeded)
|
||||||
|
ctx, cancel := context.WithCancel(s.TestCtx)
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
var done bool
|
||||||
|
go func() {
|
||||||
|
<-time.After(tests.ShortTimeout)
|
||||||
|
if !done {
|
||||||
|
s.FailNow("KeepEnvironmentsSynced is ignoring the context")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
m.KeepEnvironmentsSynced(func(_ context.Context) error { return nil }, ctx)
|
||||||
|
done = true
|
||||||
|
})
|
||||||
|
apiMock.ExpectedCalls = []*mock.Call{}
|
||||||
|
apiMock.Calls = []mock.Call{}
|
||||||
|
|
||||||
|
s.Run("retries loading environments", func() {
|
||||||
|
ctx, cancel := context.WithCancel(s.TestCtx)
|
||||||
|
|
||||||
|
apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, context.DeadlineExceeded).Once()
|
||||||
|
apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil).Run(func(_ mock.Arguments) {
|
||||||
|
cancel()
|
||||||
|
}).Once()
|
||||||
|
|
||||||
|
m.KeepEnvironmentsSynced(func(_ context.Context) error { return nil }, ctx)
|
||||||
|
apiMock.AssertExpectations(s.T())
|
||||||
|
})
|
||||||
|
apiMock.ExpectedCalls = []*mock.Call{}
|
||||||
|
apiMock.Calls = []mock.Call{}
|
||||||
|
|
||||||
|
s.Run("retries synchronizing runners", func() {
|
||||||
|
apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil)
|
||||||
|
ctx, cancel := context.WithCancel(s.TestCtx)
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
synchronizeRunners := func(ctx context.Context) error {
|
||||||
|
count++
|
||||||
|
if count >= 2 {
|
||||||
|
cancel()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return context.DeadlineExceeded
|
||||||
|
}
|
||||||
|
m.KeepEnvironmentsSynced(synchronizeRunners, ctx)
|
||||||
|
|
||||||
|
if count < 2 {
|
||||||
|
s.Fail("KeepEnvironmentsSynced is not retrying to synchronize the runners")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func mockWatchAllocations(ctx context.Context, apiMock *nomad.ExecutorAPIMock) {
|
func mockWatchAllocations(ctx context.Context, apiMock *nomad.ExecutorAPIMock) {
|
||||||
call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything)
|
call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything)
|
||||||
call.Run(func(args mock.Arguments) {
|
call.Run(func(args mock.Arguments) {
|
||||||
|
@ -267,7 +267,7 @@ func (s *ManagerTestSuite) TestUpdateRunnersLogsErrorFromWatchAllocation() {
|
|||||||
}()
|
}()
|
||||||
<-time.After(10 * time.Millisecond)
|
<-time.After(10 * time.Millisecond)
|
||||||
|
|
||||||
s.Require().Equal(1, len(hook.Entries))
|
s.Require().Equal(3, len(hook.Entries))
|
||||||
s.Equal(logrus.ErrorLevel, hook.LastEntry().Level)
|
s.Equal(logrus.ErrorLevel, hook.LastEntry().Level)
|
||||||
err, ok := hook.LastEntry().Data[logrus.ErrorKey].(error)
|
err, ok := hook.LastEntry().Data[logrus.ErrorKey].(error)
|
||||||
s.Require().True(ok)
|
s.Require().True(ok)
|
||||||
|
Reference in New Issue
Block a user