package runner import ( "context" nomadApi "github.com/hashicorp/nomad/api" "github.com/openHPI/poseidon/internal/nomad" "github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/tests" "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "testing" "time" ) func TestGetNextRunnerTestSuite(t *testing.T) { suite.Run(t, new(ManagerTestSuite)) } type ManagerTestSuite struct { suite.Suite apiMock *nomad.ExecutorAPIMock nomadRunnerManager *NomadRunnerManager exerciseEnvironment *ExecutionEnvironmentMock exerciseRunner Runner } func (s *ManagerTestSuite) SetupTest() { s.apiMock = &nomad.ExecutorAPIMock{} mockRunnerQueries(s.apiMock, []string{}) // Instantly closed context to manually start the update process in some cases ctx, cancel := context.WithCancel(context.Background()) cancel() s.nomadRunnerManager = NewNomadRunnerManager(s.apiMock, ctx) s.exerciseRunner = NewRunner(tests.DefaultRunnerID, s.nomadRunnerManager) s.exerciseEnvironment = &ExecutionEnvironmentMock{} s.setDefaultEnvironment() } func mockRunnerQueries(apiMock *nomad.ExecutorAPIMock, returnedRunnerIds []string) { // reset expected calls to allow new mocked return values apiMock.ExpectedCalls = []*mock.Call{} call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything) call.Run(func(args mock.Arguments) { <-time.After(10 * time.Minute) // 10 minutes is the default test timeout call.ReturnArguments = mock.Arguments{nil} }) apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil) apiMock.On("MarkRunnerAsUsed", mock.AnythingOfType("string"), mock.AnythingOfType("int")).Return(nil) apiMock.On("LoadRunnerIDs", tests.DefaultRunnerID).Return(returnedRunnerIds, nil) apiMock.On("JobScale", tests.DefaultRunnerID).Return(uint(len(returnedRunnerIds)), nil) apiMock.On("SetJobScale", tests.DefaultRunnerID, mock.AnythingOfType("uint"), "Runner Requested").Return(nil) apiMock.On("RegisterRunnerJob", mock.Anything).Return(nil) apiMock.On("MonitorEvaluation", mock.Anything, mock.Anything).Return(nil) } func mockIdleRunners(environmentMock *ExecutionEnvironmentMock) { idleRunner := NewLocalRunnerStorage() environmentMock.On("AddRunner", mock.Anything).Run(func(args mock.Arguments) { r, ok := args.Get(0).(Runner) if !ok { return } idleRunner.Add(r) }) sampleCall := environmentMock.On("Sample", mock.Anything) sampleCall.Run(func(args mock.Arguments) { r, ok := idleRunner.Sample() sampleCall.ReturnArguments = mock.Arguments{r, ok} }) deleteCall := environmentMock.On("DeleteRunner", mock.AnythingOfType("string")) deleteCall.Run(func(args mock.Arguments) { id, ok := args.Get(0).(string) if !ok { return } idleRunner.Delete(id) }) } func (s *ManagerTestSuite) setDefaultEnvironment() { s.exerciseEnvironment.On("ID").Return(defaultEnvironmentID) created := s.nomadRunnerManager.SetEnvironment(s.exerciseEnvironment) s.Require().True(created) } func (s *ManagerTestSuite) waitForRunnerRefresh() { <-time.After(100 * time.Millisecond) } func (s *ManagerTestSuite) TestSetEnvironmentAddsNewEnvironment() { anotherEnvironment := &ExecutionEnvironmentMock{} anotherEnvironment.On("ID").Return(anotherEnvironmentID) created := s.nomadRunnerManager.SetEnvironment(anotherEnvironment) s.Require().True(created) job, ok := s.nomadRunnerManager.environments.Get(anotherEnvironmentID) s.True(ok) s.NotNil(job) } func (s *ManagerTestSuite) TestClaimReturnsNotFoundErrorIfEnvironmentNotFound() { runner, err := s.nomadRunnerManager.Claim(anotherEnvironmentID, defaultInactivityTimeout) s.Nil(runner) s.Equal(ErrUnknownExecutionEnvironment, err) } func (s *ManagerTestSuite) TestClaimReturnsRunnerIfAvailable() { s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) receivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) s.NoError(err) s.Equal(s.exerciseRunner, receivedRunner) } func (s *ManagerTestSuite) TestClaimReturnsErrorIfNoRunnerAvailable() { s.waitForRunnerRefresh() s.exerciseEnvironment.On("Sample", mock.Anything).Return(nil, false) runner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) s.Nil(runner) s.Equal(ErrNoRunnersAvailable, err) } func (s *ManagerTestSuite) TestClaimReturnsNoRunnerOfDifferentEnvironment() { s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) receivedRunner, err := s.nomadRunnerManager.Claim(anotherEnvironmentID, defaultInactivityTimeout) s.Nil(receivedRunner) s.Error(err) } func (s *ManagerTestSuite) TestClaimDoesNotReturnTheSameRunnerTwice() { s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true).Once() s.exerciseEnvironment.On("Sample", mock.Anything). Return(NewRunner(tests.AnotherRunnerID, s.nomadRunnerManager), true).Once() firstReceivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) s.NoError(err) secondReceivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) s.NoError(err) s.NotEqual(firstReceivedRunner, secondReceivedRunner) } func (s *ManagerTestSuite) TestClaimAddsRunnerToUsedRunners() { s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) receivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) s.Require().NoError(err) savedRunner, ok := s.nomadRunnerManager.usedRunners.Get(receivedRunner.ID()) s.True(ok) s.Equal(savedRunner, receivedRunner) } func (s *ManagerTestSuite) TestClaimRemovesRunnerWhenMarkAsUsedFails() { s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) modifyMockedCall(s.apiMock, "MarkRunnerAsUsed", func(call *mock.Call) { call.Run(func(args mock.Arguments) { call.ReturnArguments = mock.Arguments{tests.ErrDefault} }) }) claimedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) s.Require().NoError(err) <-time.After(tests.ShortTimeout) // Claimed runners are marked as used asynchronously s.apiMock.AssertCalled(s.T(), "DeleteJob", claimedRunner.ID()) _, ok := s.nomadRunnerManager.usedRunners.Get(claimedRunner.ID()) s.False(ok) } func (s *ManagerTestSuite) TestGetReturnsRunnerIfRunnerIsUsed() { s.nomadRunnerManager.usedRunners.Add(s.exerciseRunner) savedRunner, err := s.nomadRunnerManager.Get(s.exerciseRunner.ID()) s.NoError(err) s.Equal(savedRunner, s.exerciseRunner) } func (s *ManagerTestSuite) TestGetReturnsErrorIfRunnerNotFound() { savedRunner, err := s.nomadRunnerManager.Get(tests.DefaultRunnerID) s.Nil(savedRunner) s.Error(err) } func (s *ManagerTestSuite) TestReturnRemovesRunnerFromUsedRunners() { s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) s.nomadRunnerManager.usedRunners.Add(s.exerciseRunner) err := s.nomadRunnerManager.Return(s.exerciseRunner) s.Nil(err) _, ok := s.nomadRunnerManager.usedRunners.Get(s.exerciseRunner.ID()) s.False(ok) } func (s *ManagerTestSuite) TestReturnCallsDeleteRunnerApiMethod() { s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) err := s.nomadRunnerManager.Return(s.exerciseRunner) s.Nil(err) s.apiMock.AssertCalled(s.T(), "DeleteJob", s.exerciseRunner.ID()) } func (s *ManagerTestSuite) TestReturnReturnsErrorWhenApiCallFailed() { s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(tests.ErrDefault) err := s.nomadRunnerManager.Return(s.exerciseRunner) s.Error(err) } func (s *ManagerTestSuite) TestUpdateRunnersLogsErrorFromWatchAllocation() { var hook *test.Hook logger, hook := test.NewNullLogger() log = logger.WithField("pkg", "runner") modifyMockedCall(s.apiMock, "WatchEventStream", func(call *mock.Call) { call.Run(func(args mock.Arguments) { call.ReturnArguments = mock.Arguments{tests.ErrDefault} }) }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go s.nomadRunnerManager.keepRunnersSynced(ctx) <-time.After(10 * time.Millisecond) s.Require().Equal(1, len(hook.Entries)) s.Equal(logrus.ErrorLevel, hook.LastEntry().Level) s.Equal(hook.LastEntry().Data[logrus.ErrorKey], tests.ErrDefault) } func (s *ManagerTestSuite) TestUpdateRunnersAddsIdleRunner() { allocation := &nomadApi.Allocation{ID: tests.DefaultRunnerID} environment, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID) s.Require().True(ok) allocation.JobID = environment.ID().ToString() mockIdleRunners(environment.(*ExecutionEnvironmentMock)) _, ok = environment.Sample(s.apiMock) s.Require().False(ok) modifyMockedCall(s.apiMock, "WatchEventStream", func(call *mock.Call) { call.Run(func(args mock.Arguments) { onCreate, ok := args.Get(1).(nomad.AllocationProcessor) s.Require().True(ok) onCreate(allocation) call.ReturnArguments = mock.Arguments{nil} }) }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go s.nomadRunnerManager.keepRunnersSynced(ctx) <-time.After(10 * time.Millisecond) _, ok = environment.Sample(s.apiMock) s.True(ok) } func (s *ManagerTestSuite) TestUpdateRunnersRemovesIdleAndUsedRunner() { allocation := &nomadApi.Allocation{JobID: tests.DefaultRunnerID} environment, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID) s.Require().True(ok) mockIdleRunners(environment.(*ExecutionEnvironmentMock)) testRunner := NewRunner(allocation.JobID, s.nomadRunnerManager) environment.AddRunner(testRunner) s.nomadRunnerManager.usedRunners.Add(testRunner) modifyMockedCall(s.apiMock, "WatchEventStream", func(call *mock.Call) { call.Run(func(args mock.Arguments) { onDelete, ok := args.Get(2).(nomad.AllocationProcessor) s.Require().True(ok) onDelete(allocation) call.ReturnArguments = mock.Arguments{nil} }) }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go s.nomadRunnerManager.keepRunnersSynced(ctx) <-time.After(10 * time.Millisecond) _, ok = environment.Sample(s.apiMock) s.False(ok) _, ok = s.nomadRunnerManager.usedRunners.Get(allocation.JobID) s.False(ok) } func modifyMockedCall(apiMock *nomad.ExecutorAPIMock, method string, modifier func(call *mock.Call)) { for _, c := range apiMock.ExpectedCalls { if c.Method == method { modifier(c) } } } func (s *ManagerTestSuite) TestOnAllocationAdded() { s.Run("does not add environment template id job", func() { environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsInteger) s.True(ok) mockIdleRunners(environment.(*ExecutionEnvironmentMock)) alloc := &nomadApi.Allocation{JobID: nomad.TemplateJobID(tests.DefaultEnvironmentIDAsInteger)} s.nomadRunnerManager.onAllocationAdded(alloc) _, ok = environment.Sample(s.apiMock) s.False(ok) }) s.Run("does not panic when environment id cannot be parsed", func() { alloc := &nomadApi.Allocation{JobID: ""} s.NotPanics(func() { s.nomadRunnerManager.onAllocationAdded(alloc) }) }) s.Run("does not panic when environment does not exist", func() { nonExistentEnvironment := dto.EnvironmentID(1234) _, ok := s.nomadRunnerManager.environments.Get(nonExistentEnvironment) s.Require().False(ok) alloc := &nomadApi.Allocation{JobID: nomad.RunnerJobID(nonExistentEnvironment, "1-1-1-1")} s.NotPanics(func() { s.nomadRunnerManager.onAllocationAdded(alloc) }) }) s.Run("adds correct job", func() { s.Run("without allocated resources", func() { environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsInteger) s.True(ok) mockIdleRunners(environment.(*ExecutionEnvironmentMock)) alloc := &nomadApi.Allocation{ JobID: tests.DefaultRunnerID, AllocatedResources: nil, } s.nomadRunnerManager.onAllocationAdded(alloc) runner, ok := environment.Sample(s.apiMock) s.True(ok) nomadJob, ok := runner.(*NomadJob) s.True(ok) s.Equal(nomadJob.id, tests.DefaultRunnerID) s.Empty(nomadJob.portMappings) }) s.Run("with mapped ports", func() { environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsInteger) s.True(ok) mockIdleRunners(environment.(*ExecutionEnvironmentMock)) alloc := &nomadApi.Allocation{ JobID: tests.DefaultRunnerID, AllocatedResources: &nomadApi.AllocatedResources{ Shared: nomadApi.AllocatedSharedResources{Ports: tests.DefaultPortMappings}, }, } s.nomadRunnerManager.onAllocationAdded(alloc) runner, ok := environment.Sample(s.apiMock) s.True(ok) nomadJob, ok := runner.(*NomadJob) s.True(ok) s.Equal(nomadJob.id, tests.DefaultRunnerID) s.Equal(nomadJob.portMappings, tests.DefaultPortMappings) }) }) }