poseidon/internal/runner/manager_test.go

package runner

import (
	"context"
	nomadApi "github.com/hashicorp/nomad/api"
	"github.com/openHPI/poseidon/internal/nomad"
	"github.com/openHPI/poseidon/pkg/dto"
	"github.com/openHPI/poseidon/tests"
	"github.com/sirupsen/logrus"
	"github.com/sirupsen/logrus/hooks/test"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/suite"
	"testing"
	"time"
)

func TestGetNextRunnerTestSuite(t *testing.T) {
	suite.Run(t, new(ManagerTestSuite))
}

type ManagerTestSuite struct {
	suite.Suite
	apiMock             *nomad.ExecutorAPIMock
	nomadRunnerManager  *NomadRunnerManager
	exerciseEnvironment *ExecutionEnvironmentMock
	exerciseRunner      Runner
}

func (s *ManagerTestSuite) SetupTest() {
	s.apiMock = &nomad.ExecutorAPIMock{}
	mockRunnerQueries(s.apiMock, []string{})
	// Instantly closed context to manually start the update process in some cases
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	s.nomadRunnerManager = NewNomadRunnerManager(s.apiMock, ctx)

	s.exerciseRunner = NewRunner(tests.DefaultRunnerID, s.nomadRunnerManager)
	s.exerciseEnvironment = &ExecutionEnvironmentMock{}
	s.setDefaultEnvironment()
}

func mockRunnerQueries(apiMock *nomad.ExecutorAPIMock, returnedRunnerIds []string) {
	// reset expected calls to allow new mocked return values
	apiMock.ExpectedCalls = []*mock.Call{}
	call := apiMock.On("WatchAllocations", mock.Anything, mock.Anything, mock.Anything)
	call.Run(func(args mock.Arguments) {
		<-time.After(10 * time.Minute) // 10 minutes is the default test timeout
		call.ReturnArguments = mock.Arguments{nil}
	})
	apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil)
	apiMock.On("MarkRunnerAsUsed", mock.AnythingOfType("string"), mock.AnythingOfType("int")).Return(nil)
	apiMock.On("LoadRunnerIDs", tests.DefaultRunnerID).Return(returnedRunnerIds, nil)
	apiMock.On("JobScale", tests.DefaultRunnerID).Return(uint(len(returnedRunnerIds)), nil)
	apiMock.On("SetJobScale", tests.DefaultRunnerID, mock.AnythingOfType("uint"), "Runner Requested").Return(nil)
	apiMock.On("RegisterRunnerJob", mock.Anything).Return(nil)
	apiMock.On("MonitorEvaluation", mock.Anything, mock.Anything).Return(nil)
}

func mockIdleRunners(environmentMock *ExecutionEnvironmentMock) {
	idleRunner := NewLocalRunnerStorage()
	environmentMock.On("AddRunner", mock.Anything).Run(func(args mock.Arguments) {
		r, ok := args.Get(0).(Runner)
		if !ok {
			return
		}
		idleRunner.Add(r)
	})
	sampleCall := environmentMock.On("Sample", mock.Anything)
	sampleCall.Run(func(args mock.Arguments) {
		r, ok := idleRunner.Sample()
		sampleCall.ReturnArguments = mock.Arguments{r, ok}
	})
	deleteCall := environmentMock.On("DeleteRunner", mock.AnythingOfType("string"))
	deleteCall.Run(func(args mock.Arguments) {
		id, ok := args.Get(0).(string)
		if !ok {
			return
		}
		idleRunner.Delete(id)
	})
}

func (s *ManagerTestSuite) setDefaultEnvironment() {
	s.exerciseEnvironment.On("ID").Return(defaultEnvironmentID)
	created := s.nomadRunnerManager.SetEnvironment(s.exerciseEnvironment)
	s.Require().True(created)
}

func (s *ManagerTestSuite) waitForRunnerRefresh() {
	<-time.After(100 * time.Millisecond)
}

func (s *ManagerTestSuite) TestSetEnvironmentAddsNewEnvironment() {
	anotherEnvironment := &ExecutionEnvironmentMock{}
	anotherEnvironment.On("ID").Return(anotherEnvironmentID)
	created := s.nomadRunnerManager.SetEnvironment(anotherEnvironment)
	s.Require().True(created)

	job, ok := s.nomadRunnerManager.environments.Get(anotherEnvironmentID)
	s.True(ok)
	s.NotNil(job)
}

func (s *ManagerTestSuite) TestClaimReturnsNotFoundErrorIfEnvironmentNotFound() {
	runner, err := s.nomadRunnerManager.Claim(anotherEnvironmentID, defaultInactivityTimeout)
	s.Nil(runner)
	s.Equal(ErrUnknownExecutionEnvironment, err)
}

func (s *ManagerTestSuite) TestClaimReturnsRunnerIfAvailable() {
	s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true)
	receivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout)
	s.NoError(err)
	s.Equal(s.exerciseRunner, receivedRunner)
}

func (s *ManagerTestSuite) TestClaimReturnsErrorIfNoRunnerAvailable() {
	s.waitForRunnerRefresh()
	s.exerciseEnvironment.On("Sample", mock.Anything).Return(nil, false)
	runner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout)
	s.Nil(runner)
	s.Equal(ErrNoRunnersAvailable, err)
}

func (s *ManagerTestSuite) TestClaimReturnsNoRunnerOfDifferentEnvironment() {
	s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true)
	receivedRunner, err := s.nomadRunnerManager.Claim(anotherEnvironmentID, defaultInactivityTimeout)
	s.Nil(receivedRunner)
	s.Error(err)
}

func (s *ManagerTestSuite) TestClaimDoesNotReturnTheSameRunnerTwice() {
	s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true).Once()
	s.exerciseEnvironment.On("Sample", mock.Anything).
		Return(NewRunner(tests.AnotherRunnerID, s.nomadRunnerManager), true).Once()

	firstReceivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout)
	s.NoError(err)
	secondReceivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout)
	s.NoError(err)
	s.NotEqual(firstReceivedRunner, secondReceivedRunner)
}

func (s *ManagerTestSuite) TestClaimAddsRunnerToUsedRunners() {
	s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true)
	receivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout)
	s.Require().NoError(err)
	savedRunner, ok := s.nomadRunnerManager.usedRunners.Get(receivedRunner.ID())
	s.True(ok)
	s.Equal(savedRunner, receivedRunner)
}

func (s *ManagerTestSuite) TestGetReturnsRunnerIfRunnerIsUsed() {
	s.nomadRunnerManager.usedRunners.Add(s.exerciseRunner)
	savedRunner, err := s.nomadRunnerManager.Get(s.exerciseRunner.ID())
	s.NoError(err)
	s.Equal(savedRunner, s.exerciseRunner)
}

func (s *ManagerTestSuite) TestGetReturnsErrorIfRunnerNotFound() {
	savedRunner, err := s.nomadRunnerManager.Get(tests.DefaultRunnerID)
	s.Nil(savedRunner)
	s.Error(err)
}

func (s *ManagerTestSuite) TestReturnRemovesRunnerFromUsedRunners() {
	s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil)
	s.nomadRunnerManager.usedRunners.Add(s.exerciseRunner)
	err := s.nomadRunnerManager.Return(s.exerciseRunner)
	s.Nil(err)
	_, ok := s.nomadRunnerManager.usedRunners.Get(s.exerciseRunner.ID())
	s.False(ok)
}

func (s *ManagerTestSuite) TestReturnCallsDeleteRunnerApiMethod() {
	s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil)
	err := s.nomadRunnerManager.Return(s.exerciseRunner)
	s.Nil(err)
	s.apiMock.AssertCalled(s.T(), "DeleteJob", s.exerciseRunner.ID())
}

func (s *ManagerTestSuite) TestReturnReturnsErrorWhenApiCallFailed() {
	s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(tests.ErrDefault)
	err := s.nomadRunnerManager.Return(s.exerciseRunner)
	s.Error(err)
}

func (s *ManagerTestSuite) TestUpdateRunnersLogsErrorFromWatchAllocation() {
	var hook *test.Hook
	logger, hook := test.NewNullLogger()
	log = logger.WithField("pkg", "runner")
	modifyMockedCall(s.apiMock, "WatchAllocations", func(call *mock.Call) {
		call.Run(func(args mock.Arguments) {
			call.ReturnArguments = mock.Arguments{tests.ErrDefault}
		})
	})

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go s.nomadRunnerManager.keepRunnersSynced(ctx)
	<-time.After(10 * time.Millisecond)

	s.Require().Equal(1, len(hook.Entries))
	s.Equal(logrus.ErrorLevel, hook.LastEntry().Level)
	s.Equal(hook.LastEntry().Data[logrus.ErrorKey], tests.ErrDefault)
}

func (s *ManagerTestSuite) TestUpdateRunnersAddsIdleRunner() {
	allocation := &nomadApi.Allocation{ID: tests.DefaultRunnerID}
	environment, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID)
	s.Require().True(ok)
	allocation.JobID = environment.ID().ToString()
	mockIdleRunners(environment.(*ExecutionEnvironmentMock))

	_, ok = environment.Sample(s.apiMock)
	s.Require().False(ok)

	modifyMockedCall(s.apiMock, "WatchAllocations", func(call *mock.Call) {
		call.Run(func(args mock.Arguments) {
			onCreate, ok := args.Get(1).(nomad.AllocationProcessor)
			s.Require().True(ok)
			onCreate(allocation)
			call.ReturnArguments = mock.Arguments{nil}
		})
	})

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go s.nomadRunnerManager.keepRunnersSynced(ctx)
	<-time.After(10 * time.Millisecond)

	_, ok = environment.Sample(s.apiMock)
	s.True(ok)
}

func (s *ManagerTestSuite) TestUpdateRunnersRemovesIdleAndUsedRunner() {
	allocation := &nomadApi.Allocation{JobID: tests.DefaultRunnerID}
	environment, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID)
	s.Require().True(ok)
	mockIdleRunners(environment.(*ExecutionEnvironmentMock))

	testRunner := NewRunner(allocation.JobID, s.nomadRunnerManager)
	environment.AddRunner(testRunner)
	s.nomadRunnerManager.usedRunners.Add(testRunner)

	modifyMockedCall(s.apiMock, "WatchAllocations", func(call *mock.Call) {
		call.Run(func(args mock.Arguments) {
			onDelete, ok := args.Get(2).(nomad.AllocationProcessor)
			s.Require().True(ok)
			onDelete(allocation)
			call.ReturnArguments = mock.Arguments{nil}
		})
	})

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go s.nomadRunnerManager.keepRunnersSynced(ctx)
	<-time.After(10 * time.Millisecond)

	_, ok = environment.Sample(s.apiMock)
	s.False(ok)
	_, ok = s.nomadRunnerManager.usedRunners.Get(allocation.JobID)
	s.False(ok)
}

func modifyMockedCall(apiMock *nomad.ExecutorAPIMock, method string, modifier func(call *mock.Call)) {
	for _, c := range apiMock.ExpectedCalls {
		if c.Method == method {
			modifier(c)
		}
	}
}

func (s *ManagerTestSuite) TestOnAllocationAdded() {
	s.Run("does not add environment template id job", func() {
		environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsInteger)
		s.True(ok)
		mockIdleRunners(environment.(*ExecutionEnvironmentMock))

		alloc := &nomadApi.Allocation{JobID: nomad.TemplateJobID(tests.DefaultEnvironmentIDAsInteger)}
		s.nomadRunnerManager.onAllocationAdded(alloc)

		_, ok = environment.Sample(s.apiMock)
		s.False(ok)
	})
	s.Run("does not panic when environment id cannot be parsed", func() {
		alloc := &nomadApi.Allocation{JobID: ""}
		s.NotPanics(func() {
			s.nomadRunnerManager.onAllocationAdded(alloc)
		})
	})
	s.Run("does not panic when environment does not exist", func() {
		nonExistentEnvironment := dto.EnvironmentID(1234)
		_, ok := s.nomadRunnerManager.environments.Get(nonExistentEnvironment)
		s.Require().False(ok)

		alloc := &nomadApi.Allocation{JobID: nomad.RunnerJobID(nonExistentEnvironment, "1-1-1-1")}
		s.NotPanics(func() {
			s.nomadRunnerManager.onAllocationAdded(alloc)
		})
	})
	s.Run("adds correct job", func() {
		s.Run("without allocated resources", func() {
			environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsInteger)
			s.True(ok)
			mockIdleRunners(environment.(*ExecutionEnvironmentMock))

			alloc := &nomadApi.Allocation{
				JobID:              tests.DefaultRunnerID,
				AllocatedResources: nil,
			}
			s.nomadRunnerManager.onAllocationAdded(alloc)

			runner, ok := environment.Sample(s.apiMock)
			s.True(ok)
			nomadJob, ok := runner.(*NomadJob)
			s.True(ok)
			s.Equal(nomadJob.id, tests.DefaultRunnerID)
			s.Empty(nomadJob.portMappings)
		})
		s.Run("with mapped ports", func() {
			environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsInteger)
			s.True(ok)
			mockIdleRunners(environment.(*ExecutionEnvironmentMock))

			alloc := &nomadApi.Allocation{
				JobID: tests.DefaultRunnerID,
				AllocatedResources: &nomadApi.AllocatedResources{
					Shared: nomadApi.AllocatedSharedResources{Ports: tests.DefaultPortMappings},
				},
			}
			s.nomadRunnerManager.onAllocationAdded(alloc)

			runner, ok := environment.Sample(s.apiMock)
			s.True(ok)
			nomadJob, ok := runner.(*NomadJob)
			s.True(ok)
			s.Equal(nomadJob.id, tests.DefaultRunnerID)
			s.Equal(nomadJob.portMappings, tests.DefaultPortMappings)
		})
	})
}