htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#190 Add unit tests for runner recovery.

Browse Source
This commit is contained in:
Maximilian Paß
2022-11-23 17:49:03 +00:00
parent f9c877cc93
commit 0c6c48c3cf
4 changed files with 132 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/environment/nomad_manager.go
View File

@ -140,18 +140,18 @@ func (m *NomadEnvironmentManager) Load() error {
}
configTaskGroup := nomad.FindAndValidateConfigTaskGroup(job)
if configTaskGroup == nil {
jobLogger.Info("Couldn't find config task group in job, skipping ...")
jobLogger.Error("FindAndValidateConfigTaskGroup is not creating the task group")
continue
}
environment := newNomadEnvironmetFromJob(job, m.api)
environment := newNomadEnvironmentFromJob(job, m.api)
m.runnerManager.StoreEnvironment(environment)
jobLogger.Info("Successfully recovered environment")
}
return nil
}
// newNomadEnvironmetFromJob creates a Nomad environment from the passed Nomad job definition.
func newNomadEnvironmetFromJob(job *nomadApi.Job, apiClient nomad.ExecutorAPI) *NomadEnvironment {
// newNomadEnvironmentFromJob creates a Nomad environment from the passed Nomad job definition.
func newNomadEnvironmentFromJob(job *nomadApi.Job, apiClient nomad.ExecutorAPI) *NomadEnvironment {
ctx, cancel := context.WithCancel(context.Background())
e := &NomadEnvironment{
apiClient: apiClient,
@ -192,7 +192,7 @@ func fetchEnvironment(id dto.EnvironmentID, apiClient nomad.ExecutorAPI) (runner
continue
}
if id == environmentID {
fetchedEnvironment = newNomadEnvironmetFromJob(job, apiClient)
fetchedEnvironment = newNomadEnvironmentFromJob(job, apiClient)
}
}
return fetchedEnvironment, nil

43
internal/environment/nomad_manager_test.go
View File

@ -8,6 +8,7 @@ import (
"github.com/openHPI/poseidon/internal/runner"
"github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/tests"
"github.com/openHPI/poseidon/tests/helpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@ -268,6 +269,48 @@ func TestNomadEnvironmentManager_List(t *testing.T) {
})
}
func TestNomadEnvironmentManager_Load(t *testing.T) {
apiMock := &nomad.ExecutorAPIMock{}
mockWatchAllocations(apiMock)
call := apiMock.On("LoadEnvironmentJobs")
apiMock.On("LoadRunnerJobs", mock.AnythingOfType("dto.EnvironmentID")).
Return([]*nomadApi.Job{}, nil)
runnerManager := runner.NewNomadRunnerManager(apiMock, context.Background())
t.Run("Stores fetched environments", func(t *testing.T) {
_, job := helpers.CreateTemplateJob()
call.Return([]*nomadApi.Job{job}, nil)
_, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
require.False(t, ok)
_, err := NewNomadEnvironmentManager(runnerManager, apiMock, "")
require.NoError(t, err)
environment, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
require.True(t, ok)
assert.Equal(t, "python:latest", environment.Image())
})
runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger)
t.Run("Processes only running environments", func(t *testing.T) {
_, job := helpers.CreateTemplateJob()
jobStatus := structs.JobStatusDead
job.Status = &jobStatus
call.Return([]*nomadApi.Job{job}, nil)
_, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
require.False(t, ok)
_, err := NewNomadEnvironmentManager(runnerManager, apiMock, "")
require.NoError(t, err)
_, ok = runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
require.False(t, ok)
})
}
func mockWatchAllocations(apiMock *nomad.ExecutorAPIMock) {
call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything)
call.Run(func(args mock.Arguments) {

81
internal/runner/nomad_manager_test.go
View File

@ -8,10 +8,14 @@ import (
"github.com/openHPI/poseidon/pkg/storage"
"github.com/openHPI/poseidon/pkg/util"
"github.com/openHPI/poseidon/tests"
"github.com/openHPI/poseidon/tests/helpers"
"github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"strconv"
"testing"
"time"
)
@ -352,3 +356,80 @@ func (s *ManagerTestSuite) TestOnAllocationAdded() {
})
})
}
func TestNomadRunnerManager_Load(t *testing.T) {
apiMock := &nomad.ExecutorAPIMock{}
mockWatchAllocations(apiMock)
apiMock.On("LoadRunnerPortMappings", mock.AnythingOfType("string")).
Return([]nomadApi.PortMapping{}, nil)
call := apiMock.On("LoadRunnerJobs", dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger))
runnerManager := NewNomadRunnerManager(apiMock, context.Background())
environmentMock := createBasicEnvironmentMock(tests.DefaultEnvironmentIDAsInteger)
environmentMock.On("ApplyPrewarmingPoolSize").Return(nil)
runnerManager.StoreEnvironment(environmentMock)
t.Run("Stores unused runner", func(t *testing.T) {
environmentMock.On("AddRunner", mock.AnythingOfType("*runner.NomadJob")).Once()
_, job := helpers.CreateTemplateJob()
jobID := tests.DefaultRunnerID
job.ID = &jobID
job.Name = &jobID
call.Return([]*nomadApi.Job{job}, nil)
runnerManager.Load()
environmentMock.AssertExpectations(t)
})
t.Run("Stores used runner", func(t *testing.T) {
_, job := helpers.CreateTemplateJob()
jobID := tests.DefaultRunnerID
job.ID = &jobID
job.Name = &jobID
configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName)
require.NotNil(t, configTaskGroup)
configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue
call.Return([]*nomadApi.Job{job}, nil)
require.Zero(t, runnerManager.usedRunners.Length())
runnerManager.Load()
_, ok := runnerManager.usedRunners.Get(tests.DefaultRunnerID)
assert.True(t, ok)
})
runnerManager.usedRunners.Purge()
t.Run("Restart timeout of used runner", func(t *testing.T) {
apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil)
timeout := 1
_, job := helpers.CreateTemplateJob()
jobID := tests.DefaultRunnerID
job.ID = &jobID
job.Name = &jobID
configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName)
require.NotNil(t, configTaskGroup)
configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue
configTaskGroup.Meta[nomad.ConfigMetaTimeoutKey] = strconv.Itoa(timeout)
call.Return([]*nomadApi.Job{job}, nil)
require.Zero(t, runnerManager.usedRunners.Length())
runnerManager.Load()
require.NotZero(t, runnerManager.usedRunners.Length())
<-time.After(time.Duration(timeout*2) * time.Second)
require.Zero(t, runnerManager.usedRunners.Length())
})
}
func mockWatchAllocations(apiMock *nomad.ExecutorAPIMock) {
call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything)
call.Run(func(args mock.Arguments) {
<-time.After(tests.DefaultTestTimeout)
call.ReturnArguments = mock.Arguments{nil}
})
}

3
tests/helpers/test_helpers.go
View File

@ -10,6 +10,7 @@ import (
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
nomadApi "github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/openHPI/poseidon/internal/config"
"github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/tests"
@ -206,6 +207,8 @@ func CreateTemplateJob() (base, job *nomadApi.Job) {
base = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority)
job = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority)
job.Datacenters = []string{"dc1"}
jobStatus := structs.JobStatusRunning
job.Status = &jobStatus
configTaskGroup := nomadApi.NewTaskGroup("config", 0)
configTaskGroup.Meta = make(map[string]string)
configTaskGroup.Meta["prewarmingPoolSize"] = "0"