From 0c6c48c3cfb716eaf892329d3c4bd17493cdb7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Pa=C3=9F?= <22845248+mpass99@users.noreply.github.com> Date: Wed, 23 Nov 2022 17:49:03 +0000 Subject: [PATCH] #190 Add unit tests for runner recovery. --- internal/environment/nomad_manager.go | 10 +-- internal/environment/nomad_manager_test.go | 43 ++++++++++++ internal/runner/nomad_manager_test.go | 81 ++++++++++++++++++++++ tests/helpers/test_helpers.go | 3 + 4 files changed, 132 insertions(+), 5 deletions(-) diff --git a/internal/environment/nomad_manager.go b/internal/environment/nomad_manager.go index 30901e2..fd0cc7e 100644 --- a/internal/environment/nomad_manager.go +++ b/internal/environment/nomad_manager.go @@ -140,18 +140,18 @@ func (m *NomadEnvironmentManager) Load() error { } configTaskGroup := nomad.FindAndValidateConfigTaskGroup(job) if configTaskGroup == nil { - jobLogger.Info("Couldn't find config task group in job, skipping ...") + jobLogger.Error("FindAndValidateConfigTaskGroup is not creating the task group") continue } - environment := newNomadEnvironmetFromJob(job, m.api) + environment := newNomadEnvironmentFromJob(job, m.api) m.runnerManager.StoreEnvironment(environment) jobLogger.Info("Successfully recovered environment") } return nil } -// newNomadEnvironmetFromJob creates a Nomad environment from the passed Nomad job definition. -func newNomadEnvironmetFromJob(job *nomadApi.Job, apiClient nomad.ExecutorAPI) *NomadEnvironment { +// newNomadEnvironmentFromJob creates a Nomad environment from the passed Nomad job definition. +func newNomadEnvironmentFromJob(job *nomadApi.Job, apiClient nomad.ExecutorAPI) *NomadEnvironment { ctx, cancel := context.WithCancel(context.Background()) e := &NomadEnvironment{ apiClient: apiClient, @@ -192,7 +192,7 @@ func fetchEnvironment(id dto.EnvironmentID, apiClient nomad.ExecutorAPI) (runner continue } if id == environmentID { - fetchedEnvironment = newNomadEnvironmetFromJob(job, apiClient) + fetchedEnvironment = newNomadEnvironmentFromJob(job, apiClient) } } return fetchedEnvironment, nil diff --git a/internal/environment/nomad_manager_test.go b/internal/environment/nomad_manager_test.go index 61775d8..9dc8f3e 100644 --- a/internal/environment/nomad_manager_test.go +++ b/internal/environment/nomad_manager_test.go @@ -8,6 +8,7 @@ import ( "github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/tests" + "github.com/openHPI/poseidon/tests/helpers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -268,6 +269,48 @@ func TestNomadEnvironmentManager_List(t *testing.T) { }) } +func TestNomadEnvironmentManager_Load(t *testing.T) { + apiMock := &nomad.ExecutorAPIMock{} + mockWatchAllocations(apiMock) + call := apiMock.On("LoadEnvironmentJobs") + apiMock.On("LoadRunnerJobs", mock.AnythingOfType("dto.EnvironmentID")). + Return([]*nomadApi.Job{}, nil) + + runnerManager := runner.NewNomadRunnerManager(apiMock, context.Background()) + + t.Run("Stores fetched environments", func(t *testing.T) { + _, job := helpers.CreateTemplateJob() + call.Return([]*nomadApi.Job{job}, nil) + + _, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) + require.False(t, ok) + + _, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") + require.NoError(t, err) + + environment, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) + require.True(t, ok) + assert.Equal(t, "python:latest", environment.Image()) + }) + + runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger) + t.Run("Processes only running environments", func(t *testing.T) { + _, job := helpers.CreateTemplateJob() + jobStatus := structs.JobStatusDead + job.Status = &jobStatus + call.Return([]*nomadApi.Job{job}, nil) + + _, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) + require.False(t, ok) + + _, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") + require.NoError(t, err) + + _, ok = runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) + require.False(t, ok) + }) +} + func mockWatchAllocations(apiMock *nomad.ExecutorAPIMock) { call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything) call.Run(func(args mock.Arguments) { diff --git a/internal/runner/nomad_manager_test.go b/internal/runner/nomad_manager_test.go index 4fe3c63..2789dcf 100644 --- a/internal/runner/nomad_manager_test.go +++ b/internal/runner/nomad_manager_test.go @@ -8,10 +8,14 @@ import ( "github.com/openHPI/poseidon/pkg/storage" "github.com/openHPI/poseidon/pkg/util" "github.com/openHPI/poseidon/tests" + "github.com/openHPI/poseidon/tests/helpers" "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "strconv" "testing" "time" ) @@ -352,3 +356,80 @@ func (s *ManagerTestSuite) TestOnAllocationAdded() { }) }) } + +func TestNomadRunnerManager_Load(t *testing.T) { + apiMock := &nomad.ExecutorAPIMock{} + mockWatchAllocations(apiMock) + apiMock.On("LoadRunnerPortMappings", mock.AnythingOfType("string")). + Return([]nomadApi.PortMapping{}, nil) + call := apiMock.On("LoadRunnerJobs", dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) + runnerManager := NewNomadRunnerManager(apiMock, context.Background()) + environmentMock := createBasicEnvironmentMock(tests.DefaultEnvironmentIDAsInteger) + environmentMock.On("ApplyPrewarmingPoolSize").Return(nil) + runnerManager.StoreEnvironment(environmentMock) + + t.Run("Stores unused runner", func(t *testing.T) { + environmentMock.On("AddRunner", mock.AnythingOfType("*runner.NomadJob")).Once() + + _, job := helpers.CreateTemplateJob() + jobID := tests.DefaultRunnerID + job.ID = &jobID + job.Name = &jobID + call.Return([]*nomadApi.Job{job}, nil) + + runnerManager.Load() + + environmentMock.AssertExpectations(t) + }) + + t.Run("Stores used runner", func(t *testing.T) { + _, job := helpers.CreateTemplateJob() + jobID := tests.DefaultRunnerID + job.ID = &jobID + job.Name = &jobID + configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName) + require.NotNil(t, configTaskGroup) + configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue + call.Return([]*nomadApi.Job{job}, nil) + + require.Zero(t, runnerManager.usedRunners.Length()) + + runnerManager.Load() + + _, ok := runnerManager.usedRunners.Get(tests.DefaultRunnerID) + assert.True(t, ok) + }) + + runnerManager.usedRunners.Purge() + t.Run("Restart timeout of used runner", func(t *testing.T) { + apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) + timeout := 1 + + _, job := helpers.CreateTemplateJob() + jobID := tests.DefaultRunnerID + job.ID = &jobID + job.Name = &jobID + configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName) + require.NotNil(t, configTaskGroup) + configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue + configTaskGroup.Meta[nomad.ConfigMetaTimeoutKey] = strconv.Itoa(timeout) + call.Return([]*nomadApi.Job{job}, nil) + + require.Zero(t, runnerManager.usedRunners.Length()) + + runnerManager.Load() + + require.NotZero(t, runnerManager.usedRunners.Length()) + + <-time.After(time.Duration(timeout*2) * time.Second) + require.Zero(t, runnerManager.usedRunners.Length()) + }) +} + +func mockWatchAllocations(apiMock *nomad.ExecutorAPIMock) { + call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything) + call.Run(func(args mock.Arguments) { + <-time.After(tests.DefaultTestTimeout) + call.ReturnArguments = mock.Arguments{nil} + }) +} diff --git a/tests/helpers/test_helpers.go b/tests/helpers/test_helpers.go index 17c4349..a6aa370 100644 --- a/tests/helpers/test_helpers.go +++ b/tests/helpers/test_helpers.go @@ -10,6 +10,7 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/websocket" nomadApi "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/nomad/structs" "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/tests" @@ -206,6 +207,8 @@ func CreateTemplateJob() (base, job *nomadApi.Job) { base = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority) job = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority) job.Datacenters = []string{"dc1"} + jobStatus := structs.JobStatusRunning + job.Status = &jobStatus configTaskGroup := nomadApi.NewTaskGroup("config", 0) configTaskGroup.Meta = make(map[string]string) configTaskGroup.Meta["prewarmingPoolSize"] = "0"