#190 Add unit tests for runner recovery.

internal/environment/nomad_manager.go

@@ -140,18 +140,18 @@ func (m *NomadEnvironmentManager) Load() error {
 		}
 		configTaskGroup := nomad.FindAndValidateConfigTaskGroup(job)
 		if configTaskGroup == nil {
-			jobLogger.Info("Couldn't find config task group in job, skipping ...")
+			jobLogger.Error("FindAndValidateConfigTaskGroup is not creating the task group")
 			continue
 		}
-		environment := newNomadEnvironmetFromJob(job, m.api)
+		environment := newNomadEnvironmentFromJob(job, m.api)
 		m.runnerManager.StoreEnvironment(environment)
 		jobLogger.Info("Successfully recovered environment")
 	}
 	return nil
 }
 
-// newNomadEnvironmetFromJob creates a Nomad environment from the passed Nomad job definition.
+// newNomadEnvironmentFromJob creates a Nomad environment from the passed Nomad job definition.
-func newNomadEnvironmetFromJob(job *nomadApi.Job, apiClient nomad.ExecutorAPI) *NomadEnvironment {
+func newNomadEnvironmentFromJob(job *nomadApi.Job, apiClient nomad.ExecutorAPI) *NomadEnvironment {
 	ctx, cancel := context.WithCancel(context.Background())
 	e := &NomadEnvironment{
 		apiClient: apiClient,
@@ -192,7 +192,7 @@ func fetchEnvironment(id dto.EnvironmentID, apiClient nomad.ExecutorAPI) (runner
 			continue
 		}
 		if id == environmentID {
-			fetchedEnvironment = newNomadEnvironmetFromJob(job, apiClient)
+			fetchedEnvironment = newNomadEnvironmentFromJob(job, apiClient)
 		}
 	}
 	return fetchedEnvironment, nil

internal/environment/nomad_manager_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/openHPI/poseidon/internal/runner"
 	"github.com/openHPI/poseidon/pkg/dto"
 	"github.com/openHPI/poseidon/tests"
+	"github.com/openHPI/poseidon/tests/helpers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -268,6 +269,48 @@ func TestNomadEnvironmentManager_List(t *testing.T) {
 	})
 }
 
+func TestNomadEnvironmentManager_Load(t *testing.T) {
+	apiMock := &nomad.ExecutorAPIMock{}
+	mockWatchAllocations(apiMock)
+	call := apiMock.On("LoadEnvironmentJobs")
+	apiMock.On("LoadRunnerJobs", mock.AnythingOfType("dto.EnvironmentID")).
+		Return([]*nomadApi.Job{}, nil)
+
+	runnerManager := runner.NewNomadRunnerManager(apiMock, context.Background())
+
+	t.Run("Stores fetched environments", func(t *testing.T) {
+		_, job := helpers.CreateTemplateJob()
+		call.Return([]*nomadApi.Job{job}, nil)
+
+		_, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
+		require.False(t, ok)
+
+		_, err := NewNomadEnvironmentManager(runnerManager, apiMock, "")
+		require.NoError(t, err)
+
+		environment, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
+		require.True(t, ok)
+		assert.Equal(t, "python:latest", environment.Image())
+	})
+
+	runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger)
+	t.Run("Processes only running environments", func(t *testing.T) {
+		_, job := helpers.CreateTemplateJob()
+		jobStatus := structs.JobStatusDead
+		job.Status = &jobStatus
+		call.Return([]*nomadApi.Job{job}, nil)
+
+		_, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
+		require.False(t, ok)
+
+		_, err := NewNomadEnvironmentManager(runnerManager, apiMock, "")
+		require.NoError(t, err)
+
+		_, ok = runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger)
+		require.False(t, ok)
+	})
+}
+
 func mockWatchAllocations(apiMock *nomad.ExecutorAPIMock) {
 	call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything)
 	call.Run(func(args mock.Arguments) {

internal/runner/nomad_manager_test.go

@@ -8,10 +8,14 @@ import (
 	"github.com/openHPI/poseidon/pkg/storage"
 	"github.com/openHPI/poseidon/pkg/util"
 	"github.com/openHPI/poseidon/tests"
+	"github.com/openHPI/poseidon/tests/helpers"
 	"github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -352,3 +356,80 @@ func (s *ManagerTestSuite) TestOnAllocationAdded() {
 		})
 	})
 }
+
+func TestNomadRunnerManager_Load(t *testing.T) {
+	apiMock := &nomad.ExecutorAPIMock{}
+	mockWatchAllocations(apiMock)
+	apiMock.On("LoadRunnerPortMappings", mock.AnythingOfType("string")).
+		Return([]nomadApi.PortMapping{}, nil)
+	call := apiMock.On("LoadRunnerJobs", dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger))
+	runnerManager := NewNomadRunnerManager(apiMock, context.Background())
+	environmentMock := createBasicEnvironmentMock(tests.DefaultEnvironmentIDAsInteger)
+	environmentMock.On("ApplyPrewarmingPoolSize").Return(nil)
+	runnerManager.StoreEnvironment(environmentMock)
+
+	t.Run("Stores unused runner", func(t *testing.T) {
+		environmentMock.On("AddRunner", mock.AnythingOfType("*runner.NomadJob")).Once()
+
+		_, job := helpers.CreateTemplateJob()
+		jobID := tests.DefaultRunnerID
+		job.ID = &jobID
+		job.Name = &jobID
+		call.Return([]*nomadApi.Job{job}, nil)
+
+		runnerManager.Load()
+
+		environmentMock.AssertExpectations(t)
+	})
+
+	t.Run("Stores used runner", func(t *testing.T) {
+		_, job := helpers.CreateTemplateJob()
+		jobID := tests.DefaultRunnerID
+		job.ID = &jobID
+		job.Name = &jobID
+		configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName)
+		require.NotNil(t, configTaskGroup)
+		configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue
+		call.Return([]*nomadApi.Job{job}, nil)
+
+		require.Zero(t, runnerManager.usedRunners.Length())
+
+		runnerManager.Load()
+
+		_, ok := runnerManager.usedRunners.Get(tests.DefaultRunnerID)
+		assert.True(t, ok)
+	})
+
+	runnerManager.usedRunners.Purge()
+	t.Run("Restart timeout of used runner", func(t *testing.T) {
+		apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil)
+		timeout := 1
+
+		_, job := helpers.CreateTemplateJob()
+		jobID := tests.DefaultRunnerID
+		job.ID = &jobID
+		job.Name = &jobID
+		configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName)
+		require.NotNil(t, configTaskGroup)
+		configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue
+		configTaskGroup.Meta[nomad.ConfigMetaTimeoutKey] = strconv.Itoa(timeout)
+		call.Return([]*nomadApi.Job{job}, nil)
+
+		require.Zero(t, runnerManager.usedRunners.Length())
+
+		runnerManager.Load()
+
+		require.NotZero(t, runnerManager.usedRunners.Length())
+
+		<-time.After(time.Duration(timeout*2) * time.Second)
+		require.Zero(t, runnerManager.usedRunners.Length())
+	})
+}
+
+func mockWatchAllocations(apiMock *nomad.ExecutorAPIMock) {
+	call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything)
+	call.Run(func(args mock.Arguments) {
+		<-time.After(tests.DefaultTestTimeout)
+		call.ReturnArguments = mock.Arguments{nil}
+	})
+}

tests/helpers/test_helpers.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/gorilla/websocket"
 	nomadApi "github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/openHPI/poseidon/internal/config"
 	"github.com/openHPI/poseidon/pkg/dto"
 	"github.com/openHPI/poseidon/tests"
@@ -206,6 +207,8 @@ func CreateTemplateJob() (base, job *nomadApi.Job) {
 	base = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority)
 	job = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority)
 	job.Datacenters = []string{"dc1"}
+	jobStatus := structs.JobStatusRunning
+	job.Status = &jobStatus
 	configTaskGroup := nomadApi.NewTaskGroup("config", 0)
 	configTaskGroup.Meta = make(map[string]string)
 	configTaskGroup.Meta["prewarmingPoolSize"] = "0"