Delete idle runners when the environment is scaled down

runner/manager.go

@@ -298,9 +298,16 @@ func (m *NomadRunnerManager) scaleEnvironment(id EnvironmentID) error {
 
 	required := int(environment.desiredIdleRunnersCount) - environment.idleRunners.Length()
 
-	log.WithField("runnersRequired", required).WithField("id", id).Debug("Scaling environment")
+	if required > 0 {
+		return m.createRunners(environment, uint(required))
+	} else {
+		return m.removeRunners(environment, uint(-required))
+	}
+}
 
-	for i := 0; i < required; i++ {
+func (m *NomadRunnerManager) createRunners(environment *NomadEnvironment, count uint) error {
+	log.WithField("runnersRequired", count).WithField("id", environment.ID()).Debug("Creating new runners")
+	for i := 0; i < int(count); i++ {
 		err := m.createRunner(environment)
 		if err != nil {
 			return fmt.Errorf("couldn't create new runner: %w", err)
@@ -323,6 +330,21 @@ func (m *NomadRunnerManager) createRunner(environment *NomadEnvironment) error {
 	return m.apiClient.RegisterRunnerJob(&template)
 }
 
+func (m *NomadRunnerManager) removeRunners(environment *NomadEnvironment, count uint) error {
+	log.WithField("runnersToDelete", count).WithField("id", environment.ID()).Debug("Removing idle runners")
+	for i := 0; i < int(count); i++ {
+		r, ok := environment.idleRunners.Sample()
+		if !ok {
+			return fmt.Errorf("could not delete expected idle runner: %w", ErrRunnerNotFound)
+		}
+		err := m.apiClient.DeleteRunner(r.Id())
+		if err != nil {
+			return fmt.Errorf("could not delete expected Nomad idle runner: %w", err)
+		}
+	}
+	return nil
+}
+
 // RunnerJobID returns the nomad job id of the runner with the given environment id and uuid.
 func RunnerJobID(environmentID EnvironmentID, uuid string) string {
 	return fmt.Sprintf("%d-%s", environmentID, uuid)

runner/manager_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/stretchr/testify/suite"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/helpers"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -254,6 +256,27 @@ func (s *ManagerTestSuite) TestUpdateRunnersRemovesIdleAndUsedRunner() {
 	s.False(ok)
 }
 
+func (s *ManagerTestSuite) TestUpdateEnvironmentRemovesIdleRunnersWhenScalingDown() {
+	job := helpers.CreateTestJob()
+	initialRunners := uint(40)
+	updatedRunners := uint(10)
+	err := s.nomadRunnerManager.registerEnvironment(anotherEnvironmentID, initialRunners, job, true)
+	s.Require().NoError(err)
+	s.apiMock.AssertNumberOfCalls(s.T(), "RegisterRunnerJob", int(initialRunners))
+	environment, ok := s.nomadRunnerManager.environments.Get(anotherEnvironmentID)
+	s.Require().True(ok)
+	for i := 0; i < int(initialRunners); i++ {
+		environment.idleRunners.Add(NewRunner("active-runner-"+strconv.Itoa(i), s.nomadRunnerManager))
+	}
+
+	s.apiMock.On("LoadRunnerIDs", anotherEnvironmentID.toString()).Return([]string{}, nil)
+	s.apiMock.On("DeleteRunner", mock.AnythingOfType("string")).Return(nil)
+
+	err = s.nomadRunnerManager.updateEnvironment(tests.AnotherEnvironmentIDAsInteger, updatedRunners, job, true)
+	s.Require().NoError(err)
+	s.apiMock.AssertNumberOfCalls(s.T(), "DeleteRunner", int(initialRunners-updatedRunners))
+}
+
 func modifyMockedCall(apiMock *nomad.ExecutorAPIMock, method string, modifier func(call *mock.Call)) {
 	for _, c := range apiMock.ExpectedCalls {
 		if c.Method == method {

tests/helpers/test_helpers.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/websocket"
+	nomadApi "github.com/hashicorp/nomad/api"
 	"github.com/stretchr/testify/mock"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
@@ -165,3 +166,30 @@ func HttpPutJSON(url string, body interface{}) (response *http.Response, err err
 	reader := bytes.NewReader(requestByteString)
 	return HttpPut(url, reader)
 }
+
+func CreateTestJob() (job *nomadApi.Job) {
+	job = nomadApi.NewBatchJob("template-0", "template-0", "region-name", 100)
+	configTaskGroup := nomadApi.NewTaskGroup("config", 0)
+	configTaskGroup.Meta = make(map[string]string)
+	configTaskGroup.Meta["environment"] = "0"
+	configTaskGroup.Meta["used"] = "false"
+	configTaskGroup.Meta["prewarmingPoolSize"] = "0"
+	configTask := nomadApi.NewTask("config", "exec")
+	configTask.Config = map[string]interface{}{"command": "whoami"}
+	configTask.Resources = nomadApi.DefaultResources()
+	configTaskGroup.AddTask(configTask)
+
+	defaultTaskGroup := nomadApi.NewTaskGroup("default-group", 1)
+	defaultTask := nomadApi.NewTask("default-task", "docker")
+	defaultTask.Config = map[string]interface{}{
+		"image":        "python:latest",
+		"command":      "sleep",
+		"args":         []string{"infinity"},
+		"network_mode": "none",
+	}
+	defaultTask.Resources = nomadApi.DefaultResources()
+	defaultTaskGroup.AddTask(defaultTask)
+
+	job.TaskGroups = []*nomadApi.TaskGroup{configTaskGroup, defaultTaskGroup}
+	return job
+}