Restore existing jobs and fix rebase (7c99eff3) issues
This commit is contained in:
@ -52,6 +52,7 @@ func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Req
|
|||||||
if err == runner.ErrUnknownExecutionEnvironment {
|
if err == runner.ErrUnknownExecutionEnvironment {
|
||||||
writeNotFound(writer, err)
|
writeNotFound(writer, err)
|
||||||
} else if err == runner.ErrNoRunnersAvailable {
|
} else if err == runner.ErrNoRunnersAvailable {
|
||||||
|
log.WithField("environment", environmentId).Warn("No runners available")
|
||||||
writeInternalServerError(writer, err, dto.ErrorNomadOverload)
|
writeInternalServerError(writer, err, dto.ErrorNomadOverload)
|
||||||
} else {
|
} else {
|
||||||
writeInternalServerError(writer, err, dto.ErrorUnknown)
|
writeInternalServerError(writer, err, dto.ErrorUnknown)
|
||||||
|
@ -128,9 +128,7 @@ type webSocketProxy struct {
|
|||||||
|
|
||||||
// upgradeConnection upgrades a connection to a websocket and returns a webSocketProxy for this connection.
|
// upgradeConnection upgrades a connection to a websocket and returns a webSocketProxy for this connection.
|
||||||
func upgradeConnection(writer http.ResponseWriter, request *http.Request) (webSocketConnection, error) {
|
func upgradeConnection(writer http.ResponseWriter, request *http.Request) (webSocketConnection, error) {
|
||||||
connUpgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool {
|
connUpgrader := websocket.Upgrader{}
|
||||||
return true
|
|
||||||
}}
|
|
||||||
connection, err := connUpgrader.Upgrade(writer, request, nil)
|
connection, err := connUpgrader.Upgrade(writer, request, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.WithError(err).Warn("Connection upgrade failed")
|
log.WithError(err).Warn("Connection upgrade failed")
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
// This job is used by the e2e tests as a demo job.
|
// This job is used by the e2e tests as a demo job.
|
||||||
|
|
||||||
job "python" {
|
job "0-default" {
|
||||||
datacenters = ["dc1"]
|
datacenters = ["dc1"]
|
||||||
type = "batch"
|
type = "batch"
|
||||||
namespace = "${NOMAD_NAMESPACE}"
|
namespace = "${NOMAD_NAMESPACE}"
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
// This is the default job configuration that is used when no path to another default configuration is given
|
// This is the default job configuration that is used when no path to another default configuration is given
|
||||||
|
|
||||||
job "python" {
|
job "0-default" {
|
||||||
datacenters = ["dc1"]
|
datacenters = ["dc1"]
|
||||||
type = "batch"
|
type = "batch"
|
||||||
|
|
||||||
@ -50,4 +50,31 @@ job "python" {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
group "config" {
|
||||||
|
// We want to store whether a task is in use in order to recover from a downtime.
|
||||||
|
// Without a separate config task, marking a task as used would result in a restart of that task,
|
||||||
|
// as the meta information is passed to the container as environment variables.
|
||||||
|
count = 0
|
||||||
|
task "config" {
|
||||||
|
driver = "exec"
|
||||||
|
config {
|
||||||
|
command = "whoami"
|
||||||
|
}
|
||||||
|
logs {
|
||||||
|
max_files = 1
|
||||||
|
max_file_size = 1
|
||||||
|
}
|
||||||
|
resources {
|
||||||
|
// minimum values
|
||||||
|
cpu = 1
|
||||||
|
memory = 10
|
||||||
|
}
|
||||||
|
}
|
||||||
|
meta {
|
||||||
|
environment = "0"
|
||||||
|
used = "false"
|
||||||
|
prewarmingPoolSize = "1"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -9,10 +9,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
DefaultTaskDriver = "docker"
|
|
||||||
)
|
|
||||||
|
|
||||||
// defaultJobHCL holds our default job in HCL format.
|
// defaultJobHCL holds our default job in HCL format.
|
||||||
// The default job is used when creating new job and provides
|
// The default job is used when creating new job and provides
|
||||||
// common settings that all the jobs share.
|
// common settings that all the jobs share.
|
||||||
@ -22,13 +18,13 @@ var defaultJobHCL string
|
|||||||
// registerDefaultJob creates a Nomad job based on the default job configuration and the given parameters.
|
// registerDefaultJob creates a Nomad job based on the default job configuration and the given parameters.
|
||||||
// It registers the job with Nomad and waits until the registration completes.
|
// It registers the job with Nomad and waits until the registration completes.
|
||||||
func (m *NomadEnvironmentManager) registerDefaultJob(
|
func (m *NomadEnvironmentManager) registerDefaultJob(
|
||||||
id string,
|
environmentID string,
|
||||||
prewarmingPoolSize, cpuLimit, memoryLimit uint,
|
prewarmingPoolSize, cpuLimit, memoryLimit uint,
|
||||||
image string,
|
image string,
|
||||||
networkAccess bool,
|
networkAccess bool,
|
||||||
exposedPorts []uint16) error {
|
exposedPorts []uint16) error {
|
||||||
// TODO: store prewarming pool size in job meta information
|
job := createDefaultJob(m.defaultJob, environmentID, prewarmingPoolSize,
|
||||||
job := createJob(m.defaultJob, nomad.DefaultJobID(id), prewarmingPoolSize, cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
|
cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
|
||||||
evalID, err := m.api.RegisterNomadJob(job)
|
evalID, err := m.api.RegisterNomadJob(job)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -36,20 +32,21 @@ func (m *NomadEnvironmentManager) registerDefaultJob(
|
|||||||
return m.api.MonitorEvaluation(evalID, context.Background())
|
return m.api.MonitorEvaluation(evalID, context.Background())
|
||||||
}
|
}
|
||||||
|
|
||||||
func createJob(
|
func createDefaultJob(
|
||||||
defaultJob nomadApi.Job,
|
defaultJob nomadApi.Job,
|
||||||
id string,
|
environmentID string,
|
||||||
prewarmingPoolSize, cpuLimit, memoryLimit uint,
|
prewarmingPoolSize, cpuLimit, memoryLimit uint,
|
||||||
image string,
|
image string,
|
||||||
networkAccess bool,
|
networkAccess bool,
|
||||||
exposedPorts []uint16) *nomadApi.Job {
|
exposedPorts []uint16) *nomadApi.Job {
|
||||||
|
|
||||||
job := defaultJob
|
job := defaultJob
|
||||||
job.ID = &id
|
defaultJobID := nomad.DefaultJobID(environmentID)
|
||||||
job.Name = &id
|
job.ID = &defaultJobID
|
||||||
|
job.Name = &defaultJobID
|
||||||
|
|
||||||
var taskGroup = createTaskGroup(&job, nomad.TaskGroupName, prewarmingPoolSize)
|
var taskGroup = createTaskGroup(&job, nomad.TaskGroupName, prewarmingPoolSize)
|
||||||
configureTask(taskGroup, nomad.TaskName, cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
|
configureTask(taskGroup, nomad.TaskName, cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
|
||||||
|
storeConfiguration(&job, environmentID, prewarmingPoolSize)
|
||||||
|
|
||||||
return &job
|
return &job
|
||||||
}
|
}
|
||||||
@ -137,16 +134,16 @@ func configureTask(
|
|||||||
exposedPorts []uint16) {
|
exposedPorts []uint16) {
|
||||||
var task *nomadApi.Task
|
var task *nomadApi.Task
|
||||||
if len(taskGroup.Tasks) == 0 {
|
if len(taskGroup.Tasks) == 0 {
|
||||||
task = nomadApi.NewTask(name, DefaultTaskDriver)
|
task = nomadApi.NewTask(name, nomad.DefaultTaskDriver)
|
||||||
taskGroup.Tasks = []*nomadApi.Task{task}
|
taskGroup.Tasks = []*nomadApi.Task{task}
|
||||||
} else {
|
} else {
|
||||||
task = taskGroup.Tasks[0]
|
task = taskGroup.Tasks[0]
|
||||||
task.Name = name
|
task.Name = name
|
||||||
}
|
}
|
||||||
integerCpuLimit := int(cpuLimit)
|
integerCPULimit := int(cpuLimit)
|
||||||
integerMemoryLimit := int(memoryLimit)
|
integerMemoryLimit := int(memoryLimit)
|
||||||
task.Resources = &nomadApi.Resources{
|
task.Resources = &nomadApi.Resources{
|
||||||
CPU: &integerCpuLimit,
|
CPU: &integerCPULimit,
|
||||||
MemoryMB: &integerMemoryLimit,
|
MemoryMB: &integerMemoryLimit,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -157,3 +154,44 @@ func configureTask(
|
|||||||
|
|
||||||
configureNetwork(taskGroup, networkAccess, exposedPorts)
|
configureNetwork(taskGroup, networkAccess, exposedPorts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func storeConfiguration(job *nomadApi.Job, id string, prewarmingPoolSize uint) {
|
||||||
|
taskGroup := getConfigTaskGroup(job)
|
||||||
|
checkForDummyTask(taskGroup)
|
||||||
|
|
||||||
|
if taskGroup.Meta == nil {
|
||||||
|
taskGroup.Meta = make(map[string]string)
|
||||||
|
}
|
||||||
|
taskGroup.Meta[nomad.ConfigMetaEnvironmentKey] = id
|
||||||
|
taskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUnusedValue
|
||||||
|
taskGroup.Meta[nomad.ConfigMetaPoolSizeKey] = strconv.Itoa(int(prewarmingPoolSize))
|
||||||
|
}
|
||||||
|
|
||||||
|
func getConfigTaskGroup(job *nomadApi.Job) *nomadApi.TaskGroup {
|
||||||
|
taskGroup := nomad.FindConfigTaskGroup(job)
|
||||||
|
if taskGroup == nil {
|
||||||
|
taskGroup = nomadApi.NewTaskGroup(nomad.ConfigTaskName, 0)
|
||||||
|
}
|
||||||
|
return taskGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkForDummyTask ensures that a dummy task is in the task group so that the group is accepted by Nomad.
|
||||||
|
func checkForDummyTask(taskGroup *nomadApi.TaskGroup) {
|
||||||
|
var task *nomadApi.Task
|
||||||
|
for _, t := range taskGroup.Tasks {
|
||||||
|
if t.Name == nomad.ConfigTaskName {
|
||||||
|
task = t
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if task == nil {
|
||||||
|
task = nomadApi.NewTask(nomad.ConfigTaskName, nomad.DefaultConfigTaskDriver)
|
||||||
|
taskGroup.Tasks = append(taskGroup.Tasks, task)
|
||||||
|
}
|
||||||
|
|
||||||
|
if task.Config == nil {
|
||||||
|
task.Config = make(map[string]interface{})
|
||||||
|
}
|
||||||
|
task.Config["command"] = nomad.DefaultConfigTaskCommand
|
||||||
|
}
|
||||||
|
@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
|
||||||
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -52,8 +53,9 @@ func createTestResources() *nomadApi.Resources {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func createTestJob() (*nomadApi.Job, *nomadApi.Job) {
|
func createTestJob() (*nomadApi.Job, *nomadApi.Job) {
|
||||||
base := nomadApi.NewBatchJob("python-job", "python-job", "region-name", 100)
|
jobID := nomad.DefaultJobID(tests.DefaultEnvironmentIDAsString)
|
||||||
job := nomadApi.NewBatchJob("python-job", "python-job", "region-name", 100)
|
base := nomadApi.NewBatchJob(jobID, jobID, "region-name", 100)
|
||||||
|
job := nomadApi.NewBatchJob(jobID, jobID, "region-name", 100)
|
||||||
task := createTestTask()
|
task := createTestTask()
|
||||||
task.Name = nomad.TaskName
|
task.Name = nomad.TaskName
|
||||||
image := "python:latest"
|
image := "python:latest"
|
||||||
@ -204,7 +206,7 @@ func TestConfigureTaskWhenNoTaskExists(t *testing.T) {
|
|||||||
|
|
||||||
expectedResources := createTestResources()
|
expectedResources := createTestResources()
|
||||||
expectedTaskGroup := *taskGroup
|
expectedTaskGroup := *taskGroup
|
||||||
expectedTask := nomadApi.NewTask("task", DefaultTaskDriver)
|
expectedTask := nomadApi.NewTask("task", nomad.DefaultTaskDriver)
|
||||||
expectedTask.Resources = expectedResources
|
expectedTask.Resources = expectedResources
|
||||||
expectedImage := "python:latest"
|
expectedImage := "python:latest"
|
||||||
expectedTask.Config = map[string]interface{}{"image": expectedImage, "network_mode": "none"}
|
expectedTask.Config = map[string]interface{}{"image": expectedImage, "network_mode": "none"}
|
||||||
@ -246,9 +248,9 @@ func TestConfigureTaskWhenTaskExists(t *testing.T) {
|
|||||||
func TestCreateJobSetsAllGivenArguments(t *testing.T) {
|
func TestCreateJobSetsAllGivenArguments(t *testing.T) {
|
||||||
testJob, base := createTestJob()
|
testJob, base := createTestJob()
|
||||||
manager := NomadEnvironmentManager{&runner.NomadRunnerManager{}, &nomad.APIClient{}, *base}
|
manager := NomadEnvironmentManager{&runner.NomadRunnerManager{}, &nomad.APIClient{}, *base}
|
||||||
job := createJob(
|
job := createDefaultJob(
|
||||||
manager.defaultJob,
|
manager.defaultJob,
|
||||||
*testJob.ID,
|
tests.DefaultEnvironmentIDAsString,
|
||||||
uint(*testJob.TaskGroups[0].Count),
|
uint(*testJob.TaskGroups[0].Count),
|
||||||
uint(*testJob.TaskGroups[0].Tasks[0].Resources.CPU),
|
uint(*testJob.TaskGroups[0].Tasks[0].Resources.CPU),
|
||||||
uint(*testJob.TaskGroups[0].Tasks[0].Resources.MemoryMB),
|
uint(*testJob.TaskGroups[0].Tasks[0].Resources.MemoryMB),
|
||||||
|
@ -13,10 +13,6 @@ var log = logging.GetLogger("environment")
|
|||||||
|
|
||||||
// Manager encapsulates API calls to the executor API for creation and deletion of execution environments.
|
// Manager encapsulates API calls to the executor API for creation and deletion of execution environments.
|
||||||
type Manager interface {
|
type Manager interface {
|
||||||
// Load fetches all already created execution environments from the executor and registers them at the runner manager.
|
|
||||||
// It should be called during the startup process (e.g. on creation of the Manager).
|
|
||||||
Load()
|
|
||||||
|
|
||||||
// CreateOrUpdate creates/updates an execution environment on the executor.
|
// CreateOrUpdate creates/updates an execution environment on the executor.
|
||||||
// Iff the job was created, the returned boolean is true and the returned error is nil.
|
// Iff the job was created, the returned boolean is true and the returned error is nil.
|
||||||
CreateOrUpdate(
|
CreateOrUpdate(
|
||||||
@ -30,7 +26,6 @@ type Manager interface {
|
|||||||
|
|
||||||
func NewNomadEnvironmentManager(runnerManager runner.Manager, apiClient nomad.ExecutorAPI) *NomadEnvironmentManager {
|
func NewNomadEnvironmentManager(runnerManager runner.Manager, apiClient nomad.ExecutorAPI) *NomadEnvironmentManager {
|
||||||
environmentManager := &NomadEnvironmentManager{runnerManager, apiClient, *parseJob(defaultJobHCL)}
|
environmentManager := &NomadEnvironmentManager{runnerManager, apiClient, *parseJob(defaultJobHCL)}
|
||||||
environmentManager.Load()
|
|
||||||
return environmentManager
|
return environmentManager
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -66,11 +61,3 @@ func (m *NomadEnvironmentManager) CreateOrUpdate(
|
|||||||
func (m *NomadEnvironmentManager) Delete(id string) {
|
func (m *NomadEnvironmentManager) Delete(id string) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *NomadEnvironmentManager) Load() {
|
|
||||||
// ToDo: remove create default execution environment for debugging purposes
|
|
||||||
_, err := m.runnerManager.CreateOrUpdateEnvironment(runner.EnvironmentID(0), 5)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
@ -60,7 +60,7 @@ func (s *CreateOrUpdateTestSuite) mockCreateOrUpdateEnvironment(exists bool) *mo
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *CreateOrUpdateTestSuite) createJobForRequest() *nomadApi.Job {
|
func (s *CreateOrUpdateTestSuite) createJobForRequest() *nomadApi.Job {
|
||||||
return createJob(s.manager.defaultJob, nomad.DefaultJobID(tests.DefaultEnvironmentIdAsString),
|
return createDefaultJob(s.manager.defaultJob, tests.DefaultEnvironmentIDAsString,
|
||||||
s.request.PrewarmingPoolSize, s.request.CPULimit, s.request.MemoryLimit,
|
s.request.PrewarmingPoolSize, s.request.CPULimit, s.request.MemoryLimit,
|
||||||
s.request.Image, s.request.NetworkAccess, s.request.ExposedPorts)
|
s.request.Image, s.request.NetworkAccess, s.request.ExposedPorts)
|
||||||
}
|
}
|
||||||
@ -121,7 +121,7 @@ func (s *CreateOrUpdateTestSuite) TestWhenEnvironmentDoesNotExistRegistersCorrec
|
|||||||
s.True(created)
|
s.True(created)
|
||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
s.runnerManagerMock.AssertCalled(s.T(), "CreateOrUpdateEnvironment",
|
s.runnerManagerMock.AssertCalled(s.T(), "CreateOrUpdateEnvironment",
|
||||||
runner.EnvironmentID(tests.DefaultEnvironmentIdAsInteger), s.request.PrewarmingPoolSize)
|
runner.EnvironmentID(tests.DefaultEnvironmentIDAsInteger), s.request.PrewarmingPoolSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *CreateOrUpdateTestSuite) TestWhenEnvironmentDoesNotExistOccurredErrorIsPassedAndNoEnvironmentRegistered() {
|
func (s *CreateOrUpdateTestSuite) TestWhenEnvironmentDoesNotExistOccurredErrorIsPassedAndNoEnvironmentRegistered() {
|
||||||
@ -130,5 +130,5 @@ func (s *CreateOrUpdateTestSuite) TestWhenEnvironmentDoesNotExistOccurredErrorIs
|
|||||||
s.registerNomadJobMockCall.Return("", tests.ErrDefault)
|
s.registerNomadJobMockCall.Return("", tests.ErrDefault)
|
||||||
created, err := s.manager.CreateOrUpdate(tests.DefaultEnvironmentIDAsString, s.request)
|
created, err := s.manager.CreateOrUpdate(tests.DefaultEnvironmentIDAsString, s.request)
|
||||||
s.False(created)
|
s.False(created)
|
||||||
s.Equal(tests.DefaultError, err)
|
s.Equal(tests.ErrDefault, err)
|
||||||
}
|
}
|
||||||
|
5
main.go
5
main.go
@ -46,7 +46,10 @@ func initServer() *http.Server {
|
|||||||
log.WithError(err).WithField("nomad url", config.Config.NomadAPIURL()).Fatal("Error parsing the nomad url")
|
log.WithError(err).WithField("nomad url", config.Config.NomadAPIURL()).Fatal("Error parsing the nomad url")
|
||||||
}
|
}
|
||||||
|
|
||||||
runnerManager := runner.NewNomadRunnerManager(nomadAPIClient, context.Background())
|
runnerManager, err := runner.NewNomadRunnerManager(nomadAPIClient, context.Background())
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Fatal("Error creating new Nomad runner manager")
|
||||||
|
}
|
||||||
environmentManager := environment.NewNomadEnvironmentManager(runnerManager, nomadAPIClient)
|
environmentManager := environment.NewNomadEnvironmentManager(runnerManager, nomadAPIClient)
|
||||||
|
|
||||||
return &http.Server{
|
return &http.Server{
|
||||||
|
@ -8,7 +8,9 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrNoAllocationsFound = errors.New("no allocation found")
|
var (
|
||||||
|
ErrNoAllocationsFound = errors.New("no allocation found")
|
||||||
|
)
|
||||||
|
|
||||||
// apiQuerier provides access to the Nomad functionality.
|
// apiQuerier provides access to the Nomad functionality.
|
||||||
type apiQuerier interface {
|
type apiQuerier interface {
|
||||||
@ -51,9 +53,8 @@ type apiQuerier interface {
|
|||||||
|
|
||||||
// nomadAPIClient implements the nomadApiQuerier interface and provides access to a real Nomad API.
|
// nomadAPIClient implements the nomadApiQuerier interface and provides access to a real Nomad API.
|
||||||
type nomadAPIClient struct {
|
type nomadAPIClient struct {
|
||||||
client *nomadApi.Client
|
client *nomadApi.Client
|
||||||
namespace string
|
namespace string
|
||||||
queryOptions *nomadApi.QueryOptions // ToDo: Remove
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (nc *nomadAPIClient) init(nomadURL *url.URL, nomadNamespace string) (err error) {
|
func (nc *nomadAPIClient) init(nomadURL *url.URL, nomadNamespace string) (err error) {
|
||||||
@ -63,15 +64,11 @@ func (nc *nomadAPIClient) init(nomadURL *url.URL, nomadNamespace string) (err er
|
|||||||
Namespace: nomadNamespace,
|
Namespace: nomadNamespace,
|
||||||
})
|
})
|
||||||
nc.namespace = nomadNamespace
|
nc.namespace = nomadNamespace
|
||||||
nc.queryOptions = &nomadApi.QueryOptions{
|
|
||||||
Namespace: nc.namespace,
|
|
||||||
}
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (nc *nomadAPIClient) DeleteRunner(runnerID string) (err error) {
|
func (nc *nomadAPIClient) DeleteRunner(runnerID string) (err error) {
|
||||||
// ToDo: Fix Namespace
|
_, _, err = nc.client.Jobs().Deregister(runnerID, true, nc.writeOptions())
|
||||||
_, _, err = nc.client.Jobs().Deregister(runnerID, true, nc.queryOptions)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,7 +86,7 @@ func (nc *nomadAPIClient) Execute(jobID string,
|
|||||||
return nc.client.Allocations().Exec(ctx, allocation, TaskName, tty, command, stdin, stdout, stderr, nil, nil)
|
return nc.client.Allocations().Exec(ctx, allocation, TaskName, tty, command, stdin, stdout, stderr, nil, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (nc *nomadApiClient) listJobs(prefix string) (jobs []*nomadApi.JobListStub, err error) {
|
func (nc *nomadAPIClient) listJobs(prefix string) (jobs []*nomadApi.JobListStub, err error) {
|
||||||
q := nomadApi.QueryOptions{
|
q := nomadApi.QueryOptions{
|
||||||
Namespace: nc.namespace,
|
Namespace: nc.namespace,
|
||||||
Prefix: prefix,
|
Prefix: prefix,
|
||||||
@ -120,7 +117,7 @@ func (nc *nomadAPIClient) EvaluationStream(evalID string, ctx context.Context) (
|
|||||||
nomadApi.TopicEvaluation: {evalID},
|
nomadApi.TopicEvaluation: {evalID},
|
||||||
},
|
},
|
||||||
0,
|
0,
|
||||||
nc.queryOptions)
|
nc.queryOptions())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -131,6 +128,18 @@ func (nc *nomadAPIClient) AllocationStream(ctx context.Context) (stream <-chan *
|
|||||||
nomadApi.TopicAllocation: {},
|
nomadApi.TopicAllocation: {},
|
||||||
},
|
},
|
||||||
0,
|
0,
|
||||||
nc.queryOptions)
|
nc.queryOptions())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (nc *nomadAPIClient) queryOptions() *nomadApi.QueryOptions {
|
||||||
|
return &nomadApi.QueryOptions{
|
||||||
|
Namespace: nc.namespace,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (nc *nomadAPIClient) writeOptions() *nomadApi.WriteOptions {
|
||||||
|
return &nomadApi.WriteOptions{
|
||||||
|
Namespace: nc.namespace,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -142,6 +142,29 @@ func (_m *ExecutorAPIMock) JobScale(jobId string) (uint, error) {
|
|||||||
return r0, r1
|
return r0, r1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadAllJobs provides a mock function with given fields:
|
||||||
|
func (_m *ExecutorAPIMock) LoadAllJobs() ([]*api.Job, error) {
|
||||||
|
ret := _m.Called()
|
||||||
|
|
||||||
|
var r0 []*api.Job
|
||||||
|
if rf, ok := ret.Get(0).(func() []*api.Job); ok {
|
||||||
|
r0 = rf()
|
||||||
|
} else {
|
||||||
|
if ret.Get(0) != nil {
|
||||||
|
r0 = ret.Get(0).([]*api.Job)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var r1 error
|
||||||
|
if rf, ok := ret.Get(1).(func() error); ok {
|
||||||
|
r1 = rf()
|
||||||
|
} else {
|
||||||
|
r1 = ret.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0, r1
|
||||||
|
}
|
||||||
|
|
||||||
// LoadJobList provides a mock function with given fields:
|
// LoadJobList provides a mock function with given fields:
|
||||||
func (_m *ExecutorAPIMock) LoadJobList() ([]*api.JobListStub, error) {
|
func (_m *ExecutorAPIMock) LoadJobList() ([]*api.JobListStub, error) {
|
||||||
ret := _m.Called()
|
ret := _m.Called()
|
||||||
@ -188,6 +211,43 @@ func (_m *ExecutorAPIMock) LoadRunners(jobID string) ([]string, error) {
|
|||||||
return r0, r1
|
return r0, r1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadTemplateJob provides a mock function with given fields: environmentID
|
||||||
|
func (_m *ExecutorAPIMock) LoadTemplateJob(environmentID string) (*api.Job, error) {
|
||||||
|
ret := _m.Called(environmentID)
|
||||||
|
|
||||||
|
var r0 *api.Job
|
||||||
|
if rf, ok := ret.Get(0).(func(string) *api.Job); ok {
|
||||||
|
r0 = rf(environmentID)
|
||||||
|
} else {
|
||||||
|
if ret.Get(0) != nil {
|
||||||
|
r0 = ret.Get(0).(*api.Job)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var r1 error
|
||||||
|
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||||
|
r1 = rf(environmentID)
|
||||||
|
} else {
|
||||||
|
r1 = ret.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0, r1
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarkRunnerAsUsed provides a mock function with given fields: runnerID
|
||||||
|
func (_m *ExecutorAPIMock) MarkRunnerAsUsed(runnerID string) error {
|
||||||
|
ret := _m.Called(runnerID)
|
||||||
|
|
||||||
|
var r0 error
|
||||||
|
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||||
|
r0 = rf(runnerID)
|
||||||
|
} else {
|
||||||
|
r0 = ret.Error(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0
|
||||||
|
}
|
||||||
|
|
||||||
// MonitorEvaluation provides a mock function with given fields: evaluationID, ctx
|
// MonitorEvaluation provides a mock function with given fields: evaluationID, ctx
|
||||||
func (_m *ExecutorAPIMock) MonitorEvaluation(evaluationID string, ctx context.Context) error {
|
func (_m *ExecutorAPIMock) MonitorEvaluation(evaluationID string, ctx context.Context) error {
|
||||||
ret := _m.Called(evaluationID, ctx)
|
ret := _m.Called(evaluationID, ctx)
|
||||||
@ -289,7 +349,7 @@ func (_m *ExecutorAPIMock) jobInfo(jobID string) (*api.Job, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// listJobs provides a mock function with given fields: prefix
|
// listJobs provides a mock function with given fields: prefix
|
||||||
func (_m *ExecutorApiMock) listJobs(prefix string) ([]*api.JobListStub, error) {
|
func (_m *ExecutorAPIMock) listJobs(prefix string) ([]*api.JobListStub, error) {
|
||||||
ret := _m.Called(prefix)
|
ret := _m.Called(prefix)
|
||||||
|
|
||||||
var r0 []*api.JobListStub
|
var r0 []*api.JobListStub
|
||||||
|
51
nomad/job.go
51
nomad/job.go
@ -1,17 +1,56 @@
|
|||||||
package nomad
|
package nomad
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
nomadApi "github.com/hashicorp/nomad/api"
|
nomadApi "github.com/hashicorp/nomad/api"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
TaskGroupName = "default-group"
|
TaskGroupName = "default-group"
|
||||||
TaskName = "default-task"
|
TaskName = "default-task"
|
||||||
DefaultJobIDFormat = "%s-default"
|
ConfigTaskGroupName = "config"
|
||||||
|
ConfigTaskName = "config"
|
||||||
|
defaultRunnerJobID = "default"
|
||||||
|
runnerJobIDFormat = "%s-%s"
|
||||||
|
DefaultTaskDriver = "docker"
|
||||||
|
DefaultConfigTaskDriver = "exec"
|
||||||
|
DefaultConfigTaskCommand = "whoami"
|
||||||
|
ConfigMetaEnvironmentKey = "environment"
|
||||||
|
ConfigMetaUsedKey = "used"
|
||||||
|
ConfigMetaUsedValue = "true"
|
||||||
|
ConfigMetaUnusedValue = "false"
|
||||||
|
ConfigMetaPoolSizeKey = "prewarmingPoolSize"
|
||||||
)
|
)
|
||||||
|
|
||||||
func DefaultJobID(id string) string {
|
func DefaultJobID(id string) string {
|
||||||
return fmt.Sprintf(DefaultJobIDFormat, id)
|
return RunnerJobID(id, defaultRunnerJobID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func RunnerJobID(environmentID, runnerID string) string {
|
||||||
|
return fmt.Sprintf(runnerJobIDFormat, environmentID, runnerID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsDefaultJobID(jobID string) bool {
|
||||||
|
parts := strings.Split(jobID, "-")
|
||||||
|
return len(parts) == 2 && parts[1] == defaultRunnerJobID
|
||||||
|
}
|
||||||
|
|
||||||
|
func FindConfigTaskGroup(job *nomadApi.Job) *nomadApi.TaskGroup {
|
||||||
|
for _, tg := range job.TaskGroups {
|
||||||
|
if *tg.Name == ConfigTaskGroupName {
|
||||||
|
return tg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func EnvironmentIDFromJobID(jobID string) string {
|
||||||
|
parts := strings.Split(jobID, "-")
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return parts[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (nc *nomadAPIClient) jobInfo(jobID string) (job *nomadApi.Job, err error) {
|
func (nc *nomadAPIClient) jobInfo(jobID string) (job *nomadApi.Job, err error) {
|
||||||
@ -21,13 +60,13 @@ func (nc *nomadAPIClient) jobInfo(jobID string) (job *nomadApi.Job, err error) {
|
|||||||
|
|
||||||
// LoadJobList loads the list of jobs from the Nomad api.
|
// LoadJobList loads the list of jobs from the Nomad api.
|
||||||
func (nc *nomadAPIClient) LoadJobList() (list []*nomadApi.JobListStub, err error) {
|
func (nc *nomadAPIClient) LoadJobList() (list []*nomadApi.JobListStub, err error) {
|
||||||
list, _, err = nc.client.Jobs().List(nc.queryOptions)
|
list, _, err = nc.client.Jobs().List(nc.queryOptions())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// JobScale returns the scale of the passed job.
|
// JobScale returns the scale of the passed job.
|
||||||
func (nc *nomadAPIClient) JobScale(jobID string) (jobScale uint, err error) {
|
func (nc *nomadAPIClient) JobScale(jobID string) (jobScale uint, err error) {
|
||||||
status, _, err := nc.client.Jobs().ScaleStatus(jobID, nc.queryOptions)
|
status, _, err := nc.client.Jobs().ScaleStatus(jobID, nc.queryOptions())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -18,6 +18,7 @@ var (
|
|||||||
ErrorExecutorCommunicationFailed = errors.New("communication with executor failed")
|
ErrorExecutorCommunicationFailed = errors.New("communication with executor failed")
|
||||||
errEvaluation = errors.New("evaluation could not complete")
|
errEvaluation = errors.New("evaluation could not complete")
|
||||||
errPlacingAllocations = errors.New("failed to place all allocations")
|
errPlacingAllocations = errors.New("failed to place all allocations")
|
||||||
|
errFindingTaskGroup = errors.New("no task group found")
|
||||||
)
|
)
|
||||||
|
|
||||||
type AllocationProcessor func(*nomadApi.Allocation)
|
type AllocationProcessor func(*nomadApi.Allocation)
|
||||||
@ -26,6 +27,9 @@ type AllocationProcessor func(*nomadApi.Allocation)
|
|||||||
type ExecutorAPI interface {
|
type ExecutorAPI interface {
|
||||||
apiQuerier
|
apiQuerier
|
||||||
|
|
||||||
|
// LoadAllJobs loads all existing jobs independent of the environment or if it is a template job.
|
||||||
|
LoadAllJobs() ([]*nomadApi.Job, error)
|
||||||
|
|
||||||
// LoadRunners loads all jobs of the specified environment which are running and not about to get stopped.
|
// LoadRunners loads all jobs of the specified environment which are running and not about to get stopped.
|
||||||
LoadRunners(environmentID string) (runnerIds []string, err error)
|
LoadRunners(environmentID string) (runnerIds []string, err error)
|
||||||
|
|
||||||
@ -46,6 +50,9 @@ type ExecutorAPI interface {
|
|||||||
// If tty is true, the command will run with a tty.
|
// If tty is true, the command will run with a tty.
|
||||||
ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool,
|
ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool,
|
||||||
stdin io.Reader, stdout, stderr io.Writer) (int, error)
|
stdin io.Reader, stdout, stderr io.Writer) (int, error)
|
||||||
|
|
||||||
|
// MarkRunnerAsUsed marks the runner with the given ID as used.
|
||||||
|
MarkRunnerAsUsed(runnerID string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// APIClient implements the ExecutorAPI interface and can be used to perform different operations on the real
|
// APIClient implements the ExecutorAPI interface and can be used to perform different operations on the real
|
||||||
@ -221,6 +228,40 @@ func checkEvaluation(eval *nomadApi.Evaluation) (err error) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *APIClient) MarkRunnerAsUsed(runnerID string) error {
|
||||||
|
job, err := a.jobInfo(runnerID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't retrieve job info: %w", err)
|
||||||
|
}
|
||||||
|
var taskGroup = FindConfigTaskGroup(job)
|
||||||
|
if taskGroup == nil {
|
||||||
|
return errFindingTaskGroup
|
||||||
|
}
|
||||||
|
taskGroup.Meta[ConfigMetaUsedKey] = ConfigMetaUsedValue
|
||||||
|
_, err = a.RegisterNomadJob(job)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't update runner config: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *APIClient) LoadAllJobs() ([]*nomadApi.Job, error) {
|
||||||
|
jobStubs, err := a.LoadJobList()
|
||||||
|
if err != nil {
|
||||||
|
return []*nomadApi.Job{}, fmt.Errorf("couldn't load jobs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobs := make([]*nomadApi.Job, 0, len(jobStubs))
|
||||||
|
for _, jobStub := range jobStubs {
|
||||||
|
job, err := a.apiQuerier.jobInfo(jobStub.ID)
|
||||||
|
if err != nil {
|
||||||
|
return []*nomadApi.Job{}, fmt.Errorf("couldn't load job info for job %v: %w", jobStub.ID, err)
|
||||||
|
}
|
||||||
|
jobs = append(jobs, job)
|
||||||
|
}
|
||||||
|
return jobs, nil
|
||||||
|
}
|
||||||
|
|
||||||
// nullReader is a struct that implements the io.Reader interface and returns nothing when reading
|
// nullReader is a struct that implements the io.Reader interface and returns nothing when reading
|
||||||
// from it.
|
// from it.
|
||||||
type nullReader struct{}
|
type nullReader struct{}
|
||||||
|
@ -62,11 +62,11 @@ func newJobListStub(id, status string, amountRunning int) *nomadApi.JobListStub
|
|||||||
|
|
||||||
func (s *LoadRunnersTestSuite) TestErrorOfUnderlyingApiCallIsPropagated() {
|
func (s *LoadRunnersTestSuite) TestErrorOfUnderlyingApiCallIsPropagated() {
|
||||||
s.mock.On("listJobs", mock.AnythingOfType("string")).
|
s.mock.On("listJobs", mock.AnythingOfType("string")).
|
||||||
Return(nil, tests.DefaultError)
|
Return(nil, tests.ErrDefault)
|
||||||
|
|
||||||
returnedIds, err := s.nomadApiClient.LoadRunners(suite.jobId)
|
returnedIds, err := s.nomadApiClient.LoadRunners(s.jobId)
|
||||||
s.Nil(returnedIds)
|
s.Nil(returnedIds)
|
||||||
s.Equal(tests.DefaultError, err)
|
s.Equal(tests.ErrDefault, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *LoadRunnersTestSuite) TestReturnsNoErrorWhenUnderlyingApiCallDoesNot() {
|
func (s *LoadRunnersTestSuite) TestReturnsNoErrorWhenUnderlyingApiCallDoesNot() {
|
||||||
@ -79,7 +79,7 @@ func (s *LoadRunnersTestSuite) TestReturnsNoErrorWhenUnderlyingApiCallDoesNot()
|
|||||||
|
|
||||||
func (s *LoadRunnersTestSuite) TestAvailableRunnerIsReturned() {
|
func (s *LoadRunnersTestSuite) TestAvailableRunnerIsReturned() {
|
||||||
s.mock.On("listJobs", mock.AnythingOfType("string")).
|
s.mock.On("listJobs", mock.AnythingOfType("string")).
|
||||||
Return([]*nomadApi.JobListStub{suite.availableRunner}, nil)
|
Return([]*nomadApi.JobListStub{s.availableRunner}, nil)
|
||||||
|
|
||||||
returnedIds, _ := s.nomadApiClient.LoadRunners(s.jobId)
|
returnedIds, _ := s.nomadApiClient.LoadRunners(s.jobId)
|
||||||
s.Len(returnedIds, 1)
|
s.Len(returnedIds, 1)
|
||||||
|
@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
nomadApi "github.com/hashicorp/nomad/api"
|
nomadApi "github.com/hashicorp/nomad/api"
|
||||||
|
"github.com/hashicorp/nomad/nomad/structs"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/logging"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/logging"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
|
||||||
"strconv"
|
"strconv"
|
||||||
@ -20,8 +21,6 @@ var (
|
|||||||
ErrRunnerNotFound = errors.New("no runner found with this id")
|
ErrRunnerNotFound = errors.New("no runner found with this id")
|
||||||
)
|
)
|
||||||
|
|
||||||
const runnerNameFormat = "%s-%s"
|
|
||||||
|
|
||||||
type EnvironmentID int
|
type EnvironmentID int
|
||||||
|
|
||||||
func (e EnvironmentID) toString() string {
|
func (e EnvironmentID) toString() string {
|
||||||
@ -35,7 +34,7 @@ type NomadJobID string
|
|||||||
type Manager interface {
|
type Manager interface {
|
||||||
// CreateOrUpdateEnvironment creates the given environment if it does not exist. Otherwise, it updates
|
// CreateOrUpdateEnvironment creates the given environment if it does not exist. Otherwise, it updates
|
||||||
// the existing environment and all runners.
|
// the existing environment and all runners.
|
||||||
CreateOrUpdateEnvironment(environmentID EnvironmentID, desiredIdleRunnersCount uint) (bool, error)
|
CreateOrUpdateEnvironment(id EnvironmentID, desiredIdleRunnersCount uint) (bool, error)
|
||||||
|
|
||||||
// Claim returns a new runner.
|
// Claim returns a new runner.
|
||||||
// It makes sure that the runner is not in use yet and returns an error if no runner could be provided.
|
// It makes sure that the runner is not in use yet and returns an error if no runner could be provided.
|
||||||
@ -59,14 +58,18 @@ type NomadRunnerManager struct {
|
|||||||
// NewNomadRunnerManager creates a new runner manager that keeps track of all runners.
|
// NewNomadRunnerManager creates a new runner manager that keeps track of all runners.
|
||||||
// It uses the apiClient for all requests and runs a background task to keep the runners in sync with Nomad.
|
// It uses the apiClient for all requests and runs a background task to keep the runners in sync with Nomad.
|
||||||
// If you cancel the context the background synchronization will be stopped.
|
// If you cancel the context the background synchronization will be stopped.
|
||||||
func NewNomadRunnerManager(apiClient nomad.ExecutorAPI, ctx context.Context) *NomadRunnerManager {
|
func NewNomadRunnerManager(apiClient nomad.ExecutorAPI, ctx context.Context) (*NomadRunnerManager, error) {
|
||||||
m := &NomadRunnerManager{
|
m := &NomadRunnerManager{
|
||||||
apiClient,
|
apiClient,
|
||||||
NewLocalNomadJobStorage(),
|
NewLocalNomadJobStorage(),
|
||||||
NewLocalRunnerStorage(),
|
NewLocalRunnerStorage(),
|
||||||
}
|
}
|
||||||
|
err := m.loadExistingEnvironments()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
go m.updateRunners(ctx)
|
go m.updateRunners(ctx)
|
||||||
return m
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type NomadEnvironment struct {
|
type NomadEnvironment struct {
|
||||||
@ -140,7 +143,7 @@ func (m *NomadRunnerManager) updateEnvironment(id EnvironmentID, desiredIdleRunn
|
|||||||
errorResult := strings.Join(occurredErrors, "\n")
|
errorResult := strings.Join(occurredErrors, "\n")
|
||||||
return fmt.Errorf("%d errors occurred when updating environment: %s", len(occurredErrors), errorResult)
|
return fmt.Errorf("%d errors occurred when updating environment: %s", len(occurredErrors), errorResult)
|
||||||
}
|
}
|
||||||
return nil
|
return m.scaleEnvironment(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *NomadRunnerManager) Claim(environmentID EnvironmentID) (Runner, error) {
|
func (m *NomadRunnerManager) Claim(environmentID EnvironmentID) (Runner, error) {
|
||||||
@ -153,10 +156,16 @@ func (m *NomadRunnerManager) Claim(environmentID EnvironmentID) (Runner, error)
|
|||||||
return nil, ErrNoRunnersAvailable
|
return nil, ErrNoRunnersAvailable
|
||||||
}
|
}
|
||||||
m.usedRunners.Add(runner)
|
m.usedRunners.Add(runner)
|
||||||
err := m.scaleEnvironment(environmentID)
|
err := m.apiClient.MarkRunnerAsUsed(runner.Id())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("can not scale up: %w", err)
|
return nil, fmt.Errorf("can't mark runner as used: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = m.scaleEnvironment(environmentID)
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).WithField("environmentID", environmentID).Error("Couldn't scale environment")
|
||||||
|
}
|
||||||
|
|
||||||
return runner, nil
|
return runner, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -188,31 +197,37 @@ func (m *NomadRunnerManager) updateRunners(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *NomadRunnerManager) onAllocationAdded(alloc *nomadApi.Allocation) {
|
func (m *NomadRunnerManager) onAllocationAdded(alloc *nomadApi.Allocation) {
|
||||||
log.WithField("id", alloc.ID).Debug("Allocation started")
|
log.WithField("id", alloc.JobID).Debug("Runner started")
|
||||||
|
|
||||||
intJobID, err := strconv.Atoi(alloc.JobID)
|
if nomad.IsDefaultJobID(alloc.JobID) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
environmentID := nomad.EnvironmentIDFromJobID(alloc.JobID)
|
||||||
|
intEnvironmentID, err := strconv.Atoi(environmentID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
job, ok := m.environments.Get(EnvironmentID(intJobID))
|
job, ok := m.environments.Get(EnvironmentID(intEnvironmentID))
|
||||||
if ok {
|
if ok {
|
||||||
job.idleRunners.Add(NewNomadAllocation(alloc.ID, m.apiClient))
|
job.idleRunners.Add(NewNomadJob(alloc.JobID, m.apiClient))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *NomadRunnerManager) onAllocationStopped(alloc *nomadApi.Allocation) {
|
func (m *NomadRunnerManager) onAllocationStopped(alloc *nomadApi.Allocation) {
|
||||||
log.WithField("id", alloc.ID).Debug("Allocation stopped")
|
log.WithField("id", alloc.JobID).Debug("Runner stopped")
|
||||||
|
|
||||||
intJobID, err := strconv.Atoi(alloc.JobID)
|
environmentID := nomad.EnvironmentIDFromJobID(alloc.JobID)
|
||||||
|
intEnvironmentID, err := strconv.Atoi(environmentID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
m.usedRunners.Delete(alloc.ID)
|
m.usedRunners.Delete(alloc.JobID)
|
||||||
job, ok := m.environments.Get(EnvironmentID(intJobID))
|
job, ok := m.environments.Get(EnvironmentID(intEnvironmentID))
|
||||||
if ok {
|
if ok {
|
||||||
job.idleRunners.Delete(alloc.ID)
|
job.idleRunners.Delete(alloc.JobID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -224,6 +239,9 @@ func (m *NomadRunnerManager) scaleEnvironment(id EnvironmentID) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
required := int(environment.desiredIdleRunnersCount) - environment.idleRunners.Length()
|
required := int(environment.desiredIdleRunnersCount) - environment.idleRunners.Length()
|
||||||
|
|
||||||
|
log.WithField("required", required).Debug("Scaling environment")
|
||||||
|
|
||||||
for i := 0; i < required; i++ {
|
for i := 0; i < required; i++ {
|
||||||
err := m.createRunner(environment)
|
err := m.createRunner(environment)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -238,7 +256,7 @@ func (m *NomadRunnerManager) createRunner(environment *NomadEnvironment) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed generating runner id")
|
return fmt.Errorf("failed generating runner id")
|
||||||
}
|
}
|
||||||
newRunnerID := fmt.Sprintf(runnerNameFormat, environment.ID().toString(), newUUID.String())
|
newRunnerID := nomad.RunnerJobID(environment.ID().toString(), newUUID.String())
|
||||||
|
|
||||||
template := *environment.templateJob
|
template := *environment.templateJob
|
||||||
template.ID = &newRunnerID
|
template.ID = &newRunnerID
|
||||||
@ -252,11 +270,11 @@ func (m *NomadRunnerManager) createRunner(environment *NomadEnvironment) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("couldn't monitor evaluation: %w", err)
|
return fmt.Errorf("couldn't monitor evaluation: %w", err)
|
||||||
}
|
}
|
||||||
environment.idleRunners.Add(NewNomadJob(newRunnerID, m.apiClient))
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *NomadRunnerManager) unusedRunners(environmentID EnvironmentID, fetchedRunnerIds []string) (newRunners []Runner) {
|
func (m *NomadRunnerManager) unusedRunners(
|
||||||
|
environmentID EnvironmentID, fetchedRunnerIds []string) (newRunners []Runner) {
|
||||||
newRunners = make([]Runner, 0)
|
newRunners = make([]Runner, 0)
|
||||||
job, ok := m.environments.Get(environmentID)
|
job, ok := m.environments.Get(environmentID)
|
||||||
if !ok {
|
if !ok {
|
||||||
@ -272,5 +290,80 @@ func (m *NomadRunnerManager) unusedRunners(environmentID EnvironmentID, fetchedR
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return
|
return newRunners
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *NomadRunnerManager) loadExistingEnvironments() error {
|
||||||
|
jobs, err := m.apiClient.LoadAllJobs()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("can't load template jobs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, job := range jobs {
|
||||||
|
m.loadExistingJob(job)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, environmentID := range m.environments.List() {
|
||||||
|
err := m.scaleEnvironment(environmentID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("can not scale up: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *NomadRunnerManager) loadExistingJob(job *nomadApi.Job) {
|
||||||
|
if *job.Status != structs.JobStatusRunning {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
jobLogger := log.WithField("jobID", *job.ID)
|
||||||
|
|
||||||
|
configTaskGroup := nomad.FindConfigTaskGroup(job)
|
||||||
|
if configTaskGroup == nil {
|
||||||
|
jobLogger.Info("Couldn't find config task group in job, skipping ...")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if configTaskGroup.Meta[nomad.ConfigMetaUsedKey] == nomad.ConfigMetaUsedValue {
|
||||||
|
m.usedRunners.Add(NewNomadJob(*job.ID, m.apiClient))
|
||||||
|
jobLogger.Info("Added job to usedRunners")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
environmentID := configTaskGroup.Meta[nomad.ConfigMetaEnvironmentKey]
|
||||||
|
environmentIDInt, err := strconv.Atoi(environmentID)
|
||||||
|
if err != nil {
|
||||||
|
jobLogger.WithField("environmentID", environmentID).
|
||||||
|
WithError(err).
|
||||||
|
Error("Couldn't convert environment id of template job to int")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
environment, ok := m.environments.Get(EnvironmentID(environmentIDInt))
|
||||||
|
if !ok {
|
||||||
|
desiredIdleRunnersCount, err := strconv.Atoi(configTaskGroup.Meta[nomad.ConfigMetaPoolSizeKey])
|
||||||
|
if err != nil {
|
||||||
|
jobLogger.WithError(err).Error("Couldn't convert pool size to int")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
environment = &NomadEnvironment{
|
||||||
|
environmentID: EnvironmentID(environmentIDInt),
|
||||||
|
idleRunners: NewLocalRunnerStorage(),
|
||||||
|
desiredIdleRunnersCount: uint(desiredIdleRunnersCount),
|
||||||
|
}
|
||||||
|
m.environments.Add(environment)
|
||||||
|
log.WithField("environmentID", environment.environmentID).Info("Added existing environment")
|
||||||
|
}
|
||||||
|
|
||||||
|
if nomad.IsDefaultJobID(*job.ID) {
|
||||||
|
environment.templateJob = job
|
||||||
|
} else {
|
||||||
|
log.WithField("jobID", *job.ID).
|
||||||
|
WithField("environmentID", environment.environmentID).
|
||||||
|
Info("Added idle runner")
|
||||||
|
environment.idleRunners.Add(NewNomadJob(*job.ID, m.apiClient))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -10,11 +10,11 @@ type ManagerMock struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Claim provides a mock function with given fields: id
|
// Claim provides a mock function with given fields: id
|
||||||
func (_m *ManagerMock) Claim(id EnvironmentId) (Runner, error) {
|
func (_m *ManagerMock) Claim(id EnvironmentID) (Runner, error) {
|
||||||
ret := _m.Called(id)
|
ret := _m.Called(id)
|
||||||
|
|
||||||
var r0 Runner
|
var r0 Runner
|
||||||
if rf, ok := ret.Get(0).(func(EnvironmentId) Runner); ok {
|
if rf, ok := ret.Get(0).(func(EnvironmentID) Runner); ok {
|
||||||
r0 = rf(id)
|
r0 = rf(id)
|
||||||
} else {
|
} else {
|
||||||
if ret.Get(0) != nil {
|
if ret.Get(0) != nil {
|
||||||
@ -23,7 +23,7 @@ func (_m *ManagerMock) Claim(id EnvironmentId) (Runner, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var r1 error
|
var r1 error
|
||||||
if rf, ok := ret.Get(1).(func(EnvironmentId) error); ok {
|
if rf, ok := ret.Get(1).(func(EnvironmentID) error); ok {
|
||||||
r1 = rf(id)
|
r1 = rf(id)
|
||||||
} else {
|
} else {
|
||||||
r1 = ret.Error(1)
|
r1 = ret.Error(1)
|
||||||
@ -32,20 +32,20 @@ func (_m *ManagerMock) Claim(id EnvironmentId) (Runner, error) {
|
|||||||
return r0, r1
|
return r0, r1
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateOrUpdateEnvironment provides a mock function with given fields: environmentId, desiredIdleRunnersCount
|
// CreateOrUpdateEnvironment provides a mock function with given fields: id, desiredIdleRunnersCount
|
||||||
func (_m *ManagerMock) CreateOrUpdateEnvironment(environmentId EnvironmentId, desiredIdleRunnersCount uint) (bool, error) {
|
func (_m *ManagerMock) CreateOrUpdateEnvironment(id EnvironmentID, desiredIdleRunnersCount uint) (bool, error) {
|
||||||
ret := _m.Called(environmentId, desiredIdleRunnersCount)
|
ret := _m.Called(id, desiredIdleRunnersCount)
|
||||||
|
|
||||||
var r0 bool
|
var r0 bool
|
||||||
if rf, ok := ret.Get(0).(func(EnvironmentId, uint) bool); ok {
|
if rf, ok := ret.Get(0).(func(EnvironmentID, uint) bool); ok {
|
||||||
r0 = rf(environmentId, desiredIdleRunnersCount)
|
r0 = rf(id, desiredIdleRunnersCount)
|
||||||
} else {
|
} else {
|
||||||
r0 = ret.Get(0).(bool)
|
r0 = ret.Get(0).(bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
var r1 error
|
var r1 error
|
||||||
if rf, ok := ret.Get(1).(func(EnvironmentId, uint) error); ok {
|
if rf, ok := ret.Get(1).(func(EnvironmentID, uint) error); ok {
|
||||||
r1 = rf(environmentId, desiredIdleRunnersCount)
|
r1 = rf(id, desiredIdleRunnersCount)
|
||||||
} else {
|
} else {
|
||||||
r1 = ret.Error(1)
|
r1 = ret.Error(1)
|
||||||
}
|
}
|
||||||
@ -53,13 +53,13 @@ func (_m *ManagerMock) CreateOrUpdateEnvironment(environmentId EnvironmentId, de
|
|||||||
return r0, r1
|
return r0, r1
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get provides a mock function with given fields: runnerId
|
// Get provides a mock function with given fields: runnerID
|
||||||
func (_m *ManagerMock) Get(runnerId string) (Runner, error) {
|
func (_m *ManagerMock) Get(runnerID string) (Runner, error) {
|
||||||
ret := _m.Called(runnerId)
|
ret := _m.Called(runnerID)
|
||||||
|
|
||||||
var r0 Runner
|
var r0 Runner
|
||||||
if rf, ok := ret.Get(0).(func(string) Runner); ok {
|
if rf, ok := ret.Get(0).(func(string) Runner); ok {
|
||||||
r0 = rf(runnerId)
|
r0 = rf(runnerID)
|
||||||
} else {
|
} else {
|
||||||
if ret.Get(0) != nil {
|
if ret.Get(0) != nil {
|
||||||
r0 = ret.Get(0).(Runner)
|
r0 = ret.Get(0).(Runner)
|
||||||
@ -68,7 +68,7 @@ func (_m *ManagerMock) Get(runnerId string) (Runner, error) {
|
|||||||
|
|
||||||
var r1 error
|
var r1 error
|
||||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||||
r1 = rf(runnerId)
|
r1 = rf(runnerID)
|
||||||
} else {
|
} else {
|
||||||
r1 = ret.Error(1)
|
r1 = ret.Error(1)
|
||||||
}
|
}
|
||||||
|
@ -31,13 +31,16 @@ type ManagerTestSuite struct {
|
|||||||
|
|
||||||
func (s *ManagerTestSuite) SetupTest() {
|
func (s *ManagerTestSuite) SetupTest() {
|
||||||
s.apiMock = &nomad.ExecutorAPIMock{}
|
s.apiMock = &nomad.ExecutorAPIMock{}
|
||||||
|
mockRunnerQueries(s.apiMock, []string{})
|
||||||
// Instantly closed context to manually start the update process in some cases
|
// Instantly closed context to manually start the update process in some cases
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
s.nomadRunnerManager = NewNomadRunnerManager(s.apiMock, ctx)
|
var err error
|
||||||
|
|
||||||
|
s.nomadRunnerManager, err = NewNomadRunnerManager(s.apiMock, ctx)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
s.exerciseRunner = NewRunner(tests.DefaultRunnerID)
|
s.exerciseRunner = NewRunner(tests.DefaultRunnerID)
|
||||||
mockRunnerQueries(s.apiMock, []string{})
|
|
||||||
s.registerDefaultEnvironment()
|
s.registerDefaultEnvironment()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -49,6 +52,8 @@ func mockRunnerQueries(apiMock *nomad.ExecutorAPIMock, returnedRunnerIds []strin
|
|||||||
<-time.After(10 * time.Minute) // 10 minutes is the default test timeout
|
<-time.After(10 * time.Minute) // 10 minutes is the default test timeout
|
||||||
call.ReturnArguments = mock.Arguments{nil}
|
call.ReturnArguments = mock.Arguments{nil}
|
||||||
})
|
})
|
||||||
|
apiMock.On("LoadAllJobs").Return([]*nomadApi.Job{}, nil)
|
||||||
|
apiMock.On("MarkRunnerAsUsed", mock.AnythingOfType("string")).Return(nil)
|
||||||
apiMock.On("LoadRunners", tests.DefaultJobID).Return(returnedRunnerIds, nil)
|
apiMock.On("LoadRunners", tests.DefaultJobID).Return(returnedRunnerIds, nil)
|
||||||
apiMock.On("JobScale", tests.DefaultJobID).Return(uint(len(returnedRunnerIds)), nil)
|
apiMock.On("JobScale", tests.DefaultJobID).Return(uint(len(returnedRunnerIds)), nil)
|
||||||
apiMock.On("SetJobScale", tests.DefaultJobID, mock.AnythingOfType("uint"), "Runner Requested").Return(nil)
|
apiMock.On("SetJobScale", tests.DefaultJobID, mock.AnythingOfType("uint"), "Runner Requested").Return(nil)
|
||||||
@ -58,7 +63,7 @@ func mockRunnerQueries(apiMock *nomad.ExecutorAPIMock, returnedRunnerIds []strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *ManagerTestSuite) registerDefaultEnvironment() {
|
func (s *ManagerTestSuite) registerDefaultEnvironment() {
|
||||||
err := s.nomadRunnerManager.registerEnvironment(defaultEnvironmentId, 0)
|
err := s.nomadRunnerManager.registerEnvironment(defaultEnvironmentID, 0)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -72,9 +77,9 @@ func (s *ManagerTestSuite) waitForRunnerRefresh() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *ManagerTestSuite) TestRegisterEnvironmentAddsNewJob() {
|
func (s *ManagerTestSuite) TestRegisterEnvironmentAddsNewJob() {
|
||||||
err := s.nomadRunnerManager.registerEnvironment(anotherEnvironmentId, defaultDesiredRunnersCount)
|
err := s.nomadRunnerManager.registerEnvironment(anotherEnvironmentID, defaultDesiredRunnersCount)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
job, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentId)
|
job, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID)
|
||||||
s.True(ok)
|
s.True(ok)
|
||||||
s.NotNil(job)
|
s.NotNil(job)
|
||||||
}
|
}
|
||||||
@ -125,7 +130,7 @@ func (s *ManagerTestSuite) TestClaimThrowsAnErrorIfNoRunnersAvailable() {
|
|||||||
|
|
||||||
func (s *ManagerTestSuite) TestClaimAddsRunnerToUsedRunners() {
|
func (s *ManagerTestSuite) TestClaimAddsRunnerToUsedRunners() {
|
||||||
s.AddIdleRunnerForDefaultEnvironment(s.exerciseRunner)
|
s.AddIdleRunnerForDefaultEnvironment(s.exerciseRunner)
|
||||||
receivedRunner, _ := s.nomadRunnerManager.Claim(defaultEnvironmentId)
|
receivedRunner, _ := s.nomadRunnerManager.Claim(defaultEnvironmentID)
|
||||||
savedRunner, ok := s.nomadRunnerManager.usedRunners.Get(receivedRunner.Id())
|
savedRunner, ok := s.nomadRunnerManager.usedRunners.Get(receivedRunner.Id())
|
||||||
s.True(ok)
|
s.True(ok)
|
||||||
s.Equal(savedRunner, receivedRunner)
|
s.Equal(savedRunner, receivedRunner)
|
||||||
@ -188,9 +193,9 @@ func (s *ManagerTestSuite) TestUpdateRunnersLogsErrorFromWatchAllocation() {
|
|||||||
|
|
||||||
func (s *ManagerTestSuite) TestUpdateRunnersAddsIdleRunner() {
|
func (s *ManagerTestSuite) TestUpdateRunnersAddsIdleRunner() {
|
||||||
allocation := &nomadApi.Allocation{ID: tests.DefaultRunnerID}
|
allocation := &nomadApi.Allocation{ID: tests.DefaultRunnerID}
|
||||||
defaultJob, ok := s.nomadRunnerManager.jobs.Get(defaultEnvironmentID)
|
defaultJob, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID)
|
||||||
s.Require().True(ok)
|
s.Require().True(ok)
|
||||||
allocation.JobID = string(defaultJob.jobID)
|
allocation.JobID = defaultJob.environmentID.toString()
|
||||||
|
|
||||||
_, ok = defaultJob.idleRunners.Get(allocation.ID)
|
_, ok = defaultJob.idleRunners.Get(allocation.ID)
|
||||||
s.Require().False(ok)
|
s.Require().False(ok)
|
||||||
@ -215,9 +220,9 @@ func (s *ManagerTestSuite) TestUpdateRunnersAddsIdleRunner() {
|
|||||||
|
|
||||||
func (s *ManagerTestSuite) TestUpdateRunnersRemovesIdleAndUsedRunner() {
|
func (s *ManagerTestSuite) TestUpdateRunnersRemovesIdleAndUsedRunner() {
|
||||||
allocation := &nomadApi.Allocation{ID: tests.DefaultRunnerID}
|
allocation := &nomadApi.Allocation{ID: tests.DefaultRunnerID}
|
||||||
defaultJob, ok := s.nomadRunnerManager.jobs.Get(defaultEnvironmentID)
|
defaultJob, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID)
|
||||||
s.Require().True(ok)
|
s.Require().True(ok)
|
||||||
allocation.JobID = string(defaultJob.jobID)
|
allocation.JobID = defaultJob.environmentID.toString()
|
||||||
|
|
||||||
testRunner := NewRunner(allocation.ID)
|
testRunner := NewRunner(allocation.ID)
|
||||||
defaultJob.idleRunners.Add(testRunner)
|
defaultJob.idleRunners.Add(testRunner)
|
||||||
|
@ -6,6 +6,9 @@ import (
|
|||||||
|
|
||||||
// NomadEnvironmentStorage is an interface for storing NomadJobs.
|
// NomadEnvironmentStorage is an interface for storing NomadJobs.
|
||||||
type NomadEnvironmentStorage interface {
|
type NomadEnvironmentStorage interface {
|
||||||
|
// List returns all keys of environments stored in this storage.
|
||||||
|
List() []EnvironmentID
|
||||||
|
|
||||||
// Add adds a job to the storage.
|
// Add adds a job to the storage.
|
||||||
// It overwrites the old job if one with the same id was already stored.
|
// It overwrites the old job if one with the same id was already stored.
|
||||||
Add(job *NomadEnvironment)
|
Add(job *NomadEnvironment)
|
||||||
@ -36,6 +39,14 @@ func NewLocalNomadJobStorage() *localNomadJobStorage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *localNomadJobStorage) List() []EnvironmentID {
|
||||||
|
keys := make([]EnvironmentID, 0, len(s.jobs))
|
||||||
|
for k := range s.jobs {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
func (s *localNomadJobStorage) Add(job *NomadEnvironment) {
|
func (s *localNomadJobStorage) Add(job *NomadEnvironment) {
|
||||||
s.Lock()
|
s.Lock()
|
||||||
defer s.Unlock()
|
defer s.Unlock()
|
||||||
|
@ -16,9 +16,9 @@ type JobStoreTestSuite struct {
|
|||||||
job *NomadEnvironment
|
job *NomadEnvironment
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *JobStoreTestSuite) SetupTest() {
|
func (s *JobStoreTestSuite) SetupTest() {
|
||||||
suite.jobStorage = NewLocalNomadJobStorage()
|
s.jobStorage = NewLocalNomadJobStorage()
|
||||||
suite.job = &NomadEnvironment{environmentID: defaultEnvironmentId}
|
s.job = &NomadEnvironment{environmentID: defaultEnvironmentID}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *JobStoreTestSuite) TestAddedJobCanBeRetrieved() {
|
func (s *JobStoreTestSuite) TestAddedJobCanBeRetrieved() {
|
||||||
@ -28,10 +28,10 @@ func (s *JobStoreTestSuite) TestAddedJobCanBeRetrieved() {
|
|||||||
s.Equal(s.job, retrievedJob)
|
s.Equal(s.job, retrievedJob)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *JobStoreTestSuite) TestJobWithSameIdOverwritesOldOne() {
|
func (s *JobStoreTestSuite) TestJobWithSameIdOverwritesOldOne() {
|
||||||
otherJobWithSameID := &NomadEnvironment{environmentID: defaultEnvironmentId}
|
otherJobWithSameID := &NomadEnvironment{environmentID: defaultEnvironmentID}
|
||||||
otherJobWithSameID.templateJob = &nomadApi.Job{}
|
otherJobWithSameID.templateJob = &nomadApi.Job{}
|
||||||
suite.NotEqual(suite.job, otherJobWithSameID)
|
s.NotEqual(s.job, otherJobWithSameID)
|
||||||
|
|
||||||
s.jobStorage.Add(s.job)
|
s.jobStorage.Add(s.job)
|
||||||
s.jobStorage.Add(otherJobWithSameID)
|
s.jobStorage.Add(otherJobWithSameID)
|
||||||
|
@ -58,11 +58,6 @@ type NomadJob struct {
|
|||||||
api nomad.ExecutorAPI
|
api nomad.ExecutorAPI
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRunner creates a new runner with the provided id.
|
|
||||||
func NewRunner(id string) Runner {
|
|
||||||
return NewNomadJob(id, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewNomadJob creates a new NomadJob with the provided id.
|
// NewNomadJob creates a new NomadJob with the provided id.
|
||||||
func NewNomadJob(id string, apiClient nomad.ExecutorAPI) *NomadJob {
|
func NewNomadJob(id string, apiClient nomad.ExecutorAPI) *NomadJob {
|
||||||
return &NomadJob{
|
return &NomadJob{
|
||||||
|
@ -254,5 +254,5 @@ func (s *UpdateFileSystemTestSuite) readFilesFromTarArchive(tarArchive io.Reader
|
|||||||
|
|
||||||
// NewRunner creates a new runner with the provided id.
|
// NewRunner creates a new runner with the provided id.
|
||||||
func NewRunner(id string) Runner {
|
func NewRunner(id string) Runner {
|
||||||
return NewNomadAllocation(id, nil)
|
return NewNomadJob(id, nil)
|
||||||
}
|
}
|
||||||
|
@ -3,8 +3,13 @@ package e2e
|
|||||||
import (
|
import (
|
||||||
nomadApi "github.com/hashicorp/nomad/api"
|
nomadApi "github.com/hashicorp/nomad/api"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
|
||||||
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/logging"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/logging"
|
||||||
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
|
||||||
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/helpers"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -51,8 +56,35 @@ func TestMain(m *testing.M) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.WithError(err).Fatal("Could not create Nomad client")
|
log.WithError(err).Fatal("Could not create Nomad client")
|
||||||
}
|
}
|
||||||
// ToDo: Add Nomad job here when it is possible to create execution environments. See #26.
|
|
||||||
log.Info("Test Run")
|
log.Info("Test Run")
|
||||||
|
createDefaultEnvironment()
|
||||||
|
|
||||||
|
// wait for environment to become ready
|
||||||
|
<-time.After(10 * time.Second)
|
||||||
|
|
||||||
code := m.Run()
|
code := m.Run()
|
||||||
|
cleanupJobsForEnvironment(&testing.T{}, "0")
|
||||||
os.Exit(code)
|
os.Exit(code)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func createDefaultEnvironment() {
|
||||||
|
path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString)
|
||||||
|
|
||||||
|
request := dto.ExecutionEnvironmentRequest{
|
||||||
|
PrewarmingPoolSize: 10,
|
||||||
|
CPULimit: 100,
|
||||||
|
MemoryLimit: 100,
|
||||||
|
Image: "drp.codemoon.xopic.de/openhpi/co_execenv_python:3.8",
|
||||||
|
NetworkAccess: false,
|
||||||
|
ExposedPorts: nil,
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := helpers.HttpPutJSON(path, request)
|
||||||
|
if err != nil || resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent {
|
||||||
|
log.Fatal("Couldn't create default environment for e2e tests")
|
||||||
|
}
|
||||||
|
err = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Failed closing body")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -6,8 +6,10 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
|
||||||
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
|
||||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/helpers"
|
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/helpers"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@ -67,10 +69,22 @@ func TestCreateOrUpdateEnvironment(t *testing.T) {
|
|||||||
validateJob(t, request)
|
validateJob(t, request)
|
||||||
})
|
})
|
||||||
|
|
||||||
_, _, err := nomadClient.Jobs().DeregisterOpts(
|
cleanupJobsForEnvironment(t, tests.AnotherEnvironmentIDAsString)
|
||||||
tests.AnotherEnvironmentIDAsString, &nomadApi.DeregisterOptions{Purge: true}, nil)
|
}
|
||||||
|
|
||||||
|
func cleanupJobsForEnvironment(t *testing.T, environmentID string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
jobListStub, _, err := nomadClient.Jobs().List(&nomadApi.QueryOptions{Prefix: environmentID})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Error when removing test job %v", err)
|
t.Fatalf("Error when listing test jobs: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, j := range jobListStub {
|
||||||
|
_, _, err := nomadClient.Jobs().DeregisterOpts(j.ID, &nomadApi.DeregisterOptions{Purge: true}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Error when removing test job %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -81,7 +95,9 @@ func assertPutReturnsStatusAndZeroContent(t *testing.T, path string,
|
|||||||
require.Nil(t, err)
|
require.Nil(t, err)
|
||||||
assert.Equal(t, status, resp.StatusCode)
|
assert.Equal(t, status, resp.StatusCode)
|
||||||
assert.Equal(t, int64(0), resp.ContentLength)
|
assert.Equal(t, int64(0), resp.ContentLength)
|
||||||
|
content, err := io.ReadAll(resp.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, string(content))
|
||||||
_ = resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -91,7 +107,7 @@ func validateJob(t *testing.T, expected dto.ExecutionEnvironmentRequest) {
|
|||||||
|
|
||||||
assertEqualValueStringPointer(t, nomadNamespace, job.Namespace)
|
assertEqualValueStringPointer(t, nomadNamespace, job.Namespace)
|
||||||
assertEqualValueStringPointer(t, "batch", job.Type)
|
assertEqualValueStringPointer(t, "batch", job.Type)
|
||||||
require.Equal(t, 1, len(job.TaskGroups))
|
require.Equal(t, 2, len(job.TaskGroups))
|
||||||
|
|
||||||
taskGroup := job.TaskGroups[0]
|
taskGroup := job.TaskGroups[0]
|
||||||
require.NotNil(t, taskGroup.Count)
|
require.NotNil(t, taskGroup.Count)
|
||||||
@ -123,7 +139,7 @@ func validateJob(t *testing.T, expected dto.ExecutionEnvironmentRequest) {
|
|||||||
|
|
||||||
func findNomadJob(t *testing.T, jobID string) *nomadApi.Job {
|
func findNomadJob(t *testing.T, jobID string) *nomadApi.Job {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
job, _, err := nomadClient.Jobs().Info(jobID, nil)
|
job, _, err := nomadClient.Jobs().Info(nomad.DefaultJobID(jobID), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Error retrieving Nomad job: %v", err)
|
t.Fatalf("Error retrieving Nomad job: %v", err)
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user