Restructure project

We previously didn't really had any structure in our project apart
from creating a new folder for each package in our project root.
Now that we have accumulated some packages, we use the well-known
Golang project layout in order to clearly communicate our intent
with packages. See https://github.com/golang-standards/project-layout

internal/nomad/api_querier.go

@@ -0,0 +1,213 @@
+package nomad
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	nomadApi "github.com/hashicorp/nomad/api"
+	"io"
+	"net/url"
+)
+
+var (
+	ErrorNoAllocationFound = errors.New("no allocation found")
+)
+
+// apiQuerier provides access to the Nomad functionality.
+type apiQuerier interface {
+	// init prepares an apiClient to be able to communicate to a provided Nomad API.
+	init(nomadURL *url.URL, nomadNamespace string) (err error)
+
+	// LoadJobList loads the list of jobs from the Nomad API.
+	LoadJobList() (list []*nomadApi.JobListStub, err error)
+
+	// JobScale returns the scale of the passed job.
+	JobScale(jobID string) (jobScale uint, err error)
+
+	// SetJobScale sets the scaling count of the passed job to Nomad.
+	SetJobScale(jobID string, count uint, reason string) (err error)
+
+	// DeleteRunner deletes the runner with the given ID.
+	DeleteRunner(runnerID string) (err error)
+
+	// Execute runs a command in the passed job.
+	Execute(jobID string, ctx context.Context, command []string, tty bool,
+		stdin io.Reader, stdout, stderr io.Writer) (int, error)
+
+	// listJobs loads all jobs with the specified prefix.
+	listJobs(prefix string) (allocationListStub []*nomadApi.JobListStub, err error)
+
+	// job returns the job of the given jobID.
+	job(jobID string) (job *nomadApi.Job, err error)
+
+	// allocation returns the first allocation of the given job.
+	allocation(jobID string) (*nomadApi.Allocation, error)
+
+	// RegisterNomadJob registers a job with Nomad.
+	// It returns the evaluation ID that can be used when listening to the Nomad event stream.
+	RegisterNomadJob(job *nomadApi.Job) (string, error)
+
+	// EvaluationStream returns a Nomad event stream filtered to return only events belonging to the
+	// given evaluation ID.
+	EvaluationStream(evalID string, ctx context.Context) (<-chan *nomadApi.Events, error)
+
+	// AllocationStream returns a Nomad event stream filtered to return only allocation events.
+	AllocationStream(ctx context.Context) (<-chan *nomadApi.Events, error)
+}
+
+// nomadAPIClient implements the nomadApiQuerier interface and provides access to a real Nomad API.
+type nomadAPIClient struct {
+	client    *nomadApi.Client
+	namespace string
+}
+
+func (nc *nomadAPIClient) init(nomadURL *url.URL, nomadNamespace string) (err error) {
+	nc.client, err = nomadApi.NewClient(&nomadApi.Config{
+		Address:   nomadURL.String(),
+		TLSConfig: &nomadApi.TLSConfig{},
+		Namespace: nomadNamespace,
+	})
+	if err != nil {
+		return fmt.Errorf("error creating new Nomad client: %w", err)
+	}
+	nc.namespace = nomadNamespace
+	return nil
+}
+
+func (nc *nomadAPIClient) DeleteRunner(runnerID string) (err error) {
+	_, _, err = nc.client.Jobs().Deregister(runnerID, true, nc.writeOptions())
+	return
+}
+
+func (nc *nomadAPIClient) Execute(runnerID string,
+	ctx context.Context, command []string, tty bool,
+	stdin io.Reader, stdout, stderr io.Writer,
+) (int, error) {
+	allocations, _, err := nc.client.Jobs().Allocations(runnerID, false, nil)
+	if err != nil {
+		return 1, fmt.Errorf("error retrieving allocations for runner: %w", err)
+	}
+	if len(allocations) == 0 {
+		return 1, ErrorNoAllocationFound
+	}
+	allocation, _, err := nc.client.Allocations().Info(allocations[0].ID, nil)
+	if err != nil {
+		return 1, fmt.Errorf("error retrieving allocation info: %w", err)
+	}
+	exitCode, err := nc.client.Allocations().Exec(ctx, allocation, TaskName, tty, command, stdin, stdout, stderr, nil, nil)
+	if err != nil {
+		return 1, fmt.Errorf("error executing command in allocation: %w", err)
+	}
+	return exitCode, nil
+}
+
+func (nc *nomadAPIClient) listJobs(prefix string) ([]*nomadApi.JobListStub, error) {
+	q := nomadApi.QueryOptions{
+		Namespace: nc.namespace,
+		Prefix:    prefix,
+	}
+	jobs, _, err := nc.client.Jobs().List(&q)
+	if err != nil {
+		return nil, fmt.Errorf("error listing Nomad jobs: %w", err)
+	}
+	return jobs, nil
+}
+
+func (nc *nomadAPIClient) RegisterNomadJob(job *nomadApi.Job) (string, error) {
+	job.Namespace = &nc.namespace
+	resp, _, err := nc.client.Jobs().Register(job, nil)
+	if err != nil {
+		return "", fmt.Errorf("error registering Nomad job: %w", err)
+	}
+	if resp.Warnings != "" {
+		log.
+			WithField("job", job).
+			WithField("warnings", resp.Warnings).
+			Warn("Received warnings when registering job")
+	}
+	return resp.EvalID, nil
+}
+
+func (nc *nomadAPIClient) EvaluationStream(evalID string, ctx context.Context) (<-chan *nomadApi.Events, error) {
+	stream, err := nc.client.EventStream().Stream(
+		ctx,
+		map[nomadApi.Topic][]string{
+			nomadApi.TopicEvaluation: {evalID},
+		},
+		0,
+		nc.queryOptions())
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving Nomad Evaluation event stream: %w", err)
+	}
+	return stream, nil
+}
+
+func (nc *nomadAPIClient) AllocationStream(ctx context.Context) (<-chan *nomadApi.Events, error) {
+	stream, err := nc.client.EventStream().Stream(
+		ctx,
+		map[nomadApi.Topic][]string{
+			nomadApi.TopicAllocation: {},
+		},
+		0,
+		nc.queryOptions())
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving Nomad Allocation event stream: %w", err)
+	}
+	return stream, nil
+}
+
+func (nc *nomadAPIClient) queryOptions() *nomadApi.QueryOptions {
+	return &nomadApi.QueryOptions{
+		Namespace: nc.namespace,
+	}
+}
+
+func (nc *nomadAPIClient) writeOptions() *nomadApi.WriteOptions {
+	return &nomadApi.WriteOptions{
+		Namespace: nc.namespace,
+	}
+}
+
+// LoadJobList loads the list of jobs from the Nomad api.
+func (nc *nomadAPIClient) LoadJobList() (list []*nomadApi.JobListStub, err error) {
+	list, _, err = nc.client.Jobs().List(nc.queryOptions())
+	return
+}
+
+// JobScale returns the scale of the passed job.
+func (nc *nomadAPIClient) JobScale(jobID string) (uint, error) {
+	status, _, err := nc.client.Jobs().ScaleStatus(jobID, nc.queryOptions())
+	if err != nil {
+		return 0, fmt.Errorf("error retrieving scale status of job: %w", err)
+	}
+	// ToDo: Consider counting also the placed and desired allocations
+	jobScale := uint(status.TaskGroups[TaskGroupName].Running)
+	return jobScale, nil
+}
+
+// SetJobScale sets the scaling count of the passed job to Nomad.
+func (nc *nomadAPIClient) SetJobScale(jobID string, count uint, reason string) (err error) {
+	intCount := int(count)
+	_, _, err = nc.client.Jobs().Scale(jobID, TaskGroupName, &intCount, reason, false, nil, nil)
+	return
+}
+
+func (nc *nomadAPIClient) job(jobID string) (job *nomadApi.Job, err error) {
+	job, _, err = nc.client.Jobs().Info(jobID, nil)
+	return
+}
+
+func (nc *nomadAPIClient) allocation(jobID string) (alloc *nomadApi.Allocation, err error) {
+	allocs, _, err := nc.client.Jobs().Allocations(jobID, false, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error requesting Nomad job allocations: %w", err)
+	}
+	if len(allocs) == 0 {
+		return nil, ErrorNoAllocationFound
+	}
+	alloc, _, err = nc.client.Allocations().Info(allocs[0].ID, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error requesting Nomad allocation info: %w", err)
+	}
+	return alloc, nil
+}

internal/nomad/api_querier_mock.go

@@ -0,0 +1,263 @@
+// Code generated by mockery v0.0.0-dev. DO NOT EDIT.
+
+package nomad
+
+import (
+	context "context"
+
+	api "github.com/hashicorp/nomad/api"
+
+	io "io"
+
+	mock "github.com/stretchr/testify/mock"
+
+	url "net/url"
+)
+
+// apiQuerierMock is an autogenerated mock type for the apiQuerier type
+type apiQuerierMock struct {
+	mock.Mock
+}
+
+// AllocationStream provides a mock function with given fields: ctx
+func (_m *apiQuerierMock) AllocationStream(ctx context.Context) (<-chan *api.Events, error) {
+	ret := _m.Called(ctx)
+
+	var r0 <-chan *api.Events
+	if rf, ok := ret.Get(0).(func(context.Context) <-chan *api.Events); ok {
+		r0 = rf(ctx)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(<-chan *api.Events)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(ctx)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// DeleteRunner provides a mock function with given fields: runnerId
+func (_m *apiQuerierMock) DeleteRunner(runnerId string) error {
+	ret := _m.Called(runnerId)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(runnerId)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// EvaluationStream provides a mock function with given fields: evalID, ctx
+func (_m *apiQuerierMock) EvaluationStream(evalID string, ctx context.Context) (<-chan *api.Events, error) {
+	ret := _m.Called(evalID, ctx)
+
+	var r0 <-chan *api.Events
+	if rf, ok := ret.Get(0).(func(string, context.Context) <-chan *api.Events); ok {
+		r0 = rf(evalID, ctx)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(<-chan *api.Events)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, context.Context) error); ok {
+		r1 = rf(evalID, ctx)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Execute provides a mock function with given fields: jobID, ctx, command, tty, stdin, stdout, stderr
+func (_m *apiQuerierMock) Execute(jobID string, ctx context.Context, command []string, tty bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
+	ret := _m.Called(jobID, ctx, command, tty, stdin, stdout, stderr)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) int); ok {
+		r0 = rf(jobID, ctx, command, tty, stdin, stdout, stderr)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) error); ok {
+		r1 = rf(jobID, ctx, command, tty, stdin, stdout, stderr)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// JobScale provides a mock function with given fields: jobID
+func (_m *apiQuerierMock) JobScale(jobId string) (uint, error) {
+	ret := _m.Called(jobId)
+
+	var r0 uint
+	if rf, ok := ret.Get(0).(func(string) uint); ok {
+		r0 = rf(jobId)
+	} else {
+		r0 = ret.Get(0).(uint)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(jobId)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// LoadJobList provides a mock function with given fields:
+func (_m *apiQuerierMock) LoadJobList() ([]*api.JobListStub, error) {
+	ret := _m.Called()
+
+	var r0 []*api.JobListStub
+	if rf, ok := ret.Get(0).(func() []*api.JobListStub); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*api.JobListStub)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// RegisterNomadJob provides a mock function with given fields: job
+func (_m *apiQuerierMock) RegisterNomadJob(job *api.Job) (string, error) {
+	ret := _m.Called(job)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(*api.Job) string); ok {
+		r0 = rf(job)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(*api.Job) error); ok {
+		r1 = rf(job)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SetJobScale provides a mock function with given fields: jobID, count, reason
+func (_m *apiQuerierMock) SetJobScale(jobId string, count uint, reason string) error {
+	ret := _m.Called(jobId, count, reason)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, uint, string) error); ok {
+		r0 = rf(jobId, count, reason)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// allocation provides a mock function with given fields: jobID
+func (_m *apiQuerierMock) allocation(jobID string) (*api.Allocation, error) {
+	ret := _m.Called(jobID)
+
+	var r0 *api.Allocation
+	if rf, ok := ret.Get(0).(func(string) *api.Allocation); ok {
+		r0 = rf(jobID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*api.Allocation)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(jobID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// init provides a mock function with given fields: nomadURL, nomadNamespace
+func (_m *apiQuerierMock) init(nomadURL *url.URL, nomadNamespace string) error {
+	ret := _m.Called(nomadURL, nomadNamespace)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(*url.URL, string) error); ok {
+		r0 = rf(nomadURL, nomadNamespace)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// job provides a mock function with given fields: jobID
+func (_m *apiQuerierMock) job(jobID string) (*api.Job, error) {
+	ret := _m.Called(jobID)
+
+	var r0 *api.Job
+	if rf, ok := ret.Get(0).(func(string) *api.Job); ok {
+		r0 = rf(jobID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*api.Job)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(jobID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// listJobs provides a mock function with given fields: prefix
+func (_m *apiQuerierMock) listJobs(prefix string) ([]*api.JobListStub, error) {
+	ret := _m.Called(prefix)
+
+	var r0 []*api.JobListStub
+	if rf, ok := ret.Get(0).(func(string) []*api.JobListStub); ok {
+		r0 = rf(prefix)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*api.JobListStub)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(prefix)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

internal/nomad/executor_api_mock.go

@@ -0,0 +1,455 @@
+// Code generated by mockery v0.0.0-dev. DO NOT EDIT.
+
+package nomad
+
+import (
+	context "context"
+
+	api "github.com/hashicorp/nomad/api"
+
+	io "io"
+
+	mock "github.com/stretchr/testify/mock"
+
+	url "net/url"
+)
+
+// ExecutorAPIMock is an autogenerated mock type for the ExecutorAPI type
+type ExecutorAPIMock struct {
+	mock.Mock
+}
+
+// AllocationStream provides a mock function with given fields: ctx
+func (_m *ExecutorAPIMock) AllocationStream(ctx context.Context) (<-chan *api.Events, error) {
+	ret := _m.Called(ctx)
+
+	var r0 <-chan *api.Events
+	if rf, ok := ret.Get(0).(func(context.Context) <-chan *api.Events); ok {
+		r0 = rf(ctx)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(<-chan *api.Events)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(ctx)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// DeleteRunner provides a mock function with given fields: runnerId
+func (_m *ExecutorAPIMock) DeleteRunner(runnerId string) error {
+	ret := _m.Called(runnerId)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(runnerId)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// EvaluationStream provides a mock function with given fields: evalID, ctx
+func (_m *ExecutorAPIMock) EvaluationStream(evalID string, ctx context.Context) (<-chan *api.Events, error) {
+	ret := _m.Called(evalID, ctx)
+
+	var r0 <-chan *api.Events
+	if rf, ok := ret.Get(0).(func(string, context.Context) <-chan *api.Events); ok {
+		r0 = rf(evalID, ctx)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(<-chan *api.Events)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, context.Context) error); ok {
+		r1 = rf(evalID, ctx)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Execute provides a mock function with given fields: jobID, ctx, command, tty, stdin, stdout, stderr
+func (_m *ExecutorAPIMock) Execute(jobID string, ctx context.Context, command []string, tty bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
+	ret := _m.Called(jobID, ctx, command, tty, stdin, stdout, stderr)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) int); ok {
+		r0 = rf(jobID, ctx, command, tty, stdin, stdout, stderr)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) error); ok {
+		r1 = rf(jobID, ctx, command, tty, stdin, stdout, stderr)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ExecuteCommand provides a mock function with given fields: allocationID, ctx, command, tty, stdin, stdout, stderr
+func (_m *ExecutorAPIMock) ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
+	ret := _m.Called(allocationID, ctx, command, tty, stdin, stdout, stderr)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) int); ok {
+		r0 = rf(allocationID, ctx, command, tty, stdin, stdout, stderr)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) error); ok {
+		r1 = rf(allocationID, ctx, command, tty, stdin, stdout, stderr)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// JobScale provides a mock function with given fields: jobID
+func (_m *ExecutorAPIMock) JobScale(jobId string) (uint, error) {
+	ret := _m.Called(jobId)
+
+	var r0 uint
+	if rf, ok := ret.Get(0).(func(string) uint); ok {
+		r0 = rf(jobId)
+	} else {
+		r0 = ret.Get(0).(uint)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(jobId)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// LoadEnvironmentJobs provides a mock function with given fields:
+func (_m *ExecutorAPIMock) LoadEnvironmentJobs() ([]*api.Job, error) {
+	ret := _m.Called()
+
+	var r0 []*api.Job
+	if rf, ok := ret.Get(0).(func() []*api.Job); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*api.Job)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// LoadJobList provides a mock function with given fields:
+func (_m *ExecutorAPIMock) LoadJobList() ([]*api.JobListStub, error) {
+	ret := _m.Called()
+
+	var r0 []*api.JobListStub
+	if rf, ok := ret.Get(0).(func() []*api.JobListStub); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*api.JobListStub)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// LoadRunnerIDs provides a mock function with given fields: environmentID
+func (_m *ExecutorAPIMock) LoadRunnerIDs(environmentID string) ([]string, error) {
+	ret := _m.Called(environmentID)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(string) []string); ok {
+		r0 = rf(environmentID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(environmentID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// LoadRunnerJobs provides a mock function with given fields: environmentID
+func (_m *ExecutorAPIMock) LoadRunnerJobs(environmentID string) ([]*api.Job, error) {
+	ret := _m.Called(environmentID)
+
+	var r0 []*api.Job
+	if rf, ok := ret.Get(0).(func(string) []*api.Job); ok {
+		r0 = rf(environmentID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*api.Job)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(environmentID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// LoadRunnerPorts provides a mock function with given fields: runnerID
+func (_m *ExecutorAPIMock) LoadRunnerPortMappings(runnerID string) ([]api.PortMapping, error) {
+	ret := _m.Called(runnerID)
+
+	var r0 []api.PortMapping
+	if rf, ok := ret.Get(0).(func(string) []api.PortMapping); ok {
+		r0 = rf(runnerID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]api.PortMapping)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(runnerID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// MarkRunnerAsUsed provides a mock function with given fields: runnerID, duration
+func (_m *ExecutorAPIMock) MarkRunnerAsUsed(runnerID string, duration int) error {
+	ret := _m.Called(runnerID, duration)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, int) error); ok {
+		r0 = rf(runnerID, duration)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// MonitorEvaluation provides a mock function with given fields: evaluationID, ctx
+func (_m *ExecutorAPIMock) MonitorEvaluation(evaluationID string, ctx context.Context) error {
+	ret := _m.Called(evaluationID, ctx)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, context.Context) error); ok {
+		r0 = rf(evaluationID, ctx)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// RegisterNomadJob provides a mock function with given fields: job
+func (_m *ExecutorAPIMock) RegisterNomadJob(job *api.Job) (string, error) {
+	ret := _m.Called(job)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(*api.Job) string); ok {
+		r0 = rf(job)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(*api.Job) error); ok {
+		r1 = rf(job)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// RegisterRunnerJob provides a mock function with given fields: template
+func (_m *ExecutorAPIMock) RegisterRunnerJob(template *api.Job) error {
+	ret := _m.Called(template)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(*api.Job) error); ok {
+		r0 = rf(template)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// RegisterTemplateJob provides a mock function with given fields: defaultJob, id, prewarmingPoolSize, cpuLimit, memoryLimit, image, networkAccess, exposedPorts
+func (_m *ExecutorAPIMock) RegisterTemplateJob(defaultJob *api.Job, id string, prewarmingPoolSize uint, cpuLimit uint, memoryLimit uint, image string, networkAccess bool, exposedPorts []uint16) (*api.Job, error) {
+	ret := _m.Called(defaultJob, id, prewarmingPoolSize, cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
+
+	var r0 *api.Job
+	if rf, ok := ret.Get(0).(func(*api.Job, string, uint, uint, uint, string, bool, []uint16) *api.Job); ok {
+		r0 = rf(defaultJob, id, prewarmingPoolSize, cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*api.Job)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(*api.Job, string, uint, uint, uint, string, bool, []uint16) error); ok {
+		r1 = rf(defaultJob, id, prewarmingPoolSize, cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SetJobScale provides a mock function with given fields: jobID, count, reason
+func (_m *ExecutorAPIMock) SetJobScale(jobId string, count uint, reason string) error {
+	ret := _m.Called(jobId, count, reason)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, uint, string) error); ok {
+		r0 = rf(jobId, count, reason)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// WatchAllocations provides a mock function with given fields: ctx, onNewAllocation, onDeletedAllocation
+func (_m *ExecutorAPIMock) WatchAllocations(ctx context.Context, onNewAllocation AllocationProcessor, onDeletedAllocation AllocationProcessor) error {
+	ret := _m.Called(ctx, onNewAllocation, onDeletedAllocation)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, AllocationProcessor, AllocationProcessor) error); ok {
+		r0 = rf(ctx, onNewAllocation, onDeletedAllocation)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// allocation provides a mock function with given fields: jobID
+func (_m *ExecutorAPIMock) allocation(jobID string) (*api.Allocation, error) {
+	ret := _m.Called(jobID)
+
+	var r0 *api.Allocation
+	if rf, ok := ret.Get(0).(func(string) *api.Allocation); ok {
+		r0 = rf(jobID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*api.Allocation)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(jobID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// init provides a mock function with given fields: nomadURL, nomadNamespace
+func (_m *ExecutorAPIMock) init(nomadURL *url.URL, nomadNamespace string) error {
+	ret := _m.Called(nomadURL, nomadNamespace)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(*url.URL, string) error); ok {
+		r0 = rf(nomadURL, nomadNamespace)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// job provides a mock function with given fields: jobID
+func (_m *ExecutorAPIMock) job(jobID string) (*api.Job, error) {
+	ret := _m.Called(jobID)
+
+	var r0 *api.Job
+	if rf, ok := ret.Get(0).(func(string) *api.Job); ok {
+		r0 = rf(jobID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*api.Job)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(jobID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// listJobs provides a mock function with given fields: prefix
+func (_m *ExecutorAPIMock) listJobs(prefix string) ([]*api.JobListStub, error) {
+	ret := _m.Called(prefix)
+
+	var r0 []*api.JobListStub
+	if rf, ok := ret.Get(0).(func(string) []*api.JobListStub); ok {
+		r0 = rf(prefix)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*api.JobListStub)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(prefix)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

internal/nomad/job.go

@@ -0,0 +1,237 @@
+package nomad
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	nomadApi "github.com/hashicorp/nomad/api"
+	"strconv"
+)
+
+const (
+	TemplateJobPrefix     = "template"
+	TaskGroupName         = "default-group"
+	TaskName              = "default-task"
+	TaskCount             = 1
+	TaskDriver            = "docker"
+	TaskCommand           = "sleep"
+	ConfigTaskGroupName   = "config"
+	ConfigTaskName        = "config"
+	ConfigTaskDriver      = "exec"
+	ConfigTaskCommand     = "true"
+	ConfigMetaUsedKey     = "used"
+	ConfigMetaUsedValue   = "true"
+	ConfigMetaUnusedValue = "false"
+	ConfigMetaTimeoutKey  = "timeout"
+	ConfigMetaPoolSizeKey = "prewarmingPoolSize"
+)
+
+var (
+	TaskArgs                     = []string{"infinity"}
+	ErrorConfigTaskGroupNotFound = errors.New("config task group not found in job")
+)
+
+// FindConfigTaskGroup returns the config task group of a job.
+// The config task group should be included in all jobs.
+func FindConfigTaskGroup(job *nomadApi.Job) *nomadApi.TaskGroup {
+	for _, tg := range job.TaskGroups {
+		if *tg.Name == ConfigTaskGroupName {
+			return tg
+		}
+	}
+	return nil
+}
+
+func SetMetaConfigValue(job *nomadApi.Job, key, value string) error {
+	configTaskGroup := FindConfigTaskGroup(job)
+	if configTaskGroup == nil {
+		return ErrorConfigTaskGroupNotFound
+	}
+	configTaskGroup.Meta[key] = value
+	return nil
+}
+
+// RegisterTemplateJob creates a Nomad job based on the default job configuration and the given parameters.
+// It registers the job with Nomad and waits until the registration completes.
+func (a *APIClient) RegisterTemplateJob(
+	basisJob *nomadApi.Job,
+	id string,
+	prewarmingPoolSize, cpuLimit, memoryLimit uint,
+	image string,
+	networkAccess bool,
+	exposedPorts []uint16) (*nomadApi.Job, error) {
+	job := CreateTemplateJob(basisJob, id, prewarmingPoolSize,
+		cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
+	evalID, err := a.apiQuerier.RegisterNomadJob(job)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't register template job: %w", err)
+	}
+	return job, a.MonitorEvaluation(evalID, context.Background())
+}
+
+// CreateTemplateJob creates a Nomad job based on the default job configuration and the given parameters.
+// It registers the job with Nomad and waits until the registration completes.
+func CreateTemplateJob(
+	basisJob *nomadApi.Job,
+	id string,
+	prewarmingPoolSize, cpuLimit, memoryLimit uint,
+	image string,
+	networkAccess bool,
+	exposedPorts []uint16) *nomadApi.Job {
+	job := *basisJob
+	job.ID = &id
+	job.Name = &id
+
+	var taskGroup = createTaskGroup(&job, TaskGroupName)
+	configureTask(taskGroup, TaskName, cpuLimit, memoryLimit, image, networkAccess, exposedPorts)
+	storeTemplateConfiguration(&job, prewarmingPoolSize)
+
+	return &job
+}
+
+func (a *APIClient) RegisterRunnerJob(template *nomadApi.Job) error {
+	storeRunnerConfiguration(template)
+
+	evalID, err := a.apiQuerier.RegisterNomadJob(template)
+	if err != nil {
+		return fmt.Errorf("couldn't register runner job: %w", err)
+	}
+	return a.MonitorEvaluation(evalID, context.Background())
+}
+
+func createTaskGroup(job *nomadApi.Job, name string) *nomadApi.TaskGroup {
+	var taskGroup *nomadApi.TaskGroup
+	if len(job.TaskGroups) == 0 {
+		taskGroup = nomadApi.NewTaskGroup(name, TaskCount)
+		job.TaskGroups = []*nomadApi.TaskGroup{taskGroup}
+	} else {
+		taskGroup = job.TaskGroups[0]
+		taskGroup.Name = &name
+		count := TaskCount
+		taskGroup.Count = &count
+	}
+	return taskGroup
+}
+
+func configureNetwork(taskGroup *nomadApi.TaskGroup, networkAccess bool, exposedPorts []uint16) {
+	if len(taskGroup.Tasks) == 0 {
+		// This function is only used internally and must be called as last step when configuring the task.
+		// This error is not recoverable.
+		log.Fatal("Can't configure network before task has been configured!")
+	}
+	task := taskGroup.Tasks[0]
+
+	if task.Config == nil {
+		task.Config = make(map[string]interface{})
+	}
+
+	if networkAccess {
+		var networkResource *nomadApi.NetworkResource
+		if len(taskGroup.Networks) == 0 {
+			networkResource = &nomadApi.NetworkResource{}
+			taskGroup.Networks = []*nomadApi.NetworkResource{networkResource}
+		} else {
+			networkResource = taskGroup.Networks[0]
+		}
+		// Prefer "bridge" network over "host" to have an isolated network namespace with bridged interface
+		// instead of joining the host network namespace.
+		networkResource.Mode = "bridge"
+		for _, portNumber := range exposedPorts {
+			port := nomadApi.Port{
+				Label: strconv.FormatUint(uint64(portNumber), 10),
+				To:    int(portNumber),
+			}
+			networkResource.DynamicPorts = append(networkResource.DynamicPorts, port)
+		}
+
+		// Explicitly set mode to override existing settings when updating job from without to with network.
+		// Don't use bridge as it collides with the bridge mode above. This results in Docker using 'bridge'
+		// mode, meaning all allocations will be attached to the `docker0` adapter and could reach other
+		// non-Nomad containers attached to it. This is avoided when using Nomads bridge network mode.
+		task.Config["network_mode"] = ""
+	} else {
+		// Somehow, we can't set the network mode to none in the NetworkResource on task group level.
+		// See https://github.com/hashicorp/nomad/issues/10540
+		task.Config["network_mode"] = "none"
+		// Explicitly set Networks to signal Nomad to remove the possibly existing networkResource
+		taskGroup.Networks = []*nomadApi.NetworkResource{}
+	}
+}
+
+func configureTask(
+	taskGroup *nomadApi.TaskGroup,
+	name string,
+	cpuLimit, memoryLimit uint,
+	image string,
+	networkAccess bool,
+	exposedPorts []uint16) {
+	var task *nomadApi.Task
+	if len(taskGroup.Tasks) == 0 {
+		task = nomadApi.NewTask(name, TaskDriver)
+		taskGroup.Tasks = []*nomadApi.Task{task}
+	} else {
+		task = taskGroup.Tasks[0]
+		task.Name = name
+	}
+	integerCPULimit := int(cpuLimit)
+	integerMemoryLimit := int(memoryLimit)
+	if task.Resources == nil {
+		task.Resources = nomadApi.DefaultResources()
+	}
+	task.Resources.CPU = &integerCPULimit
+	task.Resources.MemoryMB = &integerMemoryLimit
+
+	if task.Config == nil {
+		task.Config = make(map[string]interface{})
+	}
+	task.Config["image"] = image
+	task.Config["command"] = TaskCommand
+	task.Config["args"] = TaskArgs
+
+	configureNetwork(taskGroup, networkAccess, exposedPorts)
+}
+
+func storeTemplateConfiguration(job *nomadApi.Job, prewarmingPoolSize uint) {
+	taskGroup := findOrCreateConfigTaskGroup(job)
+
+	taskGroup.Meta = make(map[string]string)
+	taskGroup.Meta[ConfigMetaPoolSizeKey] = strconv.Itoa(int(prewarmingPoolSize))
+}
+
+func storeRunnerConfiguration(job *nomadApi.Job) {
+	taskGroup := findOrCreateConfigTaskGroup(job)
+
+	taskGroup.Meta = make(map[string]string)
+	taskGroup.Meta[ConfigMetaUsedKey] = ConfigMetaUnusedValue
+}
+
+func findOrCreateConfigTaskGroup(job *nomadApi.Job) *nomadApi.TaskGroup {
+	taskGroup := FindConfigTaskGroup(job)
+	if taskGroup == nil {
+		taskGroup = nomadApi.NewTaskGroup(ConfigTaskGroupName, 0)
+		job.AddTaskGroup(taskGroup)
+	}
+	createConfigTaskIfNotPresent(taskGroup)
+	return taskGroup
+}
+
+// createConfigTaskIfNotPresent ensures that a dummy task is in the task group so that the group is accepted by Nomad.
+func createConfigTaskIfNotPresent(taskGroup *nomadApi.TaskGroup) {
+	var task *nomadApi.Task
+	for _, t := range taskGroup.Tasks {
+		if t.Name == ConfigTaskName {
+			task = t
+			break
+		}
+	}
+
+	if task == nil {
+		task = nomadApi.NewTask(ConfigTaskName, ConfigTaskDriver)
+		taskGroup.Tasks = append(taskGroup.Tasks, task)
+	}
+
+	if task.Config == nil {
+		task.Config = make(map[string]interface{})
+	}
+	task.Config["command"] = ConfigTaskCommand
+}

internal/nomad/job_test.go

@@ -0,0 +1,279 @@
+package nomad
+
+import (
+	"fmt"
+	nomadApi "github.com/hashicorp/nomad/api"
+	"github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/helpers"
+	"strconv"
+	"testing"
+)
+
+func createTestTaskGroup() *nomadApi.TaskGroup {
+	return nomadApi.NewTaskGroup("taskGroup", 1)
+}
+
+func createTestTask() *nomadApi.Task {
+	return nomadApi.NewTask("task", "docker")
+}
+
+func createTestResources() *nomadApi.Resources {
+	result := nomadApi.DefaultResources()
+	expectedCPULimit := 1337
+	expectedMemoryLimit := 42
+	result.CPU = &expectedCPULimit
+	result.MemoryMB = &expectedMemoryLimit
+	return result
+}
+
+func TestCreateTaskGroupCreatesNewTaskGroupWhenJobHasNoTaskGroup(t *testing.T) {
+	job := nomadApi.NewBatchJob("test", "test", "test", 1)
+
+	if assert.Equal(t, 0, len(job.TaskGroups)) {
+		expectedTaskGroup := createTestTaskGroup()
+		taskGroup := createTaskGroup(job, *expectedTaskGroup.Name)
+
+		assert.Equal(t, *expectedTaskGroup, *taskGroup)
+		assert.Equal(t, []*nomadApi.TaskGroup{taskGroup}, job.TaskGroups, "it should add the task group to the job")
+	}
+}
+
+func TestCreateTaskGroupOverwritesOptionsWhenJobHasTaskGroup(t *testing.T) {
+	job := nomadApi.NewBatchJob("test", "test", "test", 1)
+	existingTaskGroup := createTestTaskGroup()
+	existingTaskGroup.Meta = map[string]string{"field": "should still exist"}
+	newTaskGroupList := []*nomadApi.TaskGroup{existingTaskGroup}
+	job.TaskGroups = newTaskGroupList
+
+	newName := *existingTaskGroup.Name + "longerName"
+	taskGroup := createTaskGroup(job, newName)
+
+	// create a new copy to avoid changing the original one as it is a pointer
+	expectedTaskGroup := *existingTaskGroup
+	expectedTaskGroup.Name = &newName
+
+	assert.Equal(t, expectedTaskGroup, *taskGroup)
+	assert.Equal(t, newTaskGroupList, job.TaskGroups, "it should not modify the jobs task group list")
+}
+
+func TestConfigureNetworkFatalsWhenNoTaskExists(t *testing.T) {
+	logger, hook := test.NewNullLogger()
+	logger.ExitFunc = func(i int) {
+		panic(i)
+	}
+	log = logger.WithField("pkg", "job_test")
+	taskGroup := createTestTaskGroup()
+	if assert.Equal(t, 0, len(taskGroup.Tasks)) {
+		assert.Panics(t, func() {
+			configureNetwork(taskGroup, false, nil)
+		})
+		assert.Equal(t, logrus.FatalLevel, hook.LastEntry().Level)
+	}
+}
+
+func TestConfigureNetworkCreatesNewNetworkWhenNoNetworkExists(t *testing.T) {
+	taskGroup := createTestTaskGroup()
+	task := createTestTask()
+	taskGroup.Tasks = []*nomadApi.Task{task}
+
+	if assert.Equal(t, 0, len(taskGroup.Networks)) {
+		configureNetwork(taskGroup, true, []uint16{})
+
+		assert.Equal(t, 1, len(taskGroup.Networks))
+	}
+}
+
+func TestConfigureNetworkDoesNotCreateNewNetworkWhenNetworkExists(t *testing.T) {
+	taskGroup := createTestTaskGroup()
+	task := createTestTask()
+	taskGroup.Tasks = []*nomadApi.Task{task}
+	networkResource := &nomadApi.NetworkResource{Mode: "bridge"}
+	taskGroup.Networks = []*nomadApi.NetworkResource{networkResource}
+
+	if assert.Equal(t, 1, len(taskGroup.Networks)) {
+		configureNetwork(taskGroup, true, []uint16{})
+
+		assert.Equal(t, 1, len(taskGroup.Networks))
+		assert.Equal(t, networkResource, taskGroup.Networks[0])
+	}
+}
+
+func TestConfigureNetworkSetsCorrectValues(t *testing.T) {
+	taskGroup := createTestTaskGroup()
+	task := createTestTask()
+	_, ok := task.Config["network_mode"]
+
+	require.False(t, ok, "Test tasks network_mode should not be set")
+
+	taskGroup.Tasks = []*nomadApi.Task{task}
+	exposedPortsTests := [][]uint16{{}, {1337}, {42, 1337}}
+
+	t.Run("with no network access", func(t *testing.T) {
+		for _, ports := range exposedPortsTests {
+			testTaskGroup := *taskGroup
+			testTask := *task
+			testTaskGroup.Tasks = []*nomadApi.Task{&testTask}
+
+			configureNetwork(&testTaskGroup, false, ports)
+			mode, ok := testTask.Config["network_mode"]
+			assert.True(t, ok)
+			assert.Equal(t, "none", mode)
+			assert.Equal(t, 0, len(testTaskGroup.Networks))
+		}
+	})
+
+	t.Run("with network access", func(t *testing.T) {
+		for _, ports := range exposedPortsTests {
+			testTaskGroup := *taskGroup
+			testTask := *task
+			testTaskGroup.Tasks = []*nomadApi.Task{&testTask}
+
+			configureNetwork(&testTaskGroup, true, ports)
+			require.Equal(t, 1, len(testTaskGroup.Networks))
+
+			networkResource := testTaskGroup.Networks[0]
+			assert.Equal(t, "bridge", networkResource.Mode)
+			require.Equal(t, len(ports), len(networkResource.DynamicPorts))
+
+			assertExpectedPorts(t, ports, networkResource)
+
+			mode, ok := testTask.Config["network_mode"]
+			assert.True(t, ok)
+			assert.Equal(t, mode, "")
+		}
+	})
+}
+
+func assertExpectedPorts(t *testing.T, expectedPorts []uint16, networkResource *nomadApi.NetworkResource) {
+	t.Helper()
+	for _, expectedPort := range expectedPorts {
+		found := false
+		for _, actualPort := range networkResource.DynamicPorts {
+			if actualPort.To == int(expectedPort) {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, fmt.Sprintf("port list should contain %v", expectedPort))
+	}
+}
+
+func TestConfigureTaskWhenNoTaskExists(t *testing.T) {
+	taskGroup := createTestTaskGroup()
+	require.Equal(t, 0, len(taskGroup.Tasks))
+
+	expectedResources := createTestResources()
+	expectedTaskGroup := *taskGroup
+	expectedTask := nomadApi.NewTask("task", TaskDriver)
+	expectedTask.Resources = expectedResources
+	expectedImage := "python:latest"
+	expectedCommand := "sleep"
+	expectedArgs := []string{"infinity"}
+	expectedTask.Config = map[string]interface{}{
+		"image": expectedImage, "command": expectedCommand, "args": expectedArgs, "network_mode": "none"}
+	expectedTaskGroup.Tasks = []*nomadApi.Task{expectedTask}
+	expectedTaskGroup.Networks = []*nomadApi.NetworkResource{}
+
+	configureTask(taskGroup, expectedTask.Name,
+		uint(*expectedResources.CPU), uint(*expectedResources.MemoryMB),
+		expectedImage, false, []uint16{})
+
+	assert.Equal(t, expectedTaskGroup, *taskGroup)
+}
+
+func TestConfigureTaskWhenTaskExists(t *testing.T) {
+	taskGroup := createTestTaskGroup()
+	task := createTestTask()
+	task.Config = map[string]interface{}{"my_custom_config": "should not be overwritten"}
+	taskGroup.Tasks = []*nomadApi.Task{task}
+	require.Equal(t, 1, len(taskGroup.Tasks))
+
+	expectedResources := createTestResources()
+	expectedTaskGroup := *taskGroup
+	expectedTask := *task
+	expectedTask.Resources = expectedResources
+	expectedImage := "python:latest"
+	expectedTask.Config["image"] = expectedImage
+	expectedTask.Config["network_mode"] = "none"
+	expectedTaskGroup.Tasks = []*nomadApi.Task{&expectedTask}
+	expectedTaskGroup.Networks = []*nomadApi.NetworkResource{}
+
+	configureTask(taskGroup, expectedTask.Name,
+		uint(*expectedResources.CPU), uint(*expectedResources.MemoryMB),
+		expectedImage, false, []uint16{})
+
+	assert.Equal(t, expectedTaskGroup, *taskGroup)
+	assert.Equal(t, task, taskGroup.Tasks[0], "it should not create a new task")
+}
+
+func TestCreateTemplateJobSetsAllGivenArguments(t *testing.T) {
+	base, testJob := helpers.CreateTemplateJob()
+	prewarmingPoolSize, err := strconv.Atoi(testJob.TaskGroups[1].Meta[ConfigMetaPoolSizeKey])
+	require.NoError(t, err)
+	job := CreateTemplateJob(
+		base,
+		tests.DefaultJobID,
+		uint(prewarmingPoolSize),
+		uint(*testJob.TaskGroups[0].Tasks[0].Resources.CPU),
+		uint(*testJob.TaskGroups[0].Tasks[0].Resources.MemoryMB),
+		testJob.TaskGroups[0].Tasks[0].Config["image"].(string),
+		false,
+		nil,
+	)
+	assert.Equal(t, *testJob, *job)
+}
+
+func TestRegisterTemplateJobFailsWhenNomadJobRegistrationFails(t *testing.T) {
+	apiMock := apiQuerierMock{}
+	expectedErr := tests.ErrDefault
+
+	apiMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return("", expectedErr)
+
+	apiClient := &APIClient{&apiMock}
+
+	_, err := apiClient.RegisterTemplateJob(&nomadApi.Job{}, tests.DefaultJobID,
+		1, 2, 3, "image", false, []uint16{})
+	assert.ErrorIs(t, err, expectedErr)
+	apiMock.AssertNotCalled(t, "EvaluationStream")
+}
+
+func TestRegisterTemplateJobSucceedsWhenMonitoringEvaluationSucceeds(t *testing.T) {
+	apiMock := apiQuerierMock{}
+	evaluationID := "id"
+
+	stream := make(chan *nomadApi.Events)
+	readonlyStream := func() <-chan *nomadApi.Events {
+		return stream
+	}()
+	// Immediately close stream to avoid any reading from it resulting in endless wait
+	close(stream)
+
+	apiMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return(evaluationID, nil)
+	apiMock.On("EvaluationStream", evaluationID, mock.AnythingOfType("*context.emptyCtx")).
+		Return(readonlyStream, nil)
+
+	apiClient := &APIClient{&apiMock}
+
+	_, err := apiClient.RegisterTemplateJob(&nomadApi.Job{}, tests.DefaultJobID,
+		1, 2, 3, "image", false, []uint16{})
+	assert.NoError(t, err)
+}
+
+func TestRegisterTemplateJobReturnsErrorWhenMonitoringEvaluationFails(t *testing.T) {
+	apiMock := apiQuerierMock{}
+	evaluationID := "id"
+
+	apiMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return(evaluationID, nil)
+	apiMock.On("EvaluationStream", evaluationID, mock.AnythingOfType("*context.emptyCtx")).Return(nil, tests.ErrDefault)
+
+	apiClient := &APIClient{&apiMock}
+
+	_, err := apiClient.RegisterTemplateJob(&nomadApi.Job{}, tests.DefaultJobID,
+		1, 2, 3, "image", false, []uint16{})
+	assert.ErrorIs(t, err, tests.ErrDefault)
+}

internal/nomad/nomad.go

@@ -0,0 +1,392 @@
+package nomad
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	nomadApi "github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/internal/config"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/pkg/logging"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/pkg/nullreader"
+	"io"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+var (
+	log                              = logging.GetLogger("nomad")
+	ErrorExecutorCommunicationFailed = errors.New("communication with executor failed")
+	ErrorEvaluation                  = errors.New("evaluation could not complete")
+	ErrorPlacingAllocations          = errors.New("failed to place all allocations")
+	ErrorLoadingJob                  = errors.New("failed to load job")
+	ErrorNoAllocatedResourcesFound   = errors.New("no allocated resources found")
+)
+
+type AllocationProcessor func(*nomadApi.Allocation)
+
+// ExecutorAPI provides access to an container orchestration solution.
+type ExecutorAPI interface {
+	apiQuerier
+
+	// LoadEnvironmentJobs loads all environment jobs.
+	LoadEnvironmentJobs() ([]*nomadApi.Job, error)
+
+	// LoadRunnerJobs loads all runner jobs specific for the environment.
+	LoadRunnerJobs(environmentID string) ([]*nomadApi.Job, error)
+
+	// LoadRunnerIDs returns the IDs of all runners of the specified environment which are running and not about to
+	// get stopped.
+	LoadRunnerIDs(environmentID string) (runnerIds []string, err error)
+
+	// LoadRunnerPortMappings returns the mapped ports of the runner.
+	LoadRunnerPortMappings(runnerID string) ([]nomadApi.PortMapping, error)
+
+	// RegisterTemplateJob creates a template job based on the default job configuration and the given parameters.
+	// It registers the job and waits until the registration completes.
+	RegisterTemplateJob(defaultJob *nomadApi.Job, id string,
+		prewarmingPoolSize, cpuLimit, memoryLimit uint,
+		image string, networkAccess bool, exposedPorts []uint16) (*nomadApi.Job, error)
+
+	// RegisterRunnerJob creates a runner job based on the template job.
+	// It registers the job and waits until the registration completes.
+	RegisterRunnerJob(template *nomadApi.Job) error
+
+	// MonitorEvaluation monitors the given evaluation ID.
+	// It waits until the evaluation reaches one of the states complete, canceled or failed.
+	// If the evaluation was not successful, an error containing the failures is returned.
+	// See also https://github.com/hashicorp/nomad/blob/7d5a9ecde95c18da94c9b6ace2565afbfdd6a40d/command/monitor.go#L175
+	MonitorEvaluation(evaluationID string, ctx context.Context) error
+
+	// WatchAllocations listens on the Nomad event stream for allocation events.
+	// Depending on the incoming event, any of the given function is executed.
+	WatchAllocations(ctx context.Context, onNewAllocation, onDeletedAllocation AllocationProcessor) error
+
+	// ExecuteCommand executes the given command in the allocation with the given id.
+	// It writes the output of the command to stdout/stderr and reads input from stdin.
+	// If tty is true, the command will run with a tty.
+	ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool,
+		stdin io.Reader, stdout, stderr io.Writer) (int, error)
+
+	// MarkRunnerAsUsed marks the runner with the given ID as used. It also stores the timeout duration in the metadata.
+	MarkRunnerAsUsed(runnerID string, duration int) error
+}
+
+// APIClient implements the ExecutorAPI interface and can be used to perform different operations on the real
+// Executor API and its return values.
+type APIClient struct {
+	apiQuerier
+}
+
+// NewExecutorAPI creates a new api client.
+// One client is usually sufficient for the complete runtime of the API.
+func NewExecutorAPI(nomadURL *url.URL, nomadNamespace string) (ExecutorAPI, error) {
+	client := &APIClient{apiQuerier: &nomadAPIClient{}}
+	err := client.init(nomadURL, nomadNamespace)
+	return client, err
+}
+
+// init prepares an apiClient to be able to communicate to a provided Nomad API.
+func (a *APIClient) init(nomadURL *url.URL, nomadNamespace string) error {
+	if err := a.apiQuerier.init(nomadURL, nomadNamespace); err != nil {
+		return fmt.Errorf("error initializing API querier: %w", err)
+	}
+	return nil
+}
+
+func (a *APIClient) LoadRunnerIDs(environmentID string) (runnerIDs []string, err error) {
+	list, err := a.listJobs(environmentID)
+	if err != nil {
+		return nil, err
+	}
+	for _, jobListStub := range list {
+		allocationRunning := jobListStub.JobSummary.Summary[TaskGroupName].Running > 0
+		if jobListStub.Status == structs.JobStatusRunning && allocationRunning {
+			runnerIDs = append(runnerIDs, jobListStub.ID)
+		}
+	}
+	return runnerIDs, nil
+}
+
+func (a *APIClient) LoadRunnerPortMappings(runnerID string) ([]nomadApi.PortMapping, error) {
+	alloc, err := a.apiQuerier.allocation(runnerID)
+	if err != nil {
+		return nil, fmt.Errorf("error querying allocation for runner %s: %w", runnerID, err)
+	}
+	if alloc.AllocatedResources == nil {
+		return nil, ErrorNoAllocatedResourcesFound
+	}
+	return alloc.AllocatedResources.Shared.Ports, nil
+}
+
+func (a *APIClient) LoadRunnerJobs(environmentID string) ([]*nomadApi.Job, error) {
+	runnerIDs, err := a.LoadRunnerIDs(environmentID)
+	if err != nil {
+		return []*nomadApi.Job{}, fmt.Errorf("couldn't load jobs: %w", err)
+	}
+
+	var occurredError error
+	jobs := make([]*nomadApi.Job, 0, len(runnerIDs))
+	for _, id := range runnerIDs {
+		job, err := a.apiQuerier.job(id)
+		if err != nil {
+			if occurredError == nil {
+				occurredError = ErrorLoadingJob
+			}
+			occurredError = fmt.Errorf("%w: couldn't load job info for runner %s - %v", occurredError, id, err)
+			continue
+		}
+		jobs = append(jobs, job)
+	}
+	return jobs, occurredError
+}
+
+func (a *APIClient) MonitorEvaluation(evaluationID string, ctx context.Context) error {
+	stream, err := a.apiQuerier.EvaluationStream(evaluationID, ctx)
+	if err != nil {
+		return fmt.Errorf("failed retrieving evaluation stream: %w", err)
+	}
+	// If ctx is canceled, the stream will be closed by Nomad and we exit the for loop.
+	return receiveAndHandleNomadAPIEvents(stream, handleEvaluationEvent)
+}
+
+func (a *APIClient) WatchAllocations(ctx context.Context,
+	onNewAllocation, onDeletedAllocation AllocationProcessor) error {
+	startTime := time.Now().UnixNano()
+	stream, err := a.AllocationStream(ctx)
+	if err != nil {
+		return fmt.Errorf("failed retrieving allocation stream: %w", err)
+	}
+	pendingAllocations := make(map[string]bool)
+
+	handler := func(event *nomadApi.Event) (bool, error) {
+		return false, handleAllocationEvent(startTime, pendingAllocations, event, onNewAllocation, onDeletedAllocation)
+	}
+
+	err = receiveAndHandleNomadAPIEvents(stream, handler)
+	return err
+}
+
+// nomadAPIEventHandler is a function that receives a nomadApi.Event and processes it.
+// It is called by an event listening loop. For each received event, the function is called.
+// If done is true, the calling function knows that it should break out of the event listening
+// loop.
+type nomadAPIEventHandler func(event *nomadApi.Event) (done bool, err error)
+
+// receiveAndHandleNomadAPIEvents receives events from the Nomad event stream and calls the handler function for
+// each received event. It skips heartbeat events and returns an error if the received events contain an error.
+func receiveAndHandleNomadAPIEvents(stream <-chan *nomadApi.Events, handler nomadAPIEventHandler) error {
+	// If original context is canceled, the stream will be closed by Nomad and we exit the for loop.
+	for events := range stream {
+		if events.IsHeartbeat() {
+			continue
+		}
+		if err := events.Err; err != nil {
+			return fmt.Errorf("error receiving events: %w", err)
+		}
+		for _, event := range events.Events {
+			// Don't take the address of the loop variable as the underlying value might change
+			eventCopy := event
+			done, err := handler(&eventCopy)
+			if err != nil || done {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// handleEvaluationEvent is a nomadAPIEventHandler that returns whether the evaluation described by the event
+// was successful.
+func handleEvaluationEvent(event *nomadApi.Event) (bool, error) {
+	eval, err := event.Evaluation()
+	if err != nil {
+		return true, fmt.Errorf("failed to monitor evaluation: %w", err)
+	}
+	switch eval.Status {
+	case structs.EvalStatusComplete, structs.EvalStatusCancelled, structs.EvalStatusFailed:
+		return true, checkEvaluation(eval)
+	}
+	return false, nil
+}
+
+// handleAllocationEvent is a nomadAPIEventHandler that processes allocation events.
+// If a new allocation is received, onNewAllocation is called. If an allocation is deleted, onDeletedAllocation
+// is called. The pendingAllocations map is used to store allocations that are pending but not started yet. Using the
+// map the state is persisted between multiple calls of this function.
+func handleAllocationEvent(startTime int64, pendingAllocations map[string]bool, event *nomadApi.Event,
+	onNewAllocation, onDeletedAllocation AllocationProcessor) error {
+	if event.Type != structs.TypeAllocationUpdated {
+		return nil
+	}
+	alloc, err := event.Allocation()
+	if err != nil {
+		return fmt.Errorf("failed to retrieve allocation from event: %w", err)
+	} else if alloc == nil {
+		return nil
+	}
+
+	// When starting the API and listening on the Nomad event stream we might get events that already
+	// happened from Nomad as it seems to buffer them for a certain duration.
+	// Ignore old events here.
+	if alloc.ModifyTime < startTime {
+		return nil
+	}
+
+	if alloc.ClientStatus == structs.AllocClientStatusRunning {
+		switch alloc.DesiredStatus {
+		case structs.AllocDesiredStatusStop:
+			onDeletedAllocation(alloc)
+		case structs.AllocDesiredStatusRun:
+			// is first event that marks the transition between pending and running?
+			_, ok := pendingAllocations[alloc.ID]
+			if ok {
+				onNewAllocation(alloc)
+				delete(pendingAllocations, alloc.ID)
+			}
+		}
+	}
+
+	if alloc.ClientStatus == structs.AllocClientStatusPending && alloc.DesiredStatus == structs.AllocDesiredStatusRun {
+		// allocation is started, wait until it runs and add to our list afterwards
+		pendingAllocations[alloc.ID] = true
+	}
+	return nil
+}
+
+// checkEvaluation checks whether the given evaluation failed.
+// If the evaluation failed, it returns an error with a message containing the failure information.
+func checkEvaluation(eval *nomadApi.Evaluation) (err error) {
+	if len(eval.FailedTGAllocs) == 0 {
+		if eval.Status != structs.EvalStatusComplete {
+			err = fmt.Errorf("%w: %q", ErrorEvaluation, eval.Status)
+		}
+	} else {
+		err = fmt.Errorf("evaluation %q finished with status %q but %w", eval.ID, eval.Status, ErrorPlacingAllocations)
+		for taskGroup, metrics := range eval.FailedTGAllocs {
+			err = fmt.Errorf("%w\n%s: %#v", err, taskGroup, metrics)
+		}
+		if eval.BlockedEval != "" {
+			err = fmt.Errorf("%w\nEvaluation %q waiting for additional capacity to place remainder", err, eval.BlockedEval)
+		}
+	}
+	return err
+}
+
+func (a *APIClient) MarkRunnerAsUsed(runnerID string, duration int) error {
+	job, err := a.job(runnerID)
+	if err != nil {
+		return fmt.Errorf("couldn't retrieve job info: %w", err)
+	}
+	err = SetMetaConfigValue(job, ConfigMetaUsedKey, ConfigMetaUsedValue)
+	if err != nil {
+		return fmt.Errorf("couldn't update runner in job as used: %w", err)
+	}
+	err = SetMetaConfigValue(job, ConfigMetaTimeoutKey, strconv.Itoa(duration))
+	if err != nil {
+		return fmt.Errorf("couldn't update runner in job with timeout: %w", err)
+	}
+	_, err = a.RegisterNomadJob(job)
+	if err != nil {
+		return fmt.Errorf("couldn't update runner config: %w", err)
+	}
+	return nil
+}
+
+func (a *APIClient) LoadEnvironmentJobs() ([]*nomadApi.Job, error) {
+	jobStubs, err := a.listJobs(TemplateJobPrefix)
+	if err != nil {
+		return []*nomadApi.Job{}, fmt.Errorf("couldn't load jobs: %w", err)
+	}
+
+	jobs := make([]*nomadApi.Job, 0, len(jobStubs))
+	for _, jobStub := range jobStubs {
+		job, err := a.apiQuerier.job(jobStub.ID)
+		if err != nil {
+			return []*nomadApi.Job{}, fmt.Errorf("couldn't load job info for job %v: %w", jobStub.ID, err)
+		}
+		jobs = append(jobs, job)
+	}
+	return jobs, nil
+}
+
+// ExecuteCommand executes the given command in the given allocation.
+// If tty is true, Nomad would normally write stdout and stderr of the command
+// both on the stdout stream. However, if the InteractiveStderr server config option is true,
+// we make sure that stdout and stderr are split correctly.
+// In order for the stderr splitting to work, the command must have the structure
+// []string{..., "sh", "-c", "my-command"}.
+func (a *APIClient) ExecuteCommand(allocationID string,
+	ctx context.Context, command []string, tty bool,
+	stdin io.Reader, stdout, stderr io.Writer) (int, error) {
+	if tty && config.Config.Server.InteractiveStderr {
+		return a.executeCommandInteractivelyWithStderr(allocationID, ctx, command, stdin, stdout, stderr)
+	}
+	exitCode, err := a.apiQuerier.Execute(allocationID, ctx, command, tty, stdin, stdout, stderr)
+	if err != nil {
+		return 1, fmt.Errorf("error executing command in API: %w", err)
+	}
+	return exitCode, nil
+}
+
+// executeCommandInteractivelyWithStderr executes the given command interactively and splits stdout
+// and stderr correctly. Normally, using Nomad to execute a command with tty=true (in order to have
+// an interactive connection and possibly a fully working shell), would result in stdout and stderr
+// to be served both over stdout. This function circumvents this by creating a fifo for the stderr
+// of the command and starting a second execution that reads the stderr from that fifo.
+func (a *APIClient) executeCommandInteractivelyWithStderr(allocationID string, ctx context.Context,
+	command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) {
+	// Use current nano time to make the stderr fifo kind of unique.
+	currentNanoTime := time.Now().UnixNano()
+	// We expect the command to be like []string{..., "sh", "-c", "my-command"}.
+	oldCommand := command[len(command)-1]
+	// Take the last command which is the one to be executed and wrap it to redirect stderr.
+	command[len(command)-1] = wrapCommandForStderrFifo(currentNanoTime, oldCommand)
+
+	stderrExitChan := make(chan int)
+	go func() {
+		// Catch stderr in separate execution.
+		exit, err := a.Execute(allocationID, ctx, stderrFifoCommand(currentNanoTime), true,
+			nullreader.NullReader{}, stderr, io.Discard)
+		if err != nil {
+			log.WithError(err).WithField("runner", allocationID).Warn("Stderr task finished with error")
+		}
+		stderrExitChan <- exit
+	}()
+
+	exit, err := a.Execute(allocationID, ctx, command, true, stdin, stdout, io.Discard)
+
+	// Wait until the stderr catch command finished to make sure we receive all output.
+	<-stderrExitChan
+	return exit, err
+}
+
+const (
+	// stderrFifoFormat represents the format we use for our stderr fifos. The %d should be unique for the execution
+	// as otherwise multiple executions are not possible.
+	// Example: "/tmp/stderr_1623330777825234133.fifo".
+	stderrFifoFormat = "/tmp/stderr_%d.fifo"
+	// stderrFifoCommandFormat, if executed, is supposed to create a fifo, read from it and remove it in the end.
+	// Example: "mkfifo my.fifo && (cat my.fifo; rm my.fifo)".
+	stderrFifoCommandFormat = "mkfifo %s && (cat %s; rm %s)"
+	// stderrWrapperCommandFormat, if executed, is supposed to wait until a fifo exists (it sleeps 10ms to reduce load
+	// cause by busy waiting on the system). Once the fifo exists, the given command is executed and its stderr
+	// redirected to the fifo.
+	// Example: "until [ -e my.fifo ]; do sleep 0.01; done; (echo \"my.fifo exists\") 2> my.fifo".
+	stderrWrapperCommandFormat = "until [ -e %s ]; do sleep 0.01; done; (%s) 2> %s"
+)
+
+func stderrFifoCommand(id int64) []string {
+	stderrFifoPath := stderrFifo(id)
+	return []string{"sh", "-c", fmt.Sprintf(stderrFifoCommandFormat, stderrFifoPath, stderrFifoPath, stderrFifoPath)}
+}
+
+func wrapCommandForStderrFifo(id int64, command string) string {
+	stderrFifoPath := stderrFifo(id)
+	return fmt.Sprintf(stderrWrapperCommandFormat, stderrFifoPath, command, stderrFifoPath)
+}
+
+func stderrFifo(id int64) string {
+	return fmt.Sprintf(stderrFifoFormat, id)
+}

internal/nomad/nomad_test.go

@@ -0,0 +1,789 @@
+package nomad
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	nomadApi "github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/mitchellh/mapstructure"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/internal/config"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/pkg/nullreader"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
+	"io"
+	"net/url"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestLoadRunnersTestSuite(t *testing.T) {
+	suite.Run(t, new(LoadRunnersTestSuite))
+}
+
+type LoadRunnersTestSuite struct {
+	suite.Suite
+	jobID                  string
+	mock                   *apiQuerierMock
+	nomadAPIClient         APIClient
+	availableRunner        *nomadApi.JobListStub
+	anotherAvailableRunner *nomadApi.JobListStub
+	pendingRunner          *nomadApi.JobListStub
+	deadRunner             *nomadApi.JobListStub
+}
+
+func (s *LoadRunnersTestSuite) SetupTest() {
+	s.jobID = tests.DefaultJobID
+
+	s.mock = &apiQuerierMock{}
+	s.nomadAPIClient = APIClient{apiQuerier: s.mock}
+
+	s.availableRunner = newJobListStub(tests.DefaultJobID, structs.JobStatusRunning, 1)
+	s.anotherAvailableRunner = newJobListStub(tests.AnotherJobID, structs.JobStatusRunning, 1)
+	s.pendingRunner = newJobListStub(tests.DefaultJobID+"-1", structs.JobStatusPending, 0)
+	s.deadRunner = newJobListStub(tests.AnotherJobID+"-1", structs.JobStatusDead, 0)
+}
+
+func newJobListStub(id, status string, amountRunning int) *nomadApi.JobListStub {
+	return &nomadApi.JobListStub{
+		ID:     id,
+		Status: status,
+		JobSummary: &nomadApi.JobSummary{
+			JobID:   id,
+			Summary: map[string]nomadApi.TaskGroupSummary{TaskGroupName: {Running: amountRunning}},
+		},
+	}
+}
+
+func (s *LoadRunnersTestSuite) TestErrorOfUnderlyingApiCallIsPropagated() {
+	s.mock.On("listJobs", mock.AnythingOfType("string")).
+		Return(nil, tests.ErrDefault)
+
+	returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID)
+	s.Nil(returnedIds)
+	s.Equal(tests.ErrDefault, err)
+}
+
+func (s *LoadRunnersTestSuite) TestReturnsNoErrorWhenUnderlyingApiCallDoesNot() {
+	s.mock.On("listJobs", mock.AnythingOfType("string")).
+		Return([]*nomadApi.JobListStub{}, nil)
+
+	_, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID)
+	s.NoError(err)
+}
+
+func (s *LoadRunnersTestSuite) TestAvailableRunnerIsReturned() {
+	s.mock.On("listJobs", mock.AnythingOfType("string")).
+		Return([]*nomadApi.JobListStub{s.availableRunner}, nil)
+
+	returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID)
+	s.Require().NoError(err)
+	s.Len(returnedIds, 1)
+	s.Equal(s.availableRunner.ID, returnedIds[0])
+}
+
+func (s *LoadRunnersTestSuite) TestPendingRunnerIsNotReturned() {
+	s.mock.On("listJobs", mock.AnythingOfType("string")).
+		Return([]*nomadApi.JobListStub{s.pendingRunner}, nil)
+
+	returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID)
+	s.Require().NoError(err)
+	s.Empty(returnedIds)
+}
+
+func (s *LoadRunnersTestSuite) TestDeadRunnerIsNotReturned() {
+	s.mock.On("listJobs", mock.AnythingOfType("string")).
+		Return([]*nomadApi.JobListStub{s.deadRunner}, nil)
+
+	returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID)
+	s.Require().NoError(err)
+	s.Empty(returnedIds)
+}
+
+func (s *LoadRunnersTestSuite) TestReturnsAllAvailableRunners() {
+	runnersList := []*nomadApi.JobListStub{
+		s.availableRunner,
+		s.anotherAvailableRunner,
+		s.pendingRunner,
+		s.deadRunner,
+	}
+	s.mock.On("listJobs", mock.AnythingOfType("string")).
+		Return(runnersList, nil)
+
+	returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID)
+	s.Require().NoError(err)
+	s.Len(returnedIds, 2)
+	s.Contains(returnedIds, s.availableRunner.ID)
+	s.Contains(returnedIds, s.anotherAvailableRunner.ID)
+}
+
+var (
+	TestURL = url.URL{
+		Scheme: "http",
+		Host:   "127.0.0.1:4646",
+	}
+)
+
+const TestNamespace = "unit-tests"
+
+func TestApiClient_init(t *testing.T) {
+	client := &APIClient{apiQuerier: &nomadAPIClient{}}
+	err := client.init(&TestURL, TestNamespace)
+	require.Nil(t, err)
+}
+
+func TestApiClientCanNotBeInitializedWithInvalidUrl(t *testing.T) {
+	client := &APIClient{apiQuerier: &nomadAPIClient{}}
+	err := client.init(&url.URL{
+		Scheme: "http",
+		Host:   "http://127.0.0.1:4646",
+	}, TestNamespace)
+	assert.NotNil(t, err)
+}
+
+func TestNewExecutorApiCanBeCreatedWithoutError(t *testing.T) {
+	expectedClient := &APIClient{apiQuerier: &nomadAPIClient{}}
+	err := expectedClient.init(&TestURL, TestNamespace)
+	require.Nil(t, err)
+
+	_, err = NewExecutorAPI(&TestURL, TestNamespace)
+	require.Nil(t, err)
+}
+
+// asynchronouslyMonitorEvaluation creates an APIClient with mocked Nomad API and
+// runs the MonitorEvaluation method in a goroutine. The mock returns a read-only
+// version of the given stream to simulate an event stream gotten from the real
+// Nomad API.
+func asynchronouslyMonitorEvaluation(stream chan *nomadApi.Events) chan error {
+	ctx := context.Background()
+	// We can only get a read-only channel once we return it from a function.
+	readOnlyStream := func() <-chan *nomadApi.Events { return stream }()
+
+	apiMock := &apiQuerierMock{}
+	apiMock.On("EvaluationStream", mock.AnythingOfType("string"), ctx).Return(readOnlyStream, nil)
+	apiClient := &APIClient{apiMock}
+
+	errChan := make(chan error)
+	go func() {
+		errChan <- apiClient.MonitorEvaluation("id", ctx)
+	}()
+	return errChan
+}
+
+func TestApiClient_MonitorEvaluationReturnsNilWhenStreamIsClosed(t *testing.T) {
+	stream := make(chan *nomadApi.Events)
+	errChan := asynchronouslyMonitorEvaluation(stream)
+
+	close(stream)
+	var err error
+	// If close doesn't terminate MonitorEvaluation, this test won't complete without a timeout.
+	select {
+	case err = <-errChan:
+	case <-time.After(time.Millisecond * 10):
+		t.Fatal("MonitorEvaluation didn't finish as expected")
+	}
+	assert.Nil(t, err)
+}
+
+func TestApiClient_MonitorEvaluationReturnsErrorWhenStreamReturnsError(t *testing.T) {
+	apiMock := &apiQuerierMock{}
+	apiMock.On("EvaluationStream", mock.AnythingOfType("string"), mock.AnythingOfType("*context.emptyCtx")).
+		Return(nil, tests.ErrDefault)
+	apiClient := &APIClient{apiMock}
+	err := apiClient.MonitorEvaluation("id", context.Background())
+	assert.ErrorIs(t, err, tests.ErrDefault)
+}
+
+type eventPayload struct {
+	Evaluation *nomadApi.Evaluation
+	Allocation *nomadApi.Allocation
+}
+
+// eventForEvaluation takes an evaluation and creates an Event with the given evaluation
+// as its payload. Nomad uses the mapstructure library to decode the payload, which we
+// simply reverse here.
+func eventForEvaluation(t *testing.T, eval *nomadApi.Evaluation) nomadApi.Event {
+	t.Helper()
+	payload := make(map[string]interface{})
+
+	err := mapstructure.Decode(eventPayload{Evaluation: eval}, &payload)
+	if err != nil {
+		t.Fatalf("Couldn't decode evaluation %v into payload map", eval)
+		return nomadApi.Event{}
+	}
+	event := nomadApi.Event{Topic: nomadApi.TopicEvaluation, Payload: payload}
+	return event
+}
+
+// simulateNomadEventStream streams the given events sequentially to the stream channel.
+// It returns how many events have been processed until an error occurred.
+func simulateNomadEventStream(
+	stream chan *nomadApi.Events,
+	errChan chan error,
+	events []*nomadApi.Events,
+) (int, error) {
+	eventsProcessed := 0
+	var e *nomadApi.Events
+	for _, e = range events {
+		select {
+		case err := <-errChan:
+			return eventsProcessed, err
+		case stream <- e:
+			eventsProcessed++
+		}
+	}
+	// Wait for last event being processed
+	var err error
+	select {
+	case <-time.After(10 * time.Millisecond):
+	case err = <-errChan:
+	}
+	return eventsProcessed, err
+}
+
+// runEvaluationMonitoring simulates events streamed from the Nomad event stream
+// to the MonitorEvaluation method. It starts the MonitorEvaluation function as a goroutine
+// and sequentially transfers the events from the given array to a channel simulating the stream.
+func runEvaluationMonitoring(events []*nomadApi.Events) (eventsProcessed int, err error) {
+	stream := make(chan *nomadApi.Events)
+	errChan := asynchronouslyMonitorEvaluation(stream)
+	return simulateNomadEventStream(stream, errChan, events)
+}
+
+func TestApiClient_MonitorEvaluationWithSuccessfulEvent(t *testing.T) {
+	eval := nomadApi.Evaluation{Status: structs.EvalStatusComplete}
+	pendingEval := nomadApi.Evaluation{Status: structs.EvalStatusPending}
+
+	// make sure that the tested function can complete
+	require.Nil(t, checkEvaluation(&eval))
+
+	events := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(t, &eval)}}
+	pendingEvaluationEvents := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(t, &pendingEval)}}
+	multipleEventsWithPending := nomadApi.Events{Events: []nomadApi.Event{
+		eventForEvaluation(t, &pendingEval), eventForEvaluation(t, &eval),
+	}}
+
+	var cases = []struct {
+		streamedEvents          []*nomadApi.Events
+		expectedEventsProcessed int
+		name                    string
+	}{
+		{[]*nomadApi.Events{&events}, 1,
+			"it completes with successful event"},
+		{[]*nomadApi.Events{&events, &events}, 1,
+			"it completes at first successful event"},
+		{[]*nomadApi.Events{{}, &events}, 2,
+			"it skips heartbeat and completes"},
+		{[]*nomadApi.Events{&pendingEvaluationEvents, &events}, 2,
+			"it skips pending evaluation and completes"},
+		{[]*nomadApi.Events{&multipleEventsWithPending}, 1,
+			"it handles multiple events per received event"},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			eventsProcessed, err := runEvaluationMonitoring(c.streamedEvents)
+			assert.Nil(t, err)
+			assert.Equal(t, c.expectedEventsProcessed, eventsProcessed)
+		})
+	}
+}
+
+func TestApiClient_MonitorEvaluationWithFailingEvent(t *testing.T) {
+	eval := nomadApi.Evaluation{Status: structs.EvalStatusFailed}
+	evalErr := checkEvaluation(&eval)
+	require.NotNil(t, evalErr)
+
+	pendingEval := nomadApi.Evaluation{Status: structs.EvalStatusPending}
+
+	events := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(t, &eval)}}
+	pendingEvaluationEvents := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(t, &pendingEval)}}
+	multipleEventsWithPending := nomadApi.Events{Events: []nomadApi.Event{
+		eventForEvaluation(t, &pendingEval), eventForEvaluation(t, &eval),
+	}}
+	eventsWithErr := nomadApi.Events{Err: tests.ErrDefault, Events: []nomadApi.Event{{}}}
+
+	var cases = []struct {
+		streamedEvents          []*nomadApi.Events
+		expectedEventsProcessed int
+		expectedError           error
+		name                    string
+	}{
+		{[]*nomadApi.Events{&events}, 1, evalErr,
+			"it fails with failing event"},
+		{[]*nomadApi.Events{&events, &events}, 1, evalErr,
+			"it fails at first failing event"},
+		{[]*nomadApi.Events{{}, &events}, 2, evalErr,
+			"it skips heartbeat and fail"},
+		{[]*nomadApi.Events{&pendingEvaluationEvents, &events}, 2, evalErr,
+			"it skips pending evaluation and fail"},
+		{[]*nomadApi.Events{&multipleEventsWithPending}, 1, evalErr,
+			"it handles multiple events per received event and fails"},
+		{[]*nomadApi.Events{&eventsWithErr}, 1, tests.ErrDefault,
+			"it fails with event error when event has error"},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			eventsProcessed, err := runEvaluationMonitoring(c.streamedEvents)
+			require.NotNil(t, err)
+			assert.Contains(t, err.Error(), c.expectedError.Error())
+			assert.Equal(t, c.expectedEventsProcessed, eventsProcessed)
+		})
+	}
+}
+
+func TestApiClient_MonitorEvaluationFailsWhenFailingToDecodeEvaluation(t *testing.T) {
+	event := nomadApi.Event{
+		Topic: nomadApi.TopicEvaluation,
+		// This should fail decoding, as Evaluation.Status is expected to be a string, not int
+		Payload: map[string]interface{}{"Evaluation": map[string]interface{}{"Status": 1}},
+	}
+	_, err := event.Evaluation()
+	require.NotNil(t, err)
+	eventsProcessed, err := runEvaluationMonitoring([]*nomadApi.Events{{Events: []nomadApi.Event{event}}})
+	assert.Error(t, err)
+	assert.Equal(t, 1, eventsProcessed)
+}
+
+func TestCheckEvaluationWithFailedAllocations(t *testing.T) {
+	testKey := "test1"
+	failedAllocs := map[string]*nomadApi.AllocationMetric{
+		testKey: {NodesExhausted: 1},
+	}
+	evaluation := nomadApi.Evaluation{FailedTGAllocs: failedAllocs, Status: structs.EvalStatusFailed}
+
+	assertMessageContainsCorrectStrings := func(msg string) {
+		assert.Contains(t, msg, evaluation.Status, "error should contain the evaluation status")
+		assert.Contains(t, msg, fmt.Sprintf("%s: %#v", testKey, failedAllocs[testKey]),
+			"error should contain the failed allocations metric")
+	}
+
+	var msgWithoutBlockedEval, msgWithBlockedEval string
+	t.Run("without blocked eval", func(t *testing.T) {
+		err := checkEvaluation(&evaluation)
+		require.NotNil(t, err)
+		msgWithoutBlockedEval = err.Error()
+		assertMessageContainsCorrectStrings(msgWithoutBlockedEval)
+	})
+
+	t.Run("with blocked eval", func(t *testing.T) {
+		evaluation.BlockedEval = "blocking-eval"
+		err := checkEvaluation(&evaluation)
+		require.NotNil(t, err)
+		msgWithBlockedEval = err.Error()
+		assertMessageContainsCorrectStrings(msgWithBlockedEval)
+	})
+
+	assert.NotEqual(t, msgWithBlockedEval, msgWithoutBlockedEval)
+}
+
+func TestCheckEvaluationWithoutFailedAllocations(t *testing.T) {
+	evaluation := nomadApi.Evaluation{FailedTGAllocs: make(map[string]*nomadApi.AllocationMetric)}
+
+	t.Run("when evaluation status complete", func(t *testing.T) {
+		evaluation.Status = structs.EvalStatusComplete
+		err := checkEvaluation(&evaluation)
+		assert.Nil(t, err)
+	})
+
+	t.Run("when evaluation status not complete", func(t *testing.T) {
+		incompleteStates := []string{structs.EvalStatusFailed, structs.EvalStatusCancelled,
+			structs.EvalStatusBlocked, structs.EvalStatusPending}
+		for _, status := range incompleteStates {
+			evaluation.Status = status
+			err := checkEvaluation(&evaluation)
+			require.NotNil(t, err)
+			assert.Contains(t, err.Error(), status, "error should contain the evaluation status")
+		}
+	})
+}
+
+func TestApiClient_WatchAllocationsIgnoresOldAllocations(t *testing.T) {
+	oldStoppedAllocation := createOldAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusStop)
+	oldPendingAllocation := createOldAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun)
+	oldRunningAllocation := createOldAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun)
+	oldAllocationEvents := nomadApi.Events{Events: []nomadApi.Event{
+		eventForAllocation(t, oldStoppedAllocation),
+		eventForAllocation(t, oldPendingAllocation),
+		eventForAllocation(t, oldRunningAllocation),
+	}}
+
+	assertWatchAllocation(t, []*nomadApi.Events{&oldAllocationEvents},
+		[]*nomadApi.Allocation(nil), []*nomadApi.Allocation(nil))
+}
+
+func createOldAllocation(clientStatus, desiredStatus string) *nomadApi.Allocation {
+	return createAllocation(time.Now().Add(-time.Minute).UnixNano(), clientStatus, desiredStatus)
+}
+
+func TestApiClient_WatchAllocationsIgnoresUnhandledEvents(t *testing.T) {
+	nodeEvents := nomadApi.Events{Events: []nomadApi.Event{
+		{
+			Topic: nomadApi.TopicNode,
+			Type:  structs.TypeNodeEvent,
+		},
+	}}
+	assertWatchAllocation(t, []*nomadApi.Events{&nodeEvents}, []*nomadApi.Allocation(nil), []*nomadApi.Allocation(nil))
+
+	planEvents := nomadApi.Events{Events: []nomadApi.Event{
+		{
+			Topic: nomadApi.TopicAllocation,
+			Type:  structs.TypePlanResult,
+		},
+	}}
+	assertWatchAllocation(t, []*nomadApi.Events{&planEvents}, []*nomadApi.Allocation(nil), []*nomadApi.Allocation(nil))
+}
+
+func TestApiClient_WatchAllocationsHandlesEvents(t *testing.T) {
+	newPendingAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun)
+	pendingAllocationEvents := nomadApi.Events{Events: []nomadApi.Event{eventForAllocation(t, newPendingAllocation)}}
+
+	newStartedAllocation := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun)
+	startAllocationEvents := nomadApi.Events{Events: []nomadApi.Event{
+		eventForAllocation(t, newPendingAllocation),
+		eventForAllocation(t, newStartedAllocation),
+	}}
+
+	newStoppedAllocation := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusStop)
+	stopAllocationEvents := nomadApi.Events{Events: []nomadApi.Event{
+		eventForAllocation(t, newPendingAllocation),
+		eventForAllocation(t, newStartedAllocation),
+		eventForAllocation(t, newStoppedAllocation),
+	}}
+
+	var cases = []struct {
+		streamedEvents             []*nomadApi.Events
+		expectedNewAllocations     []*nomadApi.Allocation
+		expectedDeletedAllocations []*nomadApi.Allocation
+		name                       string
+	}{
+		{[]*nomadApi.Events{&pendingAllocationEvents},
+			[]*nomadApi.Allocation(nil), []*nomadApi.Allocation(nil),
+			"it does not add allocation when client status is pending"},
+		{[]*nomadApi.Events{&startAllocationEvents},
+			[]*nomadApi.Allocation{newStartedAllocation},
+			[]*nomadApi.Allocation(nil),
+			"it adds allocation with matching events"},
+		{[]*nomadApi.Events{{}, &startAllocationEvents},
+			[]*nomadApi.Allocation{newStartedAllocation},
+			[]*nomadApi.Allocation(nil),
+			"it skips heartbeat and adds allocation with matching events"},
+		{[]*nomadApi.Events{&stopAllocationEvents},
+			[]*nomadApi.Allocation{newStartedAllocation},
+			[]*nomadApi.Allocation{newStoppedAllocation},
+			"it adds and deletes the allocation"},
+		{[]*nomadApi.Events{&startAllocationEvents, &startAllocationEvents},
+			[]*nomadApi.Allocation{newStartedAllocation, newStartedAllocation},
+			[]*nomadApi.Allocation(nil),
+			"it handles multiple events"},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assertWatchAllocation(t, c.streamedEvents,
+				c.expectedNewAllocations, c.expectedDeletedAllocations)
+		})
+	}
+}
+
+func TestHandleAllocationEventBuffersPendingAllocation(t *testing.T) {
+	newPendingAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun)
+	newPendingEvent := eventForAllocation(t, newPendingAllocation)
+
+	pendingMap := make(map[string]bool)
+	var noop AllocationProcessor = func(allocation *nomadApi.Allocation) {}
+
+	err := handleAllocationEvent(time.Now().UnixNano(), pendingMap, &newPendingEvent, noop, noop)
+	require.NoError(t, err)
+
+	assert.True(t, pendingMap[newPendingAllocation.ID])
+}
+
+func TestAPIClient_WatchAllocationsReturnsErrorWhenAllocationStreamCannotBeRetrieved(t *testing.T) {
+	apiMock := &apiQuerierMock{}
+	apiMock.On("AllocationStream", mock.Anything).Return(nil, tests.ErrDefault)
+	apiClient := &APIClient{apiMock}
+
+	noop := func(a *nomadApi.Allocation) {}
+	err := apiClient.WatchAllocations(context.Background(), noop, noop)
+	assert.ErrorIs(t, err, tests.ErrDefault)
+}
+
+func TestAPIClient_WatchAllocationsReturnsErrorWhenAllocationCannotBeRetrievedWithoutReceivingFurtherEvents(
+	t *testing.T) {
+	noop := func(a *nomadApi.Allocation) {}
+	event := nomadApi.Event{
+		Type:  structs.TypeAllocationUpdated,
+		Topic: nomadApi.TopicAllocation,
+		// This should fail decoding, as Allocation.ID is expected to be a string, not int
+		Payload: map[string]interface{}{"Allocation": map[string]interface{}{"ID": 1}},
+	}
+	_, err := event.Allocation()
+	require.Error(t, err)
+
+	events := []*nomadApi.Events{{Events: []nomadApi.Event{event}}, {}}
+	eventsProcessed, err := runAllocationWatching(t, events, noop, noop)
+	assert.Error(t, err)
+	assert.Equal(t, 1, eventsProcessed)
+}
+
+func assertWatchAllocation(t *testing.T, events []*nomadApi.Events,
+	expectedNewAllocations, expectedDeletedAllocations []*nomadApi.Allocation) {
+	t.Helper()
+	var newAllocations []*nomadApi.Allocation
+	onNewAllocation := func(alloc *nomadApi.Allocation) {
+		newAllocations = append(newAllocations, alloc)
+	}
+
+	var deletedAllocations []*nomadApi.Allocation
+	onDeletedAllocation := func(alloc *nomadApi.Allocation) {
+		deletedAllocations = append(deletedAllocations, alloc)
+	}
+
+	eventsProcessed, err := runAllocationWatching(t, events, onNewAllocation, onDeletedAllocation)
+	assert.NoError(t, err)
+	assert.Equal(t, len(events), eventsProcessed)
+
+	assert.Equal(t, expectedNewAllocations, newAllocations)
+	assert.Equal(t, expectedDeletedAllocations, deletedAllocations)
+}
+
+// runAllocationWatching simulates events streamed from the Nomad event stream
+// to the MonitorEvaluation method. It starts the MonitorEvaluation function as a goroutine
+// and sequentially transfers the events from the given array to a channel simulating the stream.
+func runAllocationWatching(t *testing.T, events []*nomadApi.Events,
+	onNewAllocation, onDeletedAllocation AllocationProcessor) (eventsProcessed int, err error) {
+	t.Helper()
+	stream := make(chan *nomadApi.Events)
+	errChan := asynchronouslyWatchAllocations(stream, onNewAllocation, onDeletedAllocation)
+	return simulateNomadEventStream(stream, errChan, events)
+}
+
+// asynchronouslyMonitorEvaluation creates an APIClient with mocked Nomad API and
+// runs the MonitorEvaluation method in a goroutine. The mock returns a read-only
+// version of the given stream to simulate an event stream gotten from the real
+// Nomad API.
+func asynchronouslyWatchAllocations(stream chan *nomadApi.Events,
+	onNewAllocation, onDeletedAllocation AllocationProcessor) chan error {
+	ctx := context.Background()
+	// We can only get a read-only channel once we return it from a function.
+	readOnlyStream := func() <-chan *nomadApi.Events { return stream }()
+
+	apiMock := &apiQuerierMock{}
+	apiMock.On("AllocationStream", ctx).Return(readOnlyStream, nil)
+	apiClient := &APIClient{apiMock}
+
+	errChan := make(chan error)
+	go func() {
+		errChan <- apiClient.WatchAllocations(ctx, onNewAllocation, onDeletedAllocation)
+	}()
+	return errChan
+}
+
+// eventForEvaluation takes an evaluation and creates an Event with the given evaluation
+// as its payload. Nomad uses the mapstructure library to decode the payload, which we
+// simply reverse here.
+func eventForAllocation(t *testing.T, alloc *nomadApi.Allocation) nomadApi.Event {
+	t.Helper()
+	payload := make(map[string]interface{})
+
+	err := mapstructure.Decode(eventPayload{Allocation: alloc}, &payload)
+	if err != nil {
+		t.Fatalf("Couldn't decode allocation %v into payload map", err)
+		return nomadApi.Event{}
+	}
+	event := nomadApi.Event{
+		Topic:   nomadApi.TopicAllocation,
+		Type:    structs.TypeAllocationUpdated,
+		Payload: payload,
+	}
+	return event
+}
+
+func createAllocation(modifyTime int64, clientStatus, desiredStatus string) *nomadApi.Allocation {
+	return &nomadApi.Allocation{
+		ID:            tests.DefaultRunnerID,
+		ModifyTime:    modifyTime,
+		ClientStatus:  clientStatus,
+		DesiredStatus: desiredStatus,
+	}
+}
+
+func createRecentAllocation(clientStatus, desiredStatus string) *nomadApi.Allocation {
+	return createAllocation(time.Now().Add(time.Minute).UnixNano(), clientStatus, desiredStatus)
+}
+
+func TestExecuteCommandTestSuite(t *testing.T) {
+	suite.Run(t, new(ExecuteCommandTestSuite))
+}
+
+type ExecuteCommandTestSuite struct {
+	suite.Suite
+	allocationID     string
+	ctx              context.Context
+	testCommand      string
+	testCommandArray []string
+	expectedStdout   string
+	expectedStderr   string
+	apiMock          *apiQuerierMock
+	nomadAPIClient   APIClient
+}
+
+func (s *ExecuteCommandTestSuite) SetupTest() {
+	s.allocationID = "test-allocation-id"
+	s.ctx = context.Background()
+	s.testCommand = "echo 'do nothing'"
+	s.testCommandArray = []string{"sh", "-c", s.testCommand}
+	s.expectedStdout = "stdout"
+	s.expectedStderr = "stderr"
+	s.apiMock = &apiQuerierMock{}
+	s.nomadAPIClient = APIClient{apiQuerier: s.apiMock}
+}
+
+const withTTY = true
+
+func (s *ExecuteCommandTestSuite) TestWithSeparateStderr() {
+	config.Config.Server.InteractiveStderr = true
+	commandExitCode := 42
+	stderrExitCode := 1
+
+	var stdout, stderr bytes.Buffer
+	var calledStdoutCommand, calledStderrCommand []string
+
+	// mock regular call
+	s.mockExecute(s.testCommandArray, commandExitCode, nil, func(args mock.Arguments) {
+		var ok bool
+		calledStdoutCommand, ok = args.Get(2).([]string)
+		s.Require().True(ok)
+		writer, ok := args.Get(5).(io.Writer)
+		s.Require().True(ok)
+		_, err := writer.Write([]byte(s.expectedStdout))
+		s.Require().NoError(err)
+	})
+	// mock stderr call
+	s.mockExecute(mock.AnythingOfType("[]string"), stderrExitCode, nil, func(args mock.Arguments) {
+		var ok bool
+		calledStderrCommand, ok = args.Get(2).([]string)
+		s.Require().True(ok)
+		writer, ok := args.Get(5).(io.Writer)
+		s.Require().True(ok)
+		_, err := writer.Write([]byte(s.expectedStderr))
+		s.Require().NoError(err)
+	})
+
+	exitCode, err := s.nomadAPIClient.
+		ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullreader.NullReader{}, &stdout, &stderr)
+	s.Require().NoError(err)
+
+	s.apiMock.AssertNumberOfCalls(s.T(), "Execute", 2)
+	s.Equal(commandExitCode, exitCode)
+
+	s.Run("should wrap command in stderr wrapper", func() {
+		s.Require().NotNil(calledStdoutCommand)
+		stderrWrapperCommand := fmt.Sprintf(stderrWrapperCommandFormat, stderrFifoFormat, s.testCommand, stderrFifoFormat)
+		stdoutFifoRegexp := strings.ReplaceAll(regexp.QuoteMeta(stderrWrapperCommand), "%d", "\\d*")
+		s.Regexp(stdoutFifoRegexp, calledStdoutCommand[len(calledStdoutCommand)-1])
+	})
+
+	s.Run("should call correct stderr command", func() {
+		s.Require().NotNil(calledStderrCommand)
+		stderrFifoCommand := fmt.Sprintf(stderrFifoCommandFormat, stderrFifoFormat, stderrFifoFormat, stderrFifoFormat)
+		stderrFifoRegexp := strings.ReplaceAll(regexp.QuoteMeta(stderrFifoCommand), "%d", "\\d*")
+		s.Regexp(stderrFifoRegexp, calledStderrCommand[len(calledStderrCommand)-1])
+	})
+
+	s.Run("should return correct output", func() {
+		s.Equal(s.expectedStdout, stdout.String())
+		s.Equal(s.expectedStderr, stderr.String())
+	})
+}
+
+func (s *ExecuteCommandTestSuite) TestWithSeparateStderrReturnsCommandError() {
+	config.Config.Server.InteractiveStderr = true
+	s.mockExecute(s.testCommandArray, 1, tests.ErrDefault, func(args mock.Arguments) {})
+	s.mockExecute(mock.AnythingOfType("[]string"), 1, nil, func(args mock.Arguments) {})
+	_, err := s.nomadAPIClient.
+		ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullreader.NullReader{}, io.Discard, io.Discard)
+	s.Equal(tests.ErrDefault, err)
+}
+
+func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderr() {
+	config.Config.Server.InteractiveStderr = false
+	var stdout, stderr bytes.Buffer
+	commandExitCode := 42
+
+	// mock regular call
+	s.mockExecute(s.testCommandArray, commandExitCode, nil, func(args mock.Arguments) {
+		stdout, ok := args.Get(5).(io.Writer)
+		s.Require().True(ok)
+		_, err := stdout.Write([]byte(s.expectedStdout))
+		s.Require().NoError(err)
+		stderr, ok := args.Get(6).(io.Writer)
+		s.Require().True(ok)
+		_, err = stderr.Write([]byte(s.expectedStderr))
+		s.Require().NoError(err)
+	})
+
+	exitCode, err := s.nomadAPIClient.
+		ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullreader.NullReader{}, &stdout, &stderr)
+	s.Require().NoError(err)
+
+	s.apiMock.AssertNumberOfCalls(s.T(), "Execute", 1)
+	s.Equal(commandExitCode, exitCode)
+	s.Equal(s.expectedStdout, stdout.String())
+	s.Equal(s.expectedStderr, stderr.String())
+}
+
+func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderrReturnsCommandError() {
+	config.Config.Server.InteractiveStderr = false
+	s.mockExecute(s.testCommandArray, 1, tests.ErrDefault, func(args mock.Arguments) {})
+	_, err := s.nomadAPIClient.
+		ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullreader.NullReader{}, io.Discard, io.Discard)
+	s.ErrorIs(err, tests.ErrDefault)
+}
+
+func (s *ExecuteCommandTestSuite) mockExecute(command interface{}, exitCode int,
+	err error, runFunc func(arguments mock.Arguments)) {
+	s.apiMock.On("Execute", s.allocationID, s.ctx, command, withTTY,
+		mock.Anything, mock.Anything, mock.Anything).
+		Run(runFunc).
+		Return(exitCode, err)
+}
+
+func TestAPIClient_LoadRunnerPortMappings(t *testing.T) {
+	apiMock := &apiQuerierMock{}
+	mockedCall := apiMock.On("allocation", tests.DefaultRunnerID)
+	nomadAPIClient := APIClient{apiQuerier: apiMock}
+
+	t.Run("should return error when API query fails", func(t *testing.T) {
+		mockedCall.Return(nil, tests.ErrDefault)
+		portMappings, err := nomadAPIClient.LoadRunnerPortMappings(tests.DefaultRunnerID)
+		assert.Nil(t, portMappings)
+		assert.ErrorIs(t, err, tests.ErrDefault)
+	})
+
+	t.Run("should return error when AllocatedResources is nil", func(t *testing.T) {
+		mockedCall.Return(&nomadApi.Allocation{AllocatedResources: nil}, nil)
+		portMappings, err := nomadAPIClient.LoadRunnerPortMappings(tests.DefaultRunnerID)
+		assert.ErrorIs(t, err, ErrorNoAllocatedResourcesFound)
+		assert.Nil(t, portMappings)
+	})
+
+	t.Run("should correctly return ports", func(t *testing.T) {
+		allocation := &nomadApi.Allocation{
+			AllocatedResources: &nomadApi.AllocatedResources{
+				Shared: nomadApi.AllocatedSharedResources{Ports: tests.DefaultPortMappings},
+			},
+		}
+		mockedCall.Return(allocation, nil)
+		portMappings, err := nomadAPIClient.LoadRunnerPortMappings(tests.DefaultRunnerID)
+		assert.NoError(t, err)
+		assert.Equal(t, tests.DefaultPortMappings, portMappings)
+	})
+}