Add tests and end-to-end tests for websocket execution

For unit tests, this mocks the runners Execute method with a
customizable function that operates on the request, streams and exit
channel to simulate a real execution.

End-to-end tests are moved to the tests/e2e_tests folder. The tests
folder allows us to have shared helper functions for all tests in a
separate package (tests) that is not included in the non-test build.

This also adds one second of delay before each end-to-end test case by
using the TestSetup method of suite. By slowing down test execution,
this gives Nomad time to create new allocations when a test requested a
runner. Another solution could be to increase the scale of the job to
have enough allocations for all end-to-end tests.

Co-authored-by: Maximilian Paß <maximilian.pass@student.hpi.uni-potsdam.de>

Makefile

@@ -1,6 +1,6 @@
 PROJECT_NAME := "poseidon"
 PKG := "gitlab.hpi.de/codeocean/codemoon/$(PROJECT_NAME)"
-UNIT_TESTS = $(shell go list ./... | grep -v /e2e_tests)
+UNIT_TESTS = $(shell go list ./... | grep -v /e2e)
 
 DOCKER_E2E_CONTAINER_NAME := "$(PROJECT_NAME)-e2e-tests"
 DOCKER_TAG := "poseidon:latest"
@@ -81,7 +81,7 @@ coverhtml: coverage ## Generate HTML coverage report
 
 .PHONY: e2e-test
 e2e-test: deps ## Run e2e tests
-	@go test -count=1 ./e2e_tests -v
+	@go test -count=1 ./tests/e2e -v
 
 .PHONY: e2e-docker
 e2e-docker: docker ## Run e2e tests against the Docker container

api/runners_test.go

@@ -28,7 +28,7 @@ type MiddlewareTestSuite struct {
 
 func (suite *MiddlewareTestSuite) SetupTest() {
 	suite.manager = &runner.ManagerMock{}
-	suite.runner = runner.NewRunner("runner")
+	suite.runner = runner.NewNomadAllocation("runner", nil)
 	suite.capturedRunner = nil
 	suite.runnerRequest = func(runnerId string) *http.Request {
 		path, err := suite.router.Get("test-runner-id").URL(RunnerIdKey, runnerId)
@@ -89,12 +89,15 @@ type RunnerRouteTestSuite struct {
 	runnerManager *runner.ManagerMock
 	router        *mux.Router
 	runner        runner.Runner
+	executionId   runner.ExecutionId
 }
 
 func (suite *RunnerRouteTestSuite) SetupTest() {
 	suite.runnerManager = &runner.ManagerMock{}
 	suite.router = NewRouter(suite.runnerManager, nil)
-	suite.runner = runner.NewRunner("test_runner")
+	suite.runner = runner.NewNomadAllocation("some-id", nil)
+	suite.executionId = "execution-id"
+	suite.runner.Add(suite.executionId, &dto.ExecutionRequest{})
 	suite.runnerManager.On("Get", suite.runner.Id()).Return(suite.runner, nil)
 }
 
@@ -122,8 +125,8 @@ func (suite *RunnerRouteTestSuite) TestExecuteRoute() {
 
 		suite.router.ServeHTTP(recorder, request)
 
-		var websocketResponse dto.WebsocketResponse
+		var webSocketResponse dto.ExecutionResponse
-		err = json.NewDecoder(recorder.Result().Body).Decode(&websocketResponse)
+		err = json.NewDecoder(recorder.Result().Body).Decode(&webSocketResponse)
 		if err != nil {
 			suite.T().Fatal(err)
 		}
@@ -131,15 +134,15 @@ func (suite *RunnerRouteTestSuite) TestExecuteRoute() {
 		suite.Equal(http.StatusOK, recorder.Code)
 
 		suite.Run("creates an execution request for the runner", func() {
-			url, err := url.Parse(websocketResponse.WebsocketUrl)
+			webSocketUrl, err := url.Parse(webSocketResponse.WebSocketUrl)
 			if err != nil {
 				suite.T().Fatal(err)
 			}
-			executionId := url.Query().Get(ExecutionIdKey)
+			executionId := webSocketUrl.Query().Get(ExecutionIdKey)
-			storedExecutionRequest, ok := suite.runner.Execution(runner.ExecutionId(executionId))
+			storedExecutionRequest, ok := suite.runner.Pop(runner.ExecutionId(executionId))
 
 			suite.True(ok, "No execution request with this id: ", executionId)
-			suite.Equal(executionRequest, storedExecutionRequest)
+			suite.Equal(&executionRequest, storedExecutionRequest)
 		})
 	})
 

api/websocket_connection_mock.go

@@ -0,0 +1,93 @@
+// Code generated by mockery v0.0.0-dev. DO NOT EDIT.
+
+package api
+
+import (
+	io "io"
+
+	mock "github.com/stretchr/testify/mock"
+)
+
+// webSocketConnectionMock is an autogenerated mock type for the webSocketConnection type
+type webSocketConnectionMock struct {
+	mock.Mock
+}
+
+// Close provides a mock function with given fields:
+func (_m *webSocketConnectionMock) Close() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// CloseHandler provides a mock function with given fields:
+func (_m *webSocketConnectionMock) CloseHandler() func(int, string) error {
+	ret := _m.Called()
+
+	var r0 func(int, string) error
+	if rf, ok := ret.Get(0).(func() func(int, string) error); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(func(int, string) error)
+		}
+	}
+
+	return r0
+}
+
+// NextReader provides a mock function with given fields:
+func (_m *webSocketConnectionMock) NextReader() (int, io.Reader, error) {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 io.Reader
+	if rf, ok := ret.Get(1).(func() io.Reader); ok {
+		r1 = rf()
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(io.Reader)
+		}
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func() error); ok {
+		r2 = rf()
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// SetCloseHandler provides a mock function with given fields: h
+func (_m *webSocketConnectionMock) SetCloseHandler(h func(int, string) error) {
+	_m.Called(h)
+}
+
+// WriteMessage provides a mock function with given fields: messageType, data
+func (_m *webSocketConnectionMock) WriteMessage(messageType int, data []byte) error {
+	ret := _m.Called(messageType, data)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(int, []byte) error); ok {
+		r0 = rf(messageType, data)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}

api/websocket_test.go

@@ -1,76 +1,388 @@
 package api
 
 import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"github.com/gorilla/mux"
 	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
+	"time"
 )
 
-func TestWebsocketTestSuite(t *testing.T) {
+func TestWebSocketTestSuite(t *testing.T) {
-	suite.Run(t, new(WebsocketTestSuite))
+	suite.Run(t, new(WebSocketTestSuite))
 }
 
-type WebsocketTestSuite struct {
+type WebSocketTestSuite struct {
-	RunnerRouteTestSuite
+	suite.Suite
-	server      *httptest.Server
+	router      *mux.Router
 	executionId runner.ExecutionId
+	runner      runner.Runner
+	apiMock     *nomad.ExecutorApiMock
+	server      *httptest.Server
 }
 
-func (suite *WebsocketTestSuite) SetupTest() {
+func (suite *WebSocketTestSuite) SetupTest() {
-	suite.runnerManager = &runner.ManagerMock{}
+	runnerId := "runner-id"
-	suite.router = NewRouter(suite.runnerManager, nil)
+	suite.runner, suite.apiMock = helpers.NewNomadAllocationWithMockedApiClient(runnerId)
-	suite.runner = runner.NewRunner("test_runner")
-	suite.runnerManager.On("Get", suite.runner.Id()).Return(suite.runner, nil)
-	var err error
-	suite.executionId, err = suite.runner.AddExecution(dto.ExecutionRequest{
-		Command:     "command",
-		TimeLimit:   10,
-		Environment: nil,
-	})
-	suite.Require().NoError(err)
 
+	// default execution
+	suite.executionId = "execution-id"
+	suite.runner.Add(suite.executionId, &executionRequestHead)
+	mockApiExecuteHead(suite.apiMock)
+
+	runnerManager := &runner.ManagerMock{}
+	runnerManager.On("Get", suite.runner.Id()).Return(suite.runner, nil)
+	suite.router = NewRouter(runnerManager, nil)
 	suite.server = httptest.NewServer(suite.router)
 }
 
-func (suite *WebsocketTestSuite) TearDownSuite() {
+func (suite *WebSocketTestSuite) TearDownTest() {
 	suite.server.Close()
 }
 
-func (suite *WebsocketTestSuite) websocketUrl(scheme, runnerId string, executionId runner.ExecutionId) (*url.URL, error) {
+func (suite *WebSocketTestSuite) TestWebsocketConnectionCanBeEstablished() {
-	websocketUrl, err := url.Parse(suite.server.URL)
+	wsUrl, err := suite.webSocketUrl("ws", suite.runner.Id(), suite.executionId)
-	suite.Require().NoError(err, "Error: parsing test server url")
+	suite.Require().NoError(err)
-	path, err := suite.router.Get(WebsocketPath).URL(RunnerIdKey, runnerId)
+	_, _, err = websocket.DefaultDialer.Dial(wsUrl.String(), nil)
-	suite.Require().NoError(err, "could not set runnerId")
+	suite.Require().NoError(err)
+}
+
+func (suite *WebSocketTestSuite) TestWebsocketReturns404IfExecutionDoesNotExist() {
+	wsUrl, err := suite.webSocketUrl("ws", suite.runner.Id(), "invalid-execution-id")
+	suite.Require().NoError(err)
+	_, response, _ := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+	suite.Equal(http.StatusNotFound, response.StatusCode)
+}
+
+func (suite *WebSocketTestSuite) TestWebsocketReturns400IfRequestedViaHttp() {
+	wsUrl, err := suite.webSocketUrl("http", suite.runner.Id(), suite.executionId)
+	suite.Require().NoError(err)
+	response, err := http.Get(wsUrl.String())
+	suite.Require().NoError(err)
+	// This functionality is implemented by the WebSocket library.
+	suite.Equal(http.StatusBadRequest, response.StatusCode)
+}
+
+func (suite *WebSocketTestSuite) TestWebsocketConnection() {
+	wsUrl, err := suite.webSocketUrl("ws", suite.runner.Id(), suite.executionId)
+	suite.Require().NoError(err)
+	connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+	suite.Require().NoError(err)
+	err = connection.SetReadDeadline(time.Now().Add(5 * time.Second))
+	suite.Require().NoError(err)
+
+	suite.Run("Receives start message", func() {
+		message, err := helpers.ReceiveNextWebSocketMessage(connection)
+		suite.Require().NoError(err)
+		suite.Equal(dto.WebSocketMetaStart, message.Type)
+	})
+
+	suite.Run("Executes the request in the runner", func() {
+		<-time.After(100 * time.Millisecond)
+		suite.apiMock.AssertCalled(suite.T(), "ExecuteCommand",
+			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything)
+	})
+
+	suite.Run("Can send input", func() {
+		err = connection.WriteMessage(websocket.TextMessage, []byte("Hello World\n"))
+		suite.Require().NoError(err)
+	})
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+
+	suite.Run("Receives output message", func() {
+		stdout, _, _ := helpers.WebSocketOutputMessages(messages)
+		suite.Equal("Hello World", stdout)
+	})
+
+	suite.Run("Receives exit message", func() {
+		controlMessages := helpers.WebSocketControlMessages(messages)
+		suite.Require().Equal(1, len(controlMessages))
+		suite.Equal(dto.WebSocketExit, controlMessages[0].Type)
+	})
+}
+
+func (suite *WebSocketTestSuite) TestCancelWebSocketConnection() {
+	executionId := runner.ExecutionId("sleeping-execution")
+	suite.runner.Add(executionId, &executionRequestSleep)
+	canceled := mockApiExecuteSleep(suite.apiMock)
+
+	wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId)
+	suite.Require().NoError(err)
+	connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+	suite.Require().NoError(err)
+
+	message, err := helpers.ReceiveNextWebSocketMessage(connection)
+	suite.Require().NoError(err)
+	suite.Equal(dto.WebSocketMetaStart, message.Type)
+
+	select {
+	case <-canceled:
+		suite.Fail("Execute canceled unexpected")
+	default:
+	}
+
+	err = connection.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second))
+	suite.Require().NoError(err)
+
+	select {
+	case <-canceled:
+	case <-time.After(time.Second):
+		suite.Fail("Execute not canceled")
+	}
+}
+
+func (suite *WebSocketTestSuite) TestWebSocketConnectionTimeout() {
+	executionId := runner.ExecutionId("time-out-execution")
+	limitExecution := executionRequestSleep
+	limitExecution.TimeLimit = 2
+	suite.runner.Add(executionId, &limitExecution)
+	canceled := mockApiExecuteSleep(suite.apiMock)
+
+	wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId)
+	suite.Require().NoError(err)
+	connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+	suite.Require().NoError(err)
+
+	message, err := helpers.ReceiveNextWebSocketMessage(connection)
+	suite.Require().NoError(err)
+	suite.Equal(dto.WebSocketMetaStart, message.Type)
+
+	select {
+	case <-canceled:
+		suite.Fail("Execute canceled unexpected")
+	case <-time.After(time.Duration(limitExecution.TimeLimit-1) * time.Second):
+		<-time.After(time.Second)
+	}
+
+	select {
+	case <-canceled:
+	case <-time.After(time.Second):
+		suite.Fail("Execute not canceled")
+	}
+
+	message, err = helpers.ReceiveNextWebSocketMessage(connection)
+	suite.Require().NoError(err)
+	suite.Equal(dto.WebSocketMetaTimeout, message.Type)
+}
+
+func (suite *WebSocketTestSuite) TestWebsocketStdoutAndStderr() {
+	executionId := runner.ExecutionId("ls-execution")
+	suite.runner.Add(executionId, &executionRequestLs)
+	mockApiExecuteLs(suite.apiMock)
+
+	wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId)
+	suite.Require().NoError(err)
+	connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+	suite.Require().NoError(err)
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+	stdout, stderr, _ := helpers.WebSocketOutputMessages(messages)
+
+	suite.Contains(stdout, "existing-file")
+
+	suite.Contains(stderr, "non-existing-file")
+}
+
+func (suite *WebSocketTestSuite) TestWebsocketError() {
+	executionId := runner.ExecutionId("error-execution")
+	suite.runner.Add(executionId, &executionRequestError)
+	mockApiExecuteError(suite.apiMock)
+
+	wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId)
+	suite.Require().NoError(err)
+	connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+	suite.Require().NoError(err)
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+
+	_, _, errMessages := helpers.WebSocketOutputMessages(messages)
+	suite.Equal(1, len(errMessages))
+	suite.Equal("Error executing the request", errMessages[0])
+}
+
+func (suite *WebSocketTestSuite) TestWebsocketNonZeroExit() {
+	executionId := runner.ExecutionId("exit-execution")
+	suite.runner.Add(executionId, &executionRequestExitNonZero)
+	mockApiExecuteExitNonZero(suite.apiMock)
+
+	wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId)
+	suite.Require().NoError(err)
+	connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+	suite.Require().NoError(err)
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+
+	controlMessages := helpers.WebSocketControlMessages(messages)
+	suite.Equal(2, len(controlMessages))
+	suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit, ExitCode: 42}, controlMessages[1])
+}
+
+func TestWebsocketTLS(t *testing.T) {
+	runnerId := "runner-id"
+	r, apiMock := helpers.NewNomadAllocationWithMockedApiClient(runnerId)
+
+	executionId := runner.ExecutionId("execution-id")
+	r.Add(executionId, &executionRequestLs)
+	mockApiExecuteLs(apiMock)
+
+	runnerManager := &runner.ManagerMock{}
+	runnerManager.On("Get", r.Id()).Return(r, nil)
+	router := NewRouter(runnerManager, nil)
+
+	server, err := helpers.StartTLSServer(t, router)
+	require.NoError(t, err)
+	defer server.Close()
+
+	wsUrl, err := webSocketUrl("wss", server, router, runnerId, executionId)
+	require.NoError(t, err)
+
+	config := &tls.Config{RootCAs: nil, InsecureSkipVerify: true}
+	d := websocket.Dialer{TLSClientConfig: config}
+	connection, _, err := d.Dial(wsUrl.String(), nil)
+	require.NoError(t, err)
+
+	message, err := helpers.ReceiveNextWebSocketMessage(connection)
+	require.NoError(t, err)
+	assert.Equal(t, dto.WebSocketMetaStart, message.Type)
+	_, err = helpers.ReceiveAllWebSocketMessages(connection)
+	require.Error(t, err)
+	assert.Equal(t, &websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+}
+
+func TestRawToCodeOceanWriter(t *testing.T) {
+	testMessage := "test"
+	var message []byte
+
+	connectionMock := &webSocketConnectionMock{}
+	connectionMock.On("WriteMessage", mock.AnythingOfType("int"), mock.AnythingOfType("[]uint8")).
+		Run(func(args mock.Arguments) {
+			message = args.Get(1).([]byte)
+		}).
+		Return(nil)
+	connectionMock.On("CloseHandler").Return(nil)
+	connectionMock.On("SetCloseHandler", mock.Anything).Return()
+
+	proxy, err := newWebSocketProxy(connectionMock)
+	require.NoError(t, err)
+	writer := &rawToCodeOceanWriter{
+		proxy:      proxy,
+		outputType: dto.WebSocketOutputStdout,
+	}
+
+	_, err = writer.Write([]byte(testMessage))
+	require.NoError(t, err)
+
+	expected, _ := json.Marshal(struct {
+		Type string `json:"type"`
+		Data string `json:"data"`
+	}{string(dto.WebSocketOutputStdout), testMessage})
+	assert.Equal(t, expected, message)
+}
+
+// --- Test suite specific test helpers ---
+
+func webSocketUrl(scheme string, server *httptest.Server, router *mux.Router, runnerId string, executionId runner.ExecutionId) (*url.URL, error) {
+	websocketUrl, err := url.Parse(server.URL)
+	if err != nil {
+		return nil, err
+	}
+	path, err := router.Get(WebsocketPath).URL(RunnerIdKey, runnerId)
+	if err != nil {
+		return nil, err
+	}
 	websocketUrl.Scheme = scheme
 	websocketUrl.Path = path.Path
 	websocketUrl.RawQuery = fmt.Sprintf("executionId=%s", executionId)
 	return websocketUrl, nil
 }
 
-func (suite *WebsocketTestSuite) TestWebsocketConnectionCanBeEstablished() {
+func (suite *WebSocketTestSuite) webSocketUrl(scheme, runnerId string, executionId runner.ExecutionId) (*url.URL, error) {
-	path, err := suite.websocketUrl("ws", suite.runner.Id(), suite.executionId)
+	return webSocketUrl(scheme, suite.server, suite.router, runnerId, executionId)
-	suite.Require().NoError(err)
-	_, _, err = websocket.DefaultDialer.Dial(path.String(), nil)
-	suite.Require().NoError(err)
 }
 
-func (suite *WebsocketTestSuite) TestWebsocketReturns404IfExecutionDoesNotExist() {
+var executionRequestLs = dto.ExecutionRequest{Command: "ls"}
-	wsUrl, err := suite.websocketUrl("ws", suite.runner.Id(), "invalid-execution-id")
+
-	suite.Require().NoError(err)
+// mockApiExecuteLs mocks the ExecuteCommand of an ExecutorApi to act as if 'ls existing-file non-existing-file' was executed.
-	_, response, _ := websocket.DefaultDialer.Dial(wsUrl.String(), nil)
+func mockApiExecuteLs(api *nomad.ExecutorApiMock) {
-	suite.Equal(http.StatusNotFound, response.StatusCode)
+	helpers.MockApiExecute(api, &executionRequestLs,
+		func(_ string, _ context.Context, _ []string, _ io.Reader, stdout, stderr io.Writer) (int, error) {
+			_, _ = stdout.Write([]byte("existing-file\n"))
+			_, _ = stderr.Write([]byte("ls: cannot access 'non-existing-file': No such file or directory\n"))
+			return 0, nil
+		})
 }
 
-func (suite *WebsocketTestSuite) TestWebsocketReturns400IfRequestedViaHttp() {
+var executionRequestHead = dto.ExecutionRequest{Command: "head -n 1"}
-	wsUrl, err := suite.websocketUrl("http", suite.runner.Id(), suite.executionId)
+
-	suite.Require().NoError(err)
+// mockApiExecuteHead mocks the ExecuteCommand of an ExecutorApi to act as if 'head -n 1' was executed.
-	response, err := http.Get(wsUrl.String())
+func mockApiExecuteHead(api *nomad.ExecutorApiMock) {
-	suite.Require().NoError(err)
+	helpers.MockApiExecute(api, &executionRequestHead,
-	suite.Equal(http.StatusBadRequest, response.StatusCode)
+		func(_ string, _ context.Context, _ []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
+			scanner := bufio.NewScanner(stdin)
+			for !scanner.Scan() {
+				scanner = bufio.NewScanner(stdin)
+			}
+			_, _ = stdout.Write(scanner.Bytes())
+			return 0, nil
+		})
+}
+
+var executionRequestSleep = dto.ExecutionRequest{Command: "sleep infinity"}
+
+// mockApiExecuteSleep mocks the ExecuteCommand method of an ExecutorAPI to sleep until the execution is canceled.
+func mockApiExecuteSleep(api *nomad.ExecutorApiMock) <-chan bool {
+	canceled := make(chan bool, 1)
+	helpers.MockApiExecute(api, &executionRequestSleep,
+		func(_ string, ctx context.Context, _ []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
+			<-ctx.Done()
+			close(canceled)
+			return 0, ctx.Err()
+		})
+	return canceled
+}
+
+var executionRequestError = dto.ExecutionRequest{Command: "error"}
+
+// mockApiExecuteError mocks the ExecuteCommand method of an ExecutorApi to return an error.
+func mockApiExecuteError(api *nomad.ExecutorApiMock) {
+	helpers.MockApiExecute(api, &executionRequestError,
+		func(_ string, _ context.Context, _ []string, _ io.Reader, _, _ io.Writer) (int, error) {
+			return 0, errors.New("intended error")
+		})
+}
+
+var executionRequestExitNonZero = dto.ExecutionRequest{Command: "exit 42"}
+
+// mockApiExecuteExitNonZero mocks the ExecuteCommand method of an ExecutorApi to exit with exit status 42.
+func mockApiExecuteExitNonZero(api *nomad.ExecutorApiMock) {
+	helpers.MockApiExecute(api, &executionRequestExitNonZero,
+		func(_ string, _ context.Context, _ []string, _ io.Reader, _, _ io.Writer) (int, error) {
+			return 42, nil
+		})
 }

e2e_tests/helpers.go

@@ -1,12 +0,0 @@
-package e2e_tests
-
-import (
-	"io"
-	"net/http"
-)
-
-func httpDelete(url string, body io.Reader) (response *http.Response, err error) {
-	req, _ := http.NewRequest(http.MethodDelete, url, body)
-	client := &http.Client{}
-	return client.Do(req)
-}

e2e_tests/runners_test.go

@@ -1,58 +0,0 @@
-package e2e_tests
-
-import (
-	"encoding/json"
-	"github.com/stretchr/testify/assert"
-	"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
-	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
-	"net/http"
-	"strings"
-	"testing"
-)
-
-func TestProvideRunnerRoute(t *testing.T) {
-	runnerRequestString, _ := json.Marshal(dto.RunnerRequest{})
-	reader := strings.NewReader(string(runnerRequestString))
-	resp, err := http.Post(buildURL(api.RouteRunners), "application/json", reader)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode, "The response code should be ok")
-
-	runnerResponse := new(dto.RunnerResponse)
-	err = json.NewDecoder(resp.Body).Decode(runnerResponse)
-	assert.NoError(t, err)
-
-	assert.True(t, runnerResponse.Id != "", "The response contains a runner id")
-}
-
-func newRunnerId(t *testing.T) string {
-	runnerRequestString, _ := json.Marshal(dto.RunnerRequest{})
-	reader := strings.NewReader(string(runnerRequestString))
-	resp, err := http.Post(buildURL(api.RouteRunners), "application/json", reader)
-	assert.NoError(t, err)
-	runnerResponse := new(dto.RunnerResponse)
-	_ = json.NewDecoder(resp.Body).Decode(runnerResponse)
-	return runnerResponse.Id
-}
-
-func TestDeleteRunnerRoute(t *testing.T) {
-	runnerId := newRunnerId(t)
-	assert.NotEqual(t, "", runnerId)
-
-	t.Run("Deleting the runner returns NoContent", func(t *testing.T) {
-		resp, err := httpDelete(buildURL(api.RouteRunners, "/", runnerId), nil)
-		assert.NoError(t, err)
-		assert.Equal(t, http.StatusNoContent, resp.StatusCode)
-	})
-
-	t.Run("Deleting it again returns NotFound", func(t *testing.T) {
-		resp, err := httpDelete(buildURL(api.RouteRunners, "/", runnerId), nil)
-		assert.NoError(t, err)
-		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
-	})
-
-	t.Run("Deleting non-existing runner returns NotFound", func(t *testing.T) {
-		resp, err := httpDelete(buildURL(api.RouteRunners, "/", "n0n-3x1st1ng-1d"), nil)
-		assert.NoError(t, err)
-		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
-	})
-}

environment/job_test.go

@@ -55,7 +55,7 @@ func createTestJob() (*nomadApi.Job, *nomadApi.Job) {
 	base := nomadApi.NewBatchJob("python-job", "python-job", "region-name", 100)
 	job := nomadApi.NewBatchJob("python-job", "python-job", "region-name", 100)
 	task := createTestTask()
-	task.Name = fmt.Sprintf(TaskNameFormat, *job.ID)
+	task.Name = TaskName
 	image := "python:latest"
 	task.Config = map[string]interface{}{"image": image}
 	task.Config["network_mode"] = "none"

nomad/api_querier_mock.go

@@ -7,6 +7,8 @@ import (
 
 	api "github.com/hashicorp/nomad/api"
 
+	io "io"
+
 	mock "github.com/stretchr/testify/mock"
 
 	url "net/url"
@@ -54,6 +56,27 @@ func (_m *apiQuerierMock) EvaluationStream(evalID string, ctx context.Context) (
 	return r0, r1
 }
 
+// ExecuteCommand provides a mock function with given fields: allocationID, ctx, command, stdin, stdout, stderr
+func (_m *apiQuerierMock) ExecuteCommand(allocationID string, ctx context.Context, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
+	ret := _m.Called(allocationID, ctx, command, stdin, stdout, stderr)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) int); ok {
+		r0 = rf(allocationID, ctx, command, stdin, stdout, stderr)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) error); ok {
+		r1 = rf(allocationID, ctx, command, stdin, stdout, stderr)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // JobScale provides a mock function with given fields: jobId
 func (_m *apiQuerierMock) JobScale(jobId string) (int, error) {
 	ret := _m.Called(jobId)

nomad/executor_api_mock.go

@@ -7,6 +7,8 @@ import (
 
 	api "github.com/hashicorp/nomad/api"
 
+	io "io"
+
 	mock "github.com/stretchr/testify/mock"
 
 	url "net/url"
@@ -54,6 +56,27 @@ func (_m *ExecutorApiMock) EvaluationStream(evalID string, ctx context.Context)
 	return r0, r1
 }
 
+// ExecuteCommand provides a mock function with given fields: allocationID, ctx, command, stdin, stdout, stderr
+func (_m *ExecutorApiMock) ExecuteCommand(allocationID string, ctx context.Context, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
+	ret := _m.Called(allocationID, ctx, command, stdin, stdout, stderr)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) int); ok {
+		r0 = rf(allocationID, ctx, command, stdin, stdout, stderr)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) error); ok {
+		r1 = rf(allocationID, ctx, command, stdin, stdout, stderr)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // JobScale provides a mock function with given fields: jobId
 func (_m *ExecutorApiMock) JobScale(jobId string) (int, error) {
 	ret := _m.Called(jobId)

runner/runner_mock.go

@@ -0,0 +1,94 @@
+// Code generated by mockery v0.0.0-dev. DO NOT EDIT.
+
+package runner
+
+import (
+	context "context"
+	io "io"
+
+	dto "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
+
+	mock "github.com/stretchr/testify/mock"
+)
+
+// RunnerMock is an autogenerated mock type for the Runner type
+type RunnerMock struct {
+	mock.Mock
+}
+
+// Add provides a mock function with given fields: id, executionRequest
+func (_m *RunnerMock) Add(id ExecutionId, executionRequest *dto.ExecutionRequest) {
+	_m.Called(id, executionRequest)
+}
+
+// Copy provides a mock function with given fields: _a0
+func (_m *RunnerMock) Copy(_a0 dto.FileCreation) {
+	_m.Called(_a0)
+}
+
+// Delete provides a mock function with given fields: id
+func (_m *RunnerMock) Delete(id ExecutionId) {
+	_m.Called(id)
+}
+
+// Execute provides a mock function with given fields: request, stdin, stdout, stderr
+func (_m *RunnerMock) Execute(request *dto.ExecutionRequest, stdin io.Reader, stdout io.Writer, stderr io.Writer) (<-chan ExitInfo, context.CancelFunc) {
+	ret := _m.Called(request, stdin, stdout, stderr)
+
+	var r0 <-chan ExitInfo
+	if rf, ok := ret.Get(0).(func(*dto.ExecutionRequest, io.Reader, io.Writer, io.Writer) <-chan ExitInfo); ok {
+		r0 = rf(request, stdin, stdout, stderr)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(<-chan ExitInfo)
+		}
+	}
+
+	var r1 context.CancelFunc
+	if rf, ok := ret.Get(1).(func(*dto.ExecutionRequest, io.Reader, io.Writer, io.Writer) context.CancelFunc); ok {
+		r1 = rf(request, stdin, stdout, stderr)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(context.CancelFunc)
+		}
+	}
+
+	return r0, r1
+}
+
+// Get provides a mock function with given fields: id
+func (_m *RunnerMock) Get(id ExecutionId) (*dto.ExecutionRequest, bool) {
+	ret := _m.Called(id)
+
+	var r0 *dto.ExecutionRequest
+	if rf, ok := ret.Get(0).(func(ExecutionId) *dto.ExecutionRequest); ok {
+		r0 = rf(id)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*dto.ExecutionRequest)
+		}
+	}
+
+	var r1 bool
+	if rf, ok := ret.Get(1).(func(ExecutionId) bool); ok {
+		r1 = rf(id)
+	} else {
+		r1 = ret.Get(1).(bool)
+	}
+
+	return r0, r1
+}
+
+// Id provides a mock function with given fields:
+func (_m *RunnerMock) Id() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}

runner/runner_test.go

@@ -4,39 +4,42 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
 	"testing"
+	"time"
 )
 
 func TestIdIsStored(t *testing.T) {
-	runner := NewRunner("42")
+	runner := NewNomadAllocation("42", nil)
 	assert.Equal(t, "42", runner.Id())
 }
 
 func TestMarshalRunner(t *testing.T) {
-	runner := NewRunner("42")
+	runner := NewNomadAllocation("42", nil)
 	marshal, err := json.Marshal(runner)
 	assert.NoError(t, err)
 	assert.Equal(t, "{\"runnerId\":\"42\"}", string(marshal))
 }
 
 func TestExecutionRequestIsStored(t *testing.T) {
-	runner := NewRunner("42")
+	runner := NewNomadAllocation("42", nil)
-	executionRequest := dto.ExecutionRequest{
+	executionRequest := &dto.ExecutionRequest{
 		Command:     "command",
 		TimeLimit:   10,
 		Environment: nil,
 	}
-	id, err := runner.AddExecution(executionRequest)
+	id := ExecutionId("test-execution")
-	storedExecutionRunner, ok := runner.Execution(id)
+	runner.Add(id, executionRequest)
+	storedExecutionRunner, ok := runner.Pop(id)
 
-	assert.NoError(t, err, "AddExecution should not produce an error")
 	assert.True(t, ok, "Getting an execution should not return ok false")
 	assert.Equal(t, executionRequest, storedExecutionRunner)
 }
 
 func TestNewContextReturnsNewContextWithRunner(t *testing.T) {
-	runner := NewRunner("testRunner")
+	runner := NewNomadAllocation("testRunner", nil)
 	ctx := context.Background()
 	newCtx := NewContext(ctx, runner)
 	storedRunner := newCtx.Value(runnerContextKey).(Runner)
@@ -46,7 +49,7 @@ func TestNewContextReturnsNewContextWithRunner(t *testing.T) {
 }
 
 func TestFromContextReturnsRunner(t *testing.T) {
-	runner := NewRunner("testRunner")
+	runner := NewNomadAllocation("testRunner", nil)
 	ctx := NewContext(context.Background(), runner)
 	storedRunner, ok := FromContext(ctx)
 
@@ -60,3 +63,49 @@ func TestFromContextReturnsIsNotOkWhenContextHasNoRunner(t *testing.T) {
 
 	assert.False(t, ok)
 }
+
+func TestExecuteCallsAPI(t *testing.T) {
+	apiMock := &nomad.ExecutorApiMock{}
+	apiMock.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(0, nil)
+	runner := NewNomadAllocation("testRunner", apiMock)
+
+	request := &dto.ExecutionRequest{Command: "echo 'Hello World!'"}
+	runner.Execute(request, nil, nil, nil)
+
+	<-time.After(50 * time.Millisecond)
+	apiMock.AssertCalled(t, "ExecuteCommand", "testRunner", mock.Anything, request.FullCommand(), mock.Anything, mock.Anything, mock.Anything)
+}
+
+func TestExecuteReturnsAfterTimeout(t *testing.T) {
+	apiMock := newApiMockWithTimeLimitHandling()
+	runner := NewNomadAllocation("testRunner", apiMock)
+
+	timeLimit := 1
+	execution := &dto.ExecutionRequest{TimeLimit: timeLimit}
+	exit, _ := runner.Execute(execution, nil, nil, nil)
+
+	select {
+	case <-exit:
+		assert.FailNow(t, "Execute should not terminate instantly")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	select {
+	case <-time.After(time.Duration(timeLimit) * time.Second):
+		assert.FailNow(t, "Execute should return after the time limit")
+	case exitCode := <-exit:
+		assert.Equal(t, uint8(0), exitCode.Code)
+	}
+}
+
+func newApiMockWithTimeLimitHandling() (apiMock *nomad.ExecutorApiMock) {
+	apiMock = &nomad.ExecutorApiMock{}
+	apiMock.
+		On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+		Run(func(args mock.Arguments) {
+			ctx := args.Get(1).(context.Context)
+			<-ctx.Done()
+		}).
+		Return(0, nil)
+	return
+}

runner/storage_test.go

@@ -2,6 +2,7 @@ package runner
 
 import (
 	"github.com/stretchr/testify/suite"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
 	"testing"
 )
@@ -19,6 +20,7 @@ type RunnerPoolTestSuite struct {
 func (suite *RunnerPoolTestSuite) SetupTest() {
 	suite.runnerStorage = NewLocalRunnerStorage()
 	suite.runner = NewRunner(tests.DefaultRunnerId)
+	suite.runner.Add(tests.DefaultExecutionId, &dto.ExecutionRequest{Command: "true"})
 }
 
 func (suite *RunnerPoolTestSuite) TestAddedRunnerCanBeRetrieved() {

tests/e2e/e2e_test.go

@@ -1,12 +1,12 @@
-package e2e_tests
+package e2e
 
 import (
-	"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
+	"github.com/stretchr/testify/suite"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/logging"
 	"os"
-	"strings"
 	"testing"
+	"time"
 )
 
 /*
@@ -15,7 +15,20 @@ import (
 * For the e2e tests a nomad cluster must be connected and poseidon must be running.
  */
 
-var log = logging.GetLogger("e2e_tests")
+var log = logging.GetLogger("e2e")
+
+type E2ETestSuite struct {
+	suite.Suite
+}
+
+func (suite *E2ETestSuite) SetupTest() {
+	// Waiting one second before each test allows Nomad to rescale after tests requested runners.
+	<-time.After(time.Second)
+}
+
+func TestE2ETestSuite(t *testing.T) {
+	suite.Run(t, new(E2ETestSuite))
+}
 
 // Overwrite TestMain for custom setup.
 func TestMain(m *testing.M) {
@@ -29,8 +42,3 @@ func TestMain(m *testing.M) {
 	code := m.Run()
 	os.Exit(code)
 }
-
-func buildURL(parts ...string) (url string) {
-	parts = append([]string{config.Config.PoseidonAPIURL().String(), api.RouteBase}, parts...)
-	return strings.Join(parts, "")
-}

tests/e2e/health_test.go

@@ -1,14 +1,15 @@
-package e2e_tests
+package e2e
 
 import (
 	"github.com/stretchr/testify/assert"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
 	"net/http"
 	"testing"
 )
 
 func TestHealthRoute(t *testing.T) {
-	resp, err := http.Get(buildURL(api.RouteHealth))
+	resp, err := http.Get(helpers.BuildURL(api.RouteBase, api.RouteHealth))
 	if assert.NoError(t, err) {
 		assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent")
 	}

tests/e2e/helpers/test_helpers.go

@@ -0,0 +1,134 @@
+// Package helpers contains functions that help executing tests.
+// The helper functions generally look from the client side - a Poseidon user.
+package helpers
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"github.com/gorilla/mux"
+	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/mock"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
+	"io"
+	"net/http/httptest"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// BuildURL joins multiple route paths.
+func BuildURL(parts ...string) (url string) {
+	parts = append([]string{config.Config.PoseidonAPIURL().String()}, parts...)
+	return strings.Join(parts, "")
+}
+
+// WebSocketOutputMessages extracts all stdout, stderr and error messages from the passed messages.
+// It is useful since Nomad splits the command output nondeterministic.
+func WebSocketOutputMessages(messages []*dto.WebSocketMessage) (stdout, stderr string, errors []string) {
+	for _, msg := range messages {
+		switch msg.Type {
+		case dto.WebSocketOutputStdout:
+			stdout += msg.Data
+		case dto.WebSocketOutputStderr:
+			stderr += msg.Data
+		case dto.WebSocketOutputError:
+			errors = append(errors, msg.Data)
+		}
+	}
+	return
+}
+
+// WebSocketControlMessages extracts all meta (and exit) messages from the passed messages.
+func WebSocketControlMessages(messages []*dto.WebSocketMessage) (controls []*dto.WebSocketMessage) {
+	for _, msg := range messages {
+		switch msg.Type {
+		case dto.WebSocketMetaStart, dto.WebSocketMetaTimeout, dto.WebSocketExit:
+			controls = append(controls, msg)
+		}
+	}
+	return
+}
+
+// ReceiveAllWebSocketMessages pulls all messages from the websocket connection without sending anything.
+// This function does not return unless the server closes the connection or a readDeadline is set in the WebSocket connection.
+func ReceiveAllWebSocketMessages(connection *websocket.Conn) (messages []*dto.WebSocketMessage, err error) {
+	for {
+		var message *dto.WebSocketMessage
+		message, err = ReceiveNextWebSocketMessage(connection)
+		if err != nil {
+			return
+		}
+		messages = append(messages, message)
+	}
+}
+
+// ReceiveNextWebSocketMessage pulls the next message from the websocket connection.
+// This function does not return unless the server sends a message, closes the connection or a readDeadline is set in the WebSocket connection.
+func ReceiveNextWebSocketMessage(connection *websocket.Conn) (*dto.WebSocketMessage, error) {
+	_, reader, err := connection.NextReader()
+	if err != nil {
+		return nil, err
+	}
+	message := new(dto.WebSocketMessage)
+	err = json.NewDecoder(reader).Decode(message)
+	if err != nil {
+		return nil, err
+	}
+	return message, nil
+}
+
+// StartTLSServer runs a httptest.Server with the passed mux.Router and TLS enabled.
+func StartTLSServer(t *testing.T, router *mux.Router) (server *httptest.Server, err error) {
+	dir := t.TempDir()
+	keyOut := filepath.Join(dir, "poseidon-test.key")
+	certOut := filepath.Join(dir, "poseidon-test.crt")
+
+	err = exec.Command("openssl", "req", "-x509", "-nodes", "-newkey", "rsa:2048",
+		"-keyout", keyOut, "-out", certOut, "-days", "1",
+		"-subj", "/CN=Poseidon test", "-addext", "subjectAltName=IP:127.0.0.1,DNS:localhost").Run()
+	if err != nil {
+		return nil, err
+	}
+	cert, err := tls.LoadX509KeyPair(certOut, keyOut)
+	if err != nil {
+		return nil, err
+	}
+
+	server = httptest.NewUnstartedServer(router)
+	server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}}
+	server.StartTLS()
+	return
+}
+
+func NewNomadAllocationWithMockedApiClient(runnerId string) (r runner.Runner, mock *nomad.ExecutorApiMock) {
+	mock = &nomad.ExecutorApiMock{}
+	r = runner.NewNomadAllocation(runnerId, mock)
+	return
+}
+
+// MockApiExecute mocks the ExecuteCommand method of an ExecutorApi to call the given method run when the command
+// corresponding to the given ExecutionRequest is called.
+func MockApiExecute(api *nomad.ExecutorApiMock, request *dto.ExecutionRequest,
+	run func(runnerId string, ctx context.Context, command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error)) {
+	call := api.On("ExecuteCommand",
+		mock.AnythingOfType("string"),
+		mock.Anything,
+		request.FullCommand(),
+		mock.Anything,
+		mock.Anything,
+		mock.Anything)
+	call.Run(func(args mock.Arguments) {
+		exit, err := run(args.Get(0).(string),
+			args.Get(1).(context.Context),
+			args.Get(2).([]string),
+			args.Get(3).(io.Reader),
+			args.Get(4).(io.Writer),
+			args.Get(5).(io.Writer))
+		call.ReturnArguments = mock.Arguments{exit, err}
+	})
+}

tests/e2e/runners_test.go

@@ -0,0 +1,67 @@
+package e2e
+
+import (
+	"encoding/json"
+	"github.com/stretchr/testify/assert"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func (suite *E2ETestSuite) TestProvideRunnerRoute() {
+	runnerRequestString, _ := json.Marshal(dto.RunnerRequest{})
+	reader := strings.NewReader(string(runnerRequestString))
+	resp, err := http.Post(helpers.BuildURL(api.RouteBase, api.RouteRunners), "application/json", reader)
+	suite.NoError(err)
+	suite.Equal(http.StatusOK, resp.StatusCode, "The response code should be ok")
+
+	runnerResponse := new(dto.RunnerResponse)
+	err = json.NewDecoder(resp.Body).Decode(runnerResponse)
+	suite.NoError(err)
+
+	suite.True(runnerResponse.Id != "", "The response contains a runner id")
+}
+
+func newRunnerId(t *testing.T) string {
+	runnerRequestString, _ := json.Marshal(dto.RunnerRequest{})
+	reader := strings.NewReader(string(runnerRequestString))
+	resp, err := http.Post(helpers.BuildURL(api.RouteBase, api.RouteRunners), "application/json", reader)
+	assert.NoError(t, err)
+	runnerResponse := new(dto.RunnerResponse)
+	_ = json.NewDecoder(resp.Body).Decode(runnerResponse)
+	return runnerResponse.Id
+}
+
+func (suite *E2ETestSuite) TestDeleteRunnerRoute() {
+	runnerId := newRunnerId(suite.T())
+	suite.NotEqual("", runnerId)
+
+	suite.Run("Deleting the runner returns NoContent", func() {
+		resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId), nil)
+		suite.NoError(err)
+		suite.Equal(http.StatusNoContent, resp.StatusCode)
+	})
+
+	suite.Run("Deleting it again returns NotFound", func() {
+		resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId), nil)
+		suite.NoError(err)
+		suite.Equal(http.StatusNotFound, resp.StatusCode)
+	})
+
+	suite.Run("Deleting non-existing runner returns NotFound", func() {
+		resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", "n0n-3x1st1ng-1d"), nil)
+		suite.NoError(err)
+		suite.Equal(http.StatusNotFound, resp.StatusCode)
+	})
+}
+
+// HttpDelete sends a Delete Http Request with body to the passed url.
+func httpDelete(url string, body io.Reader) (response *http.Response, err error) {
+	req, _ := http.NewRequest(http.MethodDelete, url, body)
+	client := &http.Client{}
+	return client.Do(req)
+}

tests/e2e/websocket_test.go

@@ -0,0 +1,198 @@
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/suite"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
+	"net/http"
+	"strings"
+	"time"
+)
+
+func (suite *E2ETestSuite) TestExecuteCommandRoute() {
+	runnerId, err := ProvideRunner(&suite.Suite, &dto.RunnerRequest{})
+	suite.Require().NoError(err)
+
+	webSocketURL, err := ProvideWebSocketURL(&suite.Suite, runnerId, &dto.ExecutionRequest{Command: "true"})
+	suite.Require().NoError(err)
+	suite.NotEqual("", webSocketURL)
+
+	var connection *websocket.Conn
+	var connectionClosed bool
+
+	connection, err = ConnectToWebSocket(webSocketURL)
+	suite.Require().NoError(err, "websocket connects")
+	closeHandler := connection.CloseHandler()
+	connection.SetCloseHandler(func(code int, text string) error {
+		connectionClosed = true
+		return closeHandler(code, text)
+	})
+
+	startMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
+	suite.Require().NoError(err)
+	suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, startMessage)
+
+	exitMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
+	suite.Require().NoError(err)
+	suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, exitMessage)
+
+	_, err = helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+
+	_, _, _ = connection.ReadMessage()
+	suite.True(connectionClosed, "connection should be closed")
+}
+
+func (suite *E2ETestSuite) TestOutputToStdout() {
+	connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "echo Hello World"})
+	suite.Require().NoError(err)
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+
+	suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, messages[0])
+	suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketOutputStdout, Data: "Hello World\r\n"}, messages[1])
+	suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, messages[2])
+}
+
+func (suite *E2ETestSuite) TestOutputToStderr() {
+	suite.T().Skip("known bug causing all output to be written to stdout (even if it should be written to stderr)")
+	connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "cat -invalid"})
+	suite.Require().NoError(err)
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
+
+	controlMessages := helpers.WebSocketControlMessages(messages)
+	suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, controlMessages[0])
+	suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, controlMessages[1])
+
+	stdout, stderr, errors := helpers.WebSocketOutputMessages(messages)
+	suite.NotContains(stdout, "cat: invalid option", "Stdout should not contain the error")
+	suite.Contains(stderr, "cat: invalid option", "Stderr should contain the error")
+	suite.Empty(errors)
+}
+
+func (suite *E2ETestSuite) TestCommandHead() {
+	hello := "Hello World!"
+	connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "head -n 1"})
+	suite.Require().NoError(err)
+
+	startMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
+	suite.Require().NoError(err)
+	suite.Equal(dto.WebSocketMetaStart, startMessage.Type)
+
+	err = connection.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("%s\n", hello)))
+	suite.Require().NoError(err)
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure})
+	stdout, _, _ := helpers.WebSocketOutputMessages(messages)
+	suite.Contains(stdout, fmt.Sprintf("%s\r\n%s\r\n", hello, hello))
+}
+
+func (suite *E2ETestSuite) TestCommandReturnsAfterTimeout() {
+	connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "sleep 4", TimeLimit: 1})
+	suite.Require().NoError(err)
+
+	c := make(chan bool)
+	var messages []*dto.WebSocketMessage
+	go func() {
+		messages, err = helpers.ReceiveAllWebSocketMessages(connection)
+		if !suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) {
+			suite.T().Fail()
+		}
+		close(c)
+	}()
+
+	select {
+	case <-time.After(2 * time.Second):
+		suite.T().Fatal("The execution should have returned by now")
+	case <-c:
+		if suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaTimeout}, messages[len(messages)-1]) {
+			return
+		}
+	}
+	suite.T().Fail()
+}
+
+func (suite *E2ETestSuite) TestEchoEnvironment() {
+	connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{
+		Command:     "echo $hello",
+		Environment: map[string]string{"hello": "world"},
+	})
+	suite.Require().NoError(err)
+
+	startMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
+	suite.Require().NoError(err)
+	suite.Equal(dto.WebSocketMetaStart, startMessage.Type)
+
+	messages, err := helpers.ReceiveAllWebSocketMessages(connection)
+	suite.Require().Error(err)
+	suite.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure})
+	stdout, _, _ := helpers.WebSocketOutputMessages(messages)
+	suite.Equal("world\r\n", stdout)
+}
+
+// ProvideWebSocketConnection establishes a client WebSocket connection to run the passed ExecutionRequest.
+// It requires a running Poseidon instance.
+func ProvideWebSocketConnection(suite *suite.Suite, request *dto.ExecutionRequest) (connection *websocket.Conn, err error) {
+	runnerId, err := ProvideRunner(suite, &dto.RunnerRequest{})
+	if err != nil {
+		return
+	}
+	webSocketURL, err := ProvideWebSocketURL(suite, runnerId, request)
+	if err != nil {
+		return
+	}
+	connection, err = ConnectToWebSocket(webSocketURL)
+	return
+}
+
+// ProvideRunner creates a runner with the given RunnerRequest via an external request.
+// It needs a running Poseidon instance to work.
+func ProvideRunner(suite *suite.Suite, request *dto.RunnerRequest) (string, error) {
+	url := helpers.BuildURL(api.RouteBase, api.RouteRunners)
+	runnerRequestBytes, _ := json.Marshal(request)
+	reader := strings.NewReader(string(runnerRequestBytes))
+	resp, err := http.Post(url, "application/json", reader)
+	suite.Require().NoError(err)
+	suite.Require().Equal(http.StatusOK, resp.StatusCode)
+
+	runnerResponse := new(dto.RunnerResponse)
+	err = json.NewDecoder(resp.Body).Decode(runnerResponse)
+	suite.Require().NoError(err)
+
+	return runnerResponse.Id, nil
+}
+
+// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request.
+// It requires a running Poseidon instance.
+func ProvideWebSocketURL(suite *suite.Suite, runnerId string, request *dto.ExecutionRequest) (string, error) {
+	url := helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId, api.ExecutePath)
+	executionRequestBytes, _ := json.Marshal(request)
+	reader := strings.NewReader(string(executionRequestBytes))
+	resp, err := http.Post(url, "application/json", reader)
+	suite.Require().NoError(err)
+	suite.Require().Equal(http.StatusOK, resp.StatusCode)
+
+	executionResponse := new(dto.ExecutionResponse)
+	err = json.NewDecoder(resp.Body).Decode(executionResponse)
+	suite.Require().NoError(err)
+	return executionResponse.WebSocketUrl, nil
+}
+
+// ConnectToWebSocket establish an external WebSocket connection to the provided url.
+// It requires a running Poseidon instance.
+func ConnectToWebSocket(url string) (conn *websocket.Conn, err error) {
+	conn, _, err = websocket.DefaultDialer.Dial(url, nil)
+	return
+}

tests/test_constants.go

@@ -7,4 +7,5 @@ const (
 	AnotherJobId                  = "4n0th3r-j0b-1d"
 	DefaultRunnerId               = "s0m3-r4nd0m-1d"
 	AnotherRunnerId               = "4n0th3r-runn3r-1d"
+	DefaultExecutionId            = "s0m3-3x3cu710n-1d"
 )