From 242d0175a2abd5de7578228bf3f0208577703ce9 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 20 May 2021 08:51:25 +0200 Subject: [PATCH] Add tests and end-to-end tests for websocket execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For unit tests, this mocks the runners Execute method with a customizable function that operates on the request, streams and exit channel to simulate a real execution. End-to-end tests are moved to the tests/e2e_tests folder. The tests folder allows us to have shared helper functions for all tests in a separate package (tests) that is not included in the non-test build. This also adds one second of delay before each end-to-end test case by using the TestSetup method of suite. By slowing down test execution, this gives Nomad time to create new allocations when a test requested a runner. Another solution could be to increase the scale of the job to have enough allocations for all end-to-end tests. Co-authored-by: Maximilian Paß --- Makefile | 4 +- api/runners_test.go | 19 +- api/websocket_connection_mock.go | 93 ++++++ api/websocket_test.go | 390 +++++++++++++++++++++--- e2e_tests/helpers.go | 12 - e2e_tests/runners_test.go | 58 ---- environment/job_test.go | 2 +- nomad/api_querier_mock.go | 23 ++ nomad/executor_api_mock.go | 23 ++ runner/runner_mock.go | 94 ++++++ runner/runner_test.go | 67 +++- runner/storage_test.go | 2 + {e2e_tests => tests/e2e}/e2e_test.go | 26 +- {e2e_tests => tests/e2e}/health_test.go | 5 +- tests/e2e/helpers/test_helpers.go | 134 ++++++++ tests/e2e/runners_test.go | 67 ++++ tests/e2e/websocket_test.go | 198 ++++++++++++ tests/test_constants.go | 1 + 18 files changed, 1078 insertions(+), 140 deletions(-) create mode 100644 api/websocket_connection_mock.go delete mode 100644 e2e_tests/helpers.go delete mode 100644 e2e_tests/runners_test.go create mode 100644 runner/runner_mock.go rename {e2e_tests => tests/e2e}/e2e_test.go (61%) rename {e2e_tests => tests/e2e}/health_test.go (66%) create mode 100644 tests/e2e/helpers/test_helpers.go create mode 100644 tests/e2e/runners_test.go create mode 100644 tests/e2e/websocket_test.go diff --git a/Makefile b/Makefile index 96132bc..434e523 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PROJECT_NAME := "poseidon" PKG := "gitlab.hpi.de/codeocean/codemoon/$(PROJECT_NAME)" -UNIT_TESTS = $(shell go list ./... | grep -v /e2e_tests) +UNIT_TESTS = $(shell go list ./... | grep -v /e2e) DOCKER_E2E_CONTAINER_NAME := "$(PROJECT_NAME)-e2e-tests" DOCKER_TAG := "poseidon:latest" @@ -81,7 +81,7 @@ coverhtml: coverage ## Generate HTML coverage report .PHONY: e2e-test e2e-test: deps ## Run e2e tests - @go test -count=1 ./e2e_tests -v + @go test -count=1 ./tests/e2e -v .PHONY: e2e-docker e2e-docker: docker ## Run e2e tests against the Docker container diff --git a/api/runners_test.go b/api/runners_test.go index 5b7da35..06a1000 100644 --- a/api/runners_test.go +++ b/api/runners_test.go @@ -28,7 +28,7 @@ type MiddlewareTestSuite struct { func (suite *MiddlewareTestSuite) SetupTest() { suite.manager = &runner.ManagerMock{} - suite.runner = runner.NewRunner("runner") + suite.runner = runner.NewNomadAllocation("runner", nil) suite.capturedRunner = nil suite.runnerRequest = func(runnerId string) *http.Request { path, err := suite.router.Get("test-runner-id").URL(RunnerIdKey, runnerId) @@ -89,12 +89,15 @@ type RunnerRouteTestSuite struct { runnerManager *runner.ManagerMock router *mux.Router runner runner.Runner + executionId runner.ExecutionId } func (suite *RunnerRouteTestSuite) SetupTest() { suite.runnerManager = &runner.ManagerMock{} suite.router = NewRouter(suite.runnerManager, nil) - suite.runner = runner.NewRunner("test_runner") + suite.runner = runner.NewNomadAllocation("some-id", nil) + suite.executionId = "execution-id" + suite.runner.Add(suite.executionId, &dto.ExecutionRequest{}) suite.runnerManager.On("Get", suite.runner.Id()).Return(suite.runner, nil) } @@ -122,8 +125,8 @@ func (suite *RunnerRouteTestSuite) TestExecuteRoute() { suite.router.ServeHTTP(recorder, request) - var websocketResponse dto.WebsocketResponse - err = json.NewDecoder(recorder.Result().Body).Decode(&websocketResponse) + var webSocketResponse dto.ExecutionResponse + err = json.NewDecoder(recorder.Result().Body).Decode(&webSocketResponse) if err != nil { suite.T().Fatal(err) } @@ -131,15 +134,15 @@ func (suite *RunnerRouteTestSuite) TestExecuteRoute() { suite.Equal(http.StatusOK, recorder.Code) suite.Run("creates an execution request for the runner", func() { - url, err := url.Parse(websocketResponse.WebsocketUrl) + webSocketUrl, err := url.Parse(webSocketResponse.WebSocketUrl) if err != nil { suite.T().Fatal(err) } - executionId := url.Query().Get(ExecutionIdKey) - storedExecutionRequest, ok := suite.runner.Execution(runner.ExecutionId(executionId)) + executionId := webSocketUrl.Query().Get(ExecutionIdKey) + storedExecutionRequest, ok := suite.runner.Pop(runner.ExecutionId(executionId)) suite.True(ok, "No execution request with this id: ", executionId) - suite.Equal(executionRequest, storedExecutionRequest) + suite.Equal(&executionRequest, storedExecutionRequest) }) }) diff --git a/api/websocket_connection_mock.go b/api/websocket_connection_mock.go new file mode 100644 index 0000000..792958e --- /dev/null +++ b/api/websocket_connection_mock.go @@ -0,0 +1,93 @@ +// Code generated by mockery v0.0.0-dev. DO NOT EDIT. + +package api + +import ( + io "io" + + mock "github.com/stretchr/testify/mock" +) + +// webSocketConnectionMock is an autogenerated mock type for the webSocketConnection type +type webSocketConnectionMock struct { + mock.Mock +} + +// Close provides a mock function with given fields: +func (_m *webSocketConnectionMock) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// CloseHandler provides a mock function with given fields: +func (_m *webSocketConnectionMock) CloseHandler() func(int, string) error { + ret := _m.Called() + + var r0 func(int, string) error + if rf, ok := ret.Get(0).(func() func(int, string) error); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(func(int, string) error) + } + } + + return r0 +} + +// NextReader provides a mock function with given fields: +func (_m *webSocketConnectionMock) NextReader() (int, io.Reader, error) { + ret := _m.Called() + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + var r1 io.Reader + if rf, ok := ret.Get(1).(func() io.Reader); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(io.Reader) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func() error); ok { + r2 = rf() + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// SetCloseHandler provides a mock function with given fields: h +func (_m *webSocketConnectionMock) SetCloseHandler(h func(int, string) error) { + _m.Called(h) +} + +// WriteMessage provides a mock function with given fields: messageType, data +func (_m *webSocketConnectionMock) WriteMessage(messageType int, data []byte) error { + ret := _m.Called(messageType, data) + + var r0 error + if rf, ok := ret.Get(0).(func(int, []byte) error); ok { + r0 = rf(messageType, data) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/api/websocket_test.go b/api/websocket_test.go index 0cb7a7e..83ebe34 100644 --- a/api/websocket_test.go +++ b/api/websocket_test.go @@ -1,76 +1,388 @@ package api import ( + "bufio" + "context" + "crypto/tls" + "encoding/json" + "errors" "fmt" + "github.com/gorilla/mux" "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" + "gitlab.hpi.de/codeocean/codemoon/poseidon/nomad" "gitlab.hpi.de/codeocean/codemoon/poseidon/runner" + "gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers" + "io" "net/http" "net/http/httptest" "net/url" "testing" + "time" ) -func TestWebsocketTestSuite(t *testing.T) { - suite.Run(t, new(WebsocketTestSuite)) +func TestWebSocketTestSuite(t *testing.T) { + suite.Run(t, new(WebSocketTestSuite)) } -type WebsocketTestSuite struct { - RunnerRouteTestSuite - server *httptest.Server +type WebSocketTestSuite struct { + suite.Suite + router *mux.Router executionId runner.ExecutionId + runner runner.Runner + apiMock *nomad.ExecutorApiMock + server *httptest.Server } -func (suite *WebsocketTestSuite) SetupTest() { - suite.runnerManager = &runner.ManagerMock{} - suite.router = NewRouter(suite.runnerManager, nil) - suite.runner = runner.NewRunner("test_runner") - suite.runnerManager.On("Get", suite.runner.Id()).Return(suite.runner, nil) - var err error - suite.executionId, err = suite.runner.AddExecution(dto.ExecutionRequest{ - Command: "command", - TimeLimit: 10, - Environment: nil, - }) - suite.Require().NoError(err) +func (suite *WebSocketTestSuite) SetupTest() { + runnerId := "runner-id" + suite.runner, suite.apiMock = helpers.NewNomadAllocationWithMockedApiClient(runnerId) + // default execution + suite.executionId = "execution-id" + suite.runner.Add(suite.executionId, &executionRequestHead) + mockApiExecuteHead(suite.apiMock) + + runnerManager := &runner.ManagerMock{} + runnerManager.On("Get", suite.runner.Id()).Return(suite.runner, nil) + suite.router = NewRouter(runnerManager, nil) suite.server = httptest.NewServer(suite.router) } -func (suite *WebsocketTestSuite) TearDownSuite() { +func (suite *WebSocketTestSuite) TearDownTest() { suite.server.Close() } -func (suite *WebsocketTestSuite) websocketUrl(scheme, runnerId string, executionId runner.ExecutionId) (*url.URL, error) { - websocketUrl, err := url.Parse(suite.server.URL) - suite.Require().NoError(err, "Error: parsing test server url") - path, err := suite.router.Get(WebsocketPath).URL(RunnerIdKey, runnerId) - suite.Require().NoError(err, "could not set runnerId") +func (suite *WebSocketTestSuite) TestWebsocketConnectionCanBeEstablished() { + wsUrl, err := suite.webSocketUrl("ws", suite.runner.Id(), suite.executionId) + suite.Require().NoError(err) + _, _, err = websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Require().NoError(err) +} + +func (suite *WebSocketTestSuite) TestWebsocketReturns404IfExecutionDoesNotExist() { + wsUrl, err := suite.webSocketUrl("ws", suite.runner.Id(), "invalid-execution-id") + suite.Require().NoError(err) + _, response, _ := websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Equal(http.StatusNotFound, response.StatusCode) +} + +func (suite *WebSocketTestSuite) TestWebsocketReturns400IfRequestedViaHttp() { + wsUrl, err := suite.webSocketUrl("http", suite.runner.Id(), suite.executionId) + suite.Require().NoError(err) + response, err := http.Get(wsUrl.String()) + suite.Require().NoError(err) + // This functionality is implemented by the WebSocket library. + suite.Equal(http.StatusBadRequest, response.StatusCode) +} + +func (suite *WebSocketTestSuite) TestWebsocketConnection() { + wsUrl, err := suite.webSocketUrl("ws", suite.runner.Id(), suite.executionId) + suite.Require().NoError(err) + connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Require().NoError(err) + err = connection.SetReadDeadline(time.Now().Add(5 * time.Second)) + suite.Require().NoError(err) + + suite.Run("Receives start message", func() { + message, err := helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(dto.WebSocketMetaStart, message.Type) + }) + + suite.Run("Executes the request in the runner", func() { + <-time.After(100 * time.Millisecond) + suite.apiMock.AssertCalled(suite.T(), "ExecuteCommand", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + suite.Run("Can send input", func() { + err = connection.WriteMessage(websocket.TextMessage, []byte("Hello World\n")) + suite.Require().NoError(err) + }) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) + + suite.Run("Receives output message", func() { + stdout, _, _ := helpers.WebSocketOutputMessages(messages) + suite.Equal("Hello World", stdout) + }) + + suite.Run("Receives exit message", func() { + controlMessages := helpers.WebSocketControlMessages(messages) + suite.Require().Equal(1, len(controlMessages)) + suite.Equal(dto.WebSocketExit, controlMessages[0].Type) + }) +} + +func (suite *WebSocketTestSuite) TestCancelWebSocketConnection() { + executionId := runner.ExecutionId("sleeping-execution") + suite.runner.Add(executionId, &executionRequestSleep) + canceled := mockApiExecuteSleep(suite.apiMock) + + wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId) + suite.Require().NoError(err) + connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Require().NoError(err) + + message, err := helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(dto.WebSocketMetaStart, message.Type) + + select { + case <-canceled: + suite.Fail("Execute canceled unexpected") + default: + } + + err = connection.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + suite.Require().NoError(err) + + select { + case <-canceled: + case <-time.After(time.Second): + suite.Fail("Execute not canceled") + } +} + +func (suite *WebSocketTestSuite) TestWebSocketConnectionTimeout() { + executionId := runner.ExecutionId("time-out-execution") + limitExecution := executionRequestSleep + limitExecution.TimeLimit = 2 + suite.runner.Add(executionId, &limitExecution) + canceled := mockApiExecuteSleep(suite.apiMock) + + wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId) + suite.Require().NoError(err) + connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Require().NoError(err) + + message, err := helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(dto.WebSocketMetaStart, message.Type) + + select { + case <-canceled: + suite.Fail("Execute canceled unexpected") + case <-time.After(time.Duration(limitExecution.TimeLimit-1) * time.Second): + <-time.After(time.Second) + } + + select { + case <-canceled: + case <-time.After(time.Second): + suite.Fail("Execute not canceled") + } + + message, err = helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(dto.WebSocketMetaTimeout, message.Type) +} + +func (suite *WebSocketTestSuite) TestWebsocketStdoutAndStderr() { + executionId := runner.ExecutionId("ls-execution") + suite.runner.Add(executionId, &executionRequestLs) + mockApiExecuteLs(suite.apiMock) + + wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId) + suite.Require().NoError(err) + connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Require().NoError(err) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) + stdout, stderr, _ := helpers.WebSocketOutputMessages(messages) + + suite.Contains(stdout, "existing-file") + + suite.Contains(stderr, "non-existing-file") +} + +func (suite *WebSocketTestSuite) TestWebsocketError() { + executionId := runner.ExecutionId("error-execution") + suite.runner.Add(executionId, &executionRequestError) + mockApiExecuteError(suite.apiMock) + + wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId) + suite.Require().NoError(err) + connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Require().NoError(err) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) + + _, _, errMessages := helpers.WebSocketOutputMessages(messages) + suite.Equal(1, len(errMessages)) + suite.Equal("Error executing the request", errMessages[0]) +} + +func (suite *WebSocketTestSuite) TestWebsocketNonZeroExit() { + executionId := runner.ExecutionId("exit-execution") + suite.runner.Add(executionId, &executionRequestExitNonZero) + mockApiExecuteExitNonZero(suite.apiMock) + + wsUrl, err := webSocketUrl("ws", suite.server, suite.router, suite.runner.Id(), executionId) + suite.Require().NoError(err) + connection, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) + suite.Require().NoError(err) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) + + controlMessages := helpers.WebSocketControlMessages(messages) + suite.Equal(2, len(controlMessages)) + suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit, ExitCode: 42}, controlMessages[1]) +} + +func TestWebsocketTLS(t *testing.T) { + runnerId := "runner-id" + r, apiMock := helpers.NewNomadAllocationWithMockedApiClient(runnerId) + + executionId := runner.ExecutionId("execution-id") + r.Add(executionId, &executionRequestLs) + mockApiExecuteLs(apiMock) + + runnerManager := &runner.ManagerMock{} + runnerManager.On("Get", r.Id()).Return(r, nil) + router := NewRouter(runnerManager, nil) + + server, err := helpers.StartTLSServer(t, router) + require.NoError(t, err) + defer server.Close() + + wsUrl, err := webSocketUrl("wss", server, router, runnerId, executionId) + require.NoError(t, err) + + config := &tls.Config{RootCAs: nil, InsecureSkipVerify: true} + d := websocket.Dialer{TLSClientConfig: config} + connection, _, err := d.Dial(wsUrl.String(), nil) + require.NoError(t, err) + + message, err := helpers.ReceiveNextWebSocketMessage(connection) + require.NoError(t, err) + assert.Equal(t, dto.WebSocketMetaStart, message.Type) + _, err = helpers.ReceiveAllWebSocketMessages(connection) + require.Error(t, err) + assert.Equal(t, &websocket.CloseError{Code: websocket.CloseNormalClosure}, err) +} + +func TestRawToCodeOceanWriter(t *testing.T) { + testMessage := "test" + var message []byte + + connectionMock := &webSocketConnectionMock{} + connectionMock.On("WriteMessage", mock.AnythingOfType("int"), mock.AnythingOfType("[]uint8")). + Run(func(args mock.Arguments) { + message = args.Get(1).([]byte) + }). + Return(nil) + connectionMock.On("CloseHandler").Return(nil) + connectionMock.On("SetCloseHandler", mock.Anything).Return() + + proxy, err := newWebSocketProxy(connectionMock) + require.NoError(t, err) + writer := &rawToCodeOceanWriter{ + proxy: proxy, + outputType: dto.WebSocketOutputStdout, + } + + _, err = writer.Write([]byte(testMessage)) + require.NoError(t, err) + + expected, _ := json.Marshal(struct { + Type string `json:"type"` + Data string `json:"data"` + }{string(dto.WebSocketOutputStdout), testMessage}) + assert.Equal(t, expected, message) +} + +// --- Test suite specific test helpers --- + +func webSocketUrl(scheme string, server *httptest.Server, router *mux.Router, runnerId string, executionId runner.ExecutionId) (*url.URL, error) { + websocketUrl, err := url.Parse(server.URL) + if err != nil { + return nil, err + } + path, err := router.Get(WebsocketPath).URL(RunnerIdKey, runnerId) + if err != nil { + return nil, err + } websocketUrl.Scheme = scheme websocketUrl.Path = path.Path websocketUrl.RawQuery = fmt.Sprintf("executionId=%s", executionId) return websocketUrl, nil } -func (suite *WebsocketTestSuite) TestWebsocketConnectionCanBeEstablished() { - path, err := suite.websocketUrl("ws", suite.runner.Id(), suite.executionId) - suite.Require().NoError(err) - _, _, err = websocket.DefaultDialer.Dial(path.String(), nil) - suite.Require().NoError(err) +func (suite *WebSocketTestSuite) webSocketUrl(scheme, runnerId string, executionId runner.ExecutionId) (*url.URL, error) { + return webSocketUrl(scheme, suite.server, suite.router, runnerId, executionId) } -func (suite *WebsocketTestSuite) TestWebsocketReturns404IfExecutionDoesNotExist() { - wsUrl, err := suite.websocketUrl("ws", suite.runner.Id(), "invalid-execution-id") - suite.Require().NoError(err) - _, response, _ := websocket.DefaultDialer.Dial(wsUrl.String(), nil) - suite.Equal(http.StatusNotFound, response.StatusCode) +var executionRequestLs = dto.ExecutionRequest{Command: "ls"} + +// mockApiExecuteLs mocks the ExecuteCommand of an ExecutorApi to act as if 'ls existing-file non-existing-file' was executed. +func mockApiExecuteLs(api *nomad.ExecutorApiMock) { + helpers.MockApiExecute(api, &executionRequestLs, + func(_ string, _ context.Context, _ []string, _ io.Reader, stdout, stderr io.Writer) (int, error) { + _, _ = stdout.Write([]byte("existing-file\n")) + _, _ = stderr.Write([]byte("ls: cannot access 'non-existing-file': No such file or directory\n")) + return 0, nil + }) } -func (suite *WebsocketTestSuite) TestWebsocketReturns400IfRequestedViaHttp() { - wsUrl, err := suite.websocketUrl("http", suite.runner.Id(), suite.executionId) - suite.Require().NoError(err) - response, err := http.Get(wsUrl.String()) - suite.Require().NoError(err) - suite.Equal(http.StatusBadRequest, response.StatusCode) +var executionRequestHead = dto.ExecutionRequest{Command: "head -n 1"} + +// mockApiExecuteHead mocks the ExecuteCommand of an ExecutorApi to act as if 'head -n 1' was executed. +func mockApiExecuteHead(api *nomad.ExecutorApiMock) { + helpers.MockApiExecute(api, &executionRequestHead, + func(_ string, _ context.Context, _ []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) { + scanner := bufio.NewScanner(stdin) + for !scanner.Scan() { + scanner = bufio.NewScanner(stdin) + } + _, _ = stdout.Write(scanner.Bytes()) + return 0, nil + }) +} + +var executionRequestSleep = dto.ExecutionRequest{Command: "sleep infinity"} + +// mockApiExecuteSleep mocks the ExecuteCommand method of an ExecutorAPI to sleep until the execution is canceled. +func mockApiExecuteSleep(api *nomad.ExecutorApiMock) <-chan bool { + canceled := make(chan bool, 1) + helpers.MockApiExecute(api, &executionRequestSleep, + func(_ string, ctx context.Context, _ []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) { + <-ctx.Done() + close(canceled) + return 0, ctx.Err() + }) + return canceled +} + +var executionRequestError = dto.ExecutionRequest{Command: "error"} + +// mockApiExecuteError mocks the ExecuteCommand method of an ExecutorApi to return an error. +func mockApiExecuteError(api *nomad.ExecutorApiMock) { + helpers.MockApiExecute(api, &executionRequestError, + func(_ string, _ context.Context, _ []string, _ io.Reader, _, _ io.Writer) (int, error) { + return 0, errors.New("intended error") + }) +} + +var executionRequestExitNonZero = dto.ExecutionRequest{Command: "exit 42"} + +// mockApiExecuteExitNonZero mocks the ExecuteCommand method of an ExecutorApi to exit with exit status 42. +func mockApiExecuteExitNonZero(api *nomad.ExecutorApiMock) { + helpers.MockApiExecute(api, &executionRequestExitNonZero, + func(_ string, _ context.Context, _ []string, _ io.Reader, _, _ io.Writer) (int, error) { + return 42, nil + }) } diff --git a/e2e_tests/helpers.go b/e2e_tests/helpers.go deleted file mode 100644 index 74b26fe..0000000 --- a/e2e_tests/helpers.go +++ /dev/null @@ -1,12 +0,0 @@ -package e2e_tests - -import ( - "io" - "net/http" -) - -func httpDelete(url string, body io.Reader) (response *http.Response, err error) { - req, _ := http.NewRequest(http.MethodDelete, url, body) - client := &http.Client{} - return client.Do(req) -} diff --git a/e2e_tests/runners_test.go b/e2e_tests/runners_test.go deleted file mode 100644 index 06f8483..0000000 --- a/e2e_tests/runners_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package e2e_tests - -import ( - "encoding/json" - "github.com/stretchr/testify/assert" - "gitlab.hpi.de/codeocean/codemoon/poseidon/api" - "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" - "net/http" - "strings" - "testing" -) - -func TestProvideRunnerRoute(t *testing.T) { - runnerRequestString, _ := json.Marshal(dto.RunnerRequest{}) - reader := strings.NewReader(string(runnerRequestString)) - resp, err := http.Post(buildURL(api.RouteRunners), "application/json", reader) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode, "The response code should be ok") - - runnerResponse := new(dto.RunnerResponse) - err = json.NewDecoder(resp.Body).Decode(runnerResponse) - assert.NoError(t, err) - - assert.True(t, runnerResponse.Id != "", "The response contains a runner id") -} - -func newRunnerId(t *testing.T) string { - runnerRequestString, _ := json.Marshal(dto.RunnerRequest{}) - reader := strings.NewReader(string(runnerRequestString)) - resp, err := http.Post(buildURL(api.RouteRunners), "application/json", reader) - assert.NoError(t, err) - runnerResponse := new(dto.RunnerResponse) - _ = json.NewDecoder(resp.Body).Decode(runnerResponse) - return runnerResponse.Id -} - -func TestDeleteRunnerRoute(t *testing.T) { - runnerId := newRunnerId(t) - assert.NotEqual(t, "", runnerId) - - t.Run("Deleting the runner returns NoContent", func(t *testing.T) { - resp, err := httpDelete(buildURL(api.RouteRunners, "/", runnerId), nil) - assert.NoError(t, err) - assert.Equal(t, http.StatusNoContent, resp.StatusCode) - }) - - t.Run("Deleting it again returns NotFound", func(t *testing.T) { - resp, err := httpDelete(buildURL(api.RouteRunners, "/", runnerId), nil) - assert.NoError(t, err) - assert.Equal(t, http.StatusNotFound, resp.StatusCode) - }) - - t.Run("Deleting non-existing runner returns NotFound", func(t *testing.T) { - resp, err := httpDelete(buildURL(api.RouteRunners, "/", "n0n-3x1st1ng-1d"), nil) - assert.NoError(t, err) - assert.Equal(t, http.StatusNotFound, resp.StatusCode) - }) -} diff --git a/environment/job_test.go b/environment/job_test.go index d1f8d8f..01f271f 100644 --- a/environment/job_test.go +++ b/environment/job_test.go @@ -55,7 +55,7 @@ func createTestJob() (*nomadApi.Job, *nomadApi.Job) { base := nomadApi.NewBatchJob("python-job", "python-job", "region-name", 100) job := nomadApi.NewBatchJob("python-job", "python-job", "region-name", 100) task := createTestTask() - task.Name = fmt.Sprintf(TaskNameFormat, *job.ID) + task.Name = TaskName image := "python:latest" task.Config = map[string]interface{}{"image": image} task.Config["network_mode"] = "none" diff --git a/nomad/api_querier_mock.go b/nomad/api_querier_mock.go index d471cc0..86257e7 100644 --- a/nomad/api_querier_mock.go +++ b/nomad/api_querier_mock.go @@ -7,6 +7,8 @@ import ( api "github.com/hashicorp/nomad/api" + io "io" + mock "github.com/stretchr/testify/mock" url "net/url" @@ -54,6 +56,27 @@ func (_m *apiQuerierMock) EvaluationStream(evalID string, ctx context.Context) ( return r0, r1 } +// ExecuteCommand provides a mock function with given fields: allocationID, ctx, command, stdin, stdout, stderr +func (_m *apiQuerierMock) ExecuteCommand(allocationID string, ctx context.Context, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) { + ret := _m.Called(allocationID, ctx, command, stdin, stdout, stderr) + + var r0 int + if rf, ok := ret.Get(0).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) int); ok { + r0 = rf(allocationID, ctx, command, stdin, stdout, stderr) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) error); ok { + r1 = rf(allocationID, ctx, command, stdin, stdout, stderr) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // JobScale provides a mock function with given fields: jobId func (_m *apiQuerierMock) JobScale(jobId string) (int, error) { ret := _m.Called(jobId) diff --git a/nomad/executor_api_mock.go b/nomad/executor_api_mock.go index dcbce12..6d6636b 100644 --- a/nomad/executor_api_mock.go +++ b/nomad/executor_api_mock.go @@ -7,6 +7,8 @@ import ( api "github.com/hashicorp/nomad/api" + io "io" + mock "github.com/stretchr/testify/mock" url "net/url" @@ -54,6 +56,27 @@ func (_m *ExecutorApiMock) EvaluationStream(evalID string, ctx context.Context) return r0, r1 } +// ExecuteCommand provides a mock function with given fields: allocationID, ctx, command, stdin, stdout, stderr +func (_m *ExecutorApiMock) ExecuteCommand(allocationID string, ctx context.Context, command []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) { + ret := _m.Called(allocationID, ctx, command, stdin, stdout, stderr) + + var r0 int + if rf, ok := ret.Get(0).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) int); ok { + r0 = rf(allocationID, ctx, command, stdin, stdout, stderr) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, context.Context, []string, io.Reader, io.Writer, io.Writer) error); ok { + r1 = rf(allocationID, ctx, command, stdin, stdout, stderr) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // JobScale provides a mock function with given fields: jobId func (_m *ExecutorApiMock) JobScale(jobId string) (int, error) { ret := _m.Called(jobId) diff --git a/runner/runner_mock.go b/runner/runner_mock.go new file mode 100644 index 0000000..4b3b04b --- /dev/null +++ b/runner/runner_mock.go @@ -0,0 +1,94 @@ +// Code generated by mockery v0.0.0-dev. DO NOT EDIT. + +package runner + +import ( + context "context" + io "io" + + dto "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" + + mock "github.com/stretchr/testify/mock" +) + +// RunnerMock is an autogenerated mock type for the Runner type +type RunnerMock struct { + mock.Mock +} + +// Add provides a mock function with given fields: id, executionRequest +func (_m *RunnerMock) Add(id ExecutionId, executionRequest *dto.ExecutionRequest) { + _m.Called(id, executionRequest) +} + +// Copy provides a mock function with given fields: _a0 +func (_m *RunnerMock) Copy(_a0 dto.FileCreation) { + _m.Called(_a0) +} + +// Delete provides a mock function with given fields: id +func (_m *RunnerMock) Delete(id ExecutionId) { + _m.Called(id) +} + +// Execute provides a mock function with given fields: request, stdin, stdout, stderr +func (_m *RunnerMock) Execute(request *dto.ExecutionRequest, stdin io.Reader, stdout io.Writer, stderr io.Writer) (<-chan ExitInfo, context.CancelFunc) { + ret := _m.Called(request, stdin, stdout, stderr) + + var r0 <-chan ExitInfo + if rf, ok := ret.Get(0).(func(*dto.ExecutionRequest, io.Reader, io.Writer, io.Writer) <-chan ExitInfo); ok { + r0 = rf(request, stdin, stdout, stderr) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan ExitInfo) + } + } + + var r1 context.CancelFunc + if rf, ok := ret.Get(1).(func(*dto.ExecutionRequest, io.Reader, io.Writer, io.Writer) context.CancelFunc); ok { + r1 = rf(request, stdin, stdout, stderr) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(context.CancelFunc) + } + } + + return r0, r1 +} + +// Get provides a mock function with given fields: id +func (_m *RunnerMock) Get(id ExecutionId) (*dto.ExecutionRequest, bool) { + ret := _m.Called(id) + + var r0 *dto.ExecutionRequest + if rf, ok := ret.Get(0).(func(ExecutionId) *dto.ExecutionRequest); ok { + r0 = rf(id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*dto.ExecutionRequest) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(ExecutionId) bool); ok { + r1 = rf(id) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// Id provides a mock function with given fields: +func (_m *RunnerMock) Id() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} diff --git a/runner/runner_test.go b/runner/runner_test.go index ce7e024..1c94ab2 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -4,39 +4,42 @@ import ( "context" "encoding/json" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" + "gitlab.hpi.de/codeocean/codemoon/poseidon/nomad" "testing" + "time" ) func TestIdIsStored(t *testing.T) { - runner := NewRunner("42") + runner := NewNomadAllocation("42", nil) assert.Equal(t, "42", runner.Id()) } func TestMarshalRunner(t *testing.T) { - runner := NewRunner("42") + runner := NewNomadAllocation("42", nil) marshal, err := json.Marshal(runner) assert.NoError(t, err) assert.Equal(t, "{\"runnerId\":\"42\"}", string(marshal)) } func TestExecutionRequestIsStored(t *testing.T) { - runner := NewRunner("42") - executionRequest := dto.ExecutionRequest{ + runner := NewNomadAllocation("42", nil) + executionRequest := &dto.ExecutionRequest{ Command: "command", TimeLimit: 10, Environment: nil, } - id, err := runner.AddExecution(executionRequest) - storedExecutionRunner, ok := runner.Execution(id) + id := ExecutionId("test-execution") + runner.Add(id, executionRequest) + storedExecutionRunner, ok := runner.Pop(id) - assert.NoError(t, err, "AddExecution should not produce an error") assert.True(t, ok, "Getting an execution should not return ok false") assert.Equal(t, executionRequest, storedExecutionRunner) } func TestNewContextReturnsNewContextWithRunner(t *testing.T) { - runner := NewRunner("testRunner") + runner := NewNomadAllocation("testRunner", nil) ctx := context.Background() newCtx := NewContext(ctx, runner) storedRunner := newCtx.Value(runnerContextKey).(Runner) @@ -46,7 +49,7 @@ func TestNewContextReturnsNewContextWithRunner(t *testing.T) { } func TestFromContextReturnsRunner(t *testing.T) { - runner := NewRunner("testRunner") + runner := NewNomadAllocation("testRunner", nil) ctx := NewContext(context.Background(), runner) storedRunner, ok := FromContext(ctx) @@ -60,3 +63,49 @@ func TestFromContextReturnsIsNotOkWhenContextHasNoRunner(t *testing.T) { assert.False(t, ok) } + +func TestExecuteCallsAPI(t *testing.T) { + apiMock := &nomad.ExecutorApiMock{} + apiMock.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(0, nil) + runner := NewNomadAllocation("testRunner", apiMock) + + request := &dto.ExecutionRequest{Command: "echo 'Hello World!'"} + runner.Execute(request, nil, nil, nil) + + <-time.After(50 * time.Millisecond) + apiMock.AssertCalled(t, "ExecuteCommand", "testRunner", mock.Anything, request.FullCommand(), mock.Anything, mock.Anything, mock.Anything) +} + +func TestExecuteReturnsAfterTimeout(t *testing.T) { + apiMock := newApiMockWithTimeLimitHandling() + runner := NewNomadAllocation("testRunner", apiMock) + + timeLimit := 1 + execution := &dto.ExecutionRequest{TimeLimit: timeLimit} + exit, _ := runner.Execute(execution, nil, nil, nil) + + select { + case <-exit: + assert.FailNow(t, "Execute should not terminate instantly") + case <-time.After(50 * time.Millisecond): + } + + select { + case <-time.After(time.Duration(timeLimit) * time.Second): + assert.FailNow(t, "Execute should return after the time limit") + case exitCode := <-exit: + assert.Equal(t, uint8(0), exitCode.Code) + } +} + +func newApiMockWithTimeLimitHandling() (apiMock *nomad.ExecutorApiMock) { + apiMock = &nomad.ExecutorApiMock{} + apiMock. + On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + ctx := args.Get(1).(context.Context) + <-ctx.Done() + }). + Return(0, nil) + return +} diff --git a/runner/storage_test.go b/runner/storage_test.go index 34f69cc..6208c89 100644 --- a/runner/storage_test.go +++ b/runner/storage_test.go @@ -2,6 +2,7 @@ package runner import ( "github.com/stretchr/testify/suite" + "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" "gitlab.hpi.de/codeocean/codemoon/poseidon/tests" "testing" ) @@ -19,6 +20,7 @@ type RunnerPoolTestSuite struct { func (suite *RunnerPoolTestSuite) SetupTest() { suite.runnerStorage = NewLocalRunnerStorage() suite.runner = NewRunner(tests.DefaultRunnerId) + suite.runner.Add(tests.DefaultExecutionId, &dto.ExecutionRequest{Command: "true"}) } func (suite *RunnerPoolTestSuite) TestAddedRunnerCanBeRetrieved() { diff --git a/e2e_tests/e2e_test.go b/tests/e2e/e2e_test.go similarity index 61% rename from e2e_tests/e2e_test.go rename to tests/e2e/e2e_test.go index 5b2c6f9..24114a4 100644 --- a/e2e_tests/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -1,12 +1,12 @@ -package e2e_tests +package e2e import ( - "gitlab.hpi.de/codeocean/codemoon/poseidon/api" + "github.com/stretchr/testify/suite" "gitlab.hpi.de/codeocean/codemoon/poseidon/config" "gitlab.hpi.de/codeocean/codemoon/poseidon/logging" "os" - "strings" "testing" + "time" ) /* @@ -15,7 +15,20 @@ import ( * For the e2e tests a nomad cluster must be connected and poseidon must be running. */ -var log = logging.GetLogger("e2e_tests") +var log = logging.GetLogger("e2e") + +type E2ETestSuite struct { + suite.Suite +} + +func (suite *E2ETestSuite) SetupTest() { + // Waiting one second before each test allows Nomad to rescale after tests requested runners. + <-time.After(time.Second) +} + +func TestE2ETestSuite(t *testing.T) { + suite.Run(t, new(E2ETestSuite)) +} // Overwrite TestMain for custom setup. func TestMain(m *testing.M) { @@ -29,8 +42,3 @@ func TestMain(m *testing.M) { code := m.Run() os.Exit(code) } - -func buildURL(parts ...string) (url string) { - parts = append([]string{config.Config.PoseidonAPIURL().String(), api.RouteBase}, parts...) - return strings.Join(parts, "") -} diff --git a/e2e_tests/health_test.go b/tests/e2e/health_test.go similarity index 66% rename from e2e_tests/health_test.go rename to tests/e2e/health_test.go index c720744..9376ad6 100644 --- a/e2e_tests/health_test.go +++ b/tests/e2e/health_test.go @@ -1,14 +1,15 @@ -package e2e_tests +package e2e import ( "github.com/stretchr/testify/assert" "gitlab.hpi.de/codeocean/codemoon/poseidon/api" + "gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers" "net/http" "testing" ) func TestHealthRoute(t *testing.T) { - resp, err := http.Get(buildURL(api.RouteHealth)) + resp, err := http.Get(helpers.BuildURL(api.RouteBase, api.RouteHealth)) if assert.NoError(t, err) { assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent") } diff --git a/tests/e2e/helpers/test_helpers.go b/tests/e2e/helpers/test_helpers.go new file mode 100644 index 0000000..25c76bb --- /dev/null +++ b/tests/e2e/helpers/test_helpers.go @@ -0,0 +1,134 @@ +// Package helpers contains functions that help executing tests. +// The helper functions generally look from the client side - a Poseidon user. +package helpers + +import ( + "context" + "crypto/tls" + "encoding/json" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/mock" + "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" + "gitlab.hpi.de/codeocean/codemoon/poseidon/config" + "gitlab.hpi.de/codeocean/codemoon/poseidon/nomad" + "gitlab.hpi.de/codeocean/codemoon/poseidon/runner" + "io" + "net/http/httptest" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// BuildURL joins multiple route paths. +func BuildURL(parts ...string) (url string) { + parts = append([]string{config.Config.PoseidonAPIURL().String()}, parts...) + return strings.Join(parts, "") +} + +// WebSocketOutputMessages extracts all stdout, stderr and error messages from the passed messages. +// It is useful since Nomad splits the command output nondeterministic. +func WebSocketOutputMessages(messages []*dto.WebSocketMessage) (stdout, stderr string, errors []string) { + for _, msg := range messages { + switch msg.Type { + case dto.WebSocketOutputStdout: + stdout += msg.Data + case dto.WebSocketOutputStderr: + stderr += msg.Data + case dto.WebSocketOutputError: + errors = append(errors, msg.Data) + } + } + return +} + +// WebSocketControlMessages extracts all meta (and exit) messages from the passed messages. +func WebSocketControlMessages(messages []*dto.WebSocketMessage) (controls []*dto.WebSocketMessage) { + for _, msg := range messages { + switch msg.Type { + case dto.WebSocketMetaStart, dto.WebSocketMetaTimeout, dto.WebSocketExit: + controls = append(controls, msg) + } + } + return +} + +// ReceiveAllWebSocketMessages pulls all messages from the websocket connection without sending anything. +// This function does not return unless the server closes the connection or a readDeadline is set in the WebSocket connection. +func ReceiveAllWebSocketMessages(connection *websocket.Conn) (messages []*dto.WebSocketMessage, err error) { + for { + var message *dto.WebSocketMessage + message, err = ReceiveNextWebSocketMessage(connection) + if err != nil { + return + } + messages = append(messages, message) + } +} + +// ReceiveNextWebSocketMessage pulls the next message from the websocket connection. +// This function does not return unless the server sends a message, closes the connection or a readDeadline is set in the WebSocket connection. +func ReceiveNextWebSocketMessage(connection *websocket.Conn) (*dto.WebSocketMessage, error) { + _, reader, err := connection.NextReader() + if err != nil { + return nil, err + } + message := new(dto.WebSocketMessage) + err = json.NewDecoder(reader).Decode(message) + if err != nil { + return nil, err + } + return message, nil +} + +// StartTLSServer runs a httptest.Server with the passed mux.Router and TLS enabled. +func StartTLSServer(t *testing.T, router *mux.Router) (server *httptest.Server, err error) { + dir := t.TempDir() + keyOut := filepath.Join(dir, "poseidon-test.key") + certOut := filepath.Join(dir, "poseidon-test.crt") + + err = exec.Command("openssl", "req", "-x509", "-nodes", "-newkey", "rsa:2048", + "-keyout", keyOut, "-out", certOut, "-days", "1", + "-subj", "/CN=Poseidon test", "-addext", "subjectAltName=IP:127.0.0.1,DNS:localhost").Run() + if err != nil { + return nil, err + } + cert, err := tls.LoadX509KeyPair(certOut, keyOut) + if err != nil { + return nil, err + } + + server = httptest.NewUnstartedServer(router) + server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + server.StartTLS() + return +} + +func NewNomadAllocationWithMockedApiClient(runnerId string) (r runner.Runner, mock *nomad.ExecutorApiMock) { + mock = &nomad.ExecutorApiMock{} + r = runner.NewNomadAllocation(runnerId, mock) + return +} + +// MockApiExecute mocks the ExecuteCommand method of an ExecutorApi to call the given method run when the command +// corresponding to the given ExecutionRequest is called. +func MockApiExecute(api *nomad.ExecutorApiMock, request *dto.ExecutionRequest, + run func(runnerId string, ctx context.Context, command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error)) { + call := api.On("ExecuteCommand", + mock.AnythingOfType("string"), + mock.Anything, + request.FullCommand(), + mock.Anything, + mock.Anything, + mock.Anything) + call.Run(func(args mock.Arguments) { + exit, err := run(args.Get(0).(string), + args.Get(1).(context.Context), + args.Get(2).([]string), + args.Get(3).(io.Reader), + args.Get(4).(io.Writer), + args.Get(5).(io.Writer)) + call.ReturnArguments = mock.Arguments{exit, err} + }) +} diff --git a/tests/e2e/runners_test.go b/tests/e2e/runners_test.go new file mode 100644 index 0000000..cdc01e7 --- /dev/null +++ b/tests/e2e/runners_test.go @@ -0,0 +1,67 @@ +package e2e + +import ( + "encoding/json" + "github.com/stretchr/testify/assert" + "gitlab.hpi.de/codeocean/codemoon/poseidon/api" + "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" + "gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers" + "io" + "net/http" + "strings" + "testing" +) + +func (suite *E2ETestSuite) TestProvideRunnerRoute() { + runnerRequestString, _ := json.Marshal(dto.RunnerRequest{}) + reader := strings.NewReader(string(runnerRequestString)) + resp, err := http.Post(helpers.BuildURL(api.RouteBase, api.RouteRunners), "application/json", reader) + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode, "The response code should be ok") + + runnerResponse := new(dto.RunnerResponse) + err = json.NewDecoder(resp.Body).Decode(runnerResponse) + suite.NoError(err) + + suite.True(runnerResponse.Id != "", "The response contains a runner id") +} + +func newRunnerId(t *testing.T) string { + runnerRequestString, _ := json.Marshal(dto.RunnerRequest{}) + reader := strings.NewReader(string(runnerRequestString)) + resp, err := http.Post(helpers.BuildURL(api.RouteBase, api.RouteRunners), "application/json", reader) + assert.NoError(t, err) + runnerResponse := new(dto.RunnerResponse) + _ = json.NewDecoder(resp.Body).Decode(runnerResponse) + return runnerResponse.Id +} + +func (suite *E2ETestSuite) TestDeleteRunnerRoute() { + runnerId := newRunnerId(suite.T()) + suite.NotEqual("", runnerId) + + suite.Run("Deleting the runner returns NoContent", func() { + resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId), nil) + suite.NoError(err) + suite.Equal(http.StatusNoContent, resp.StatusCode) + }) + + suite.Run("Deleting it again returns NotFound", func() { + resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId), nil) + suite.NoError(err) + suite.Equal(http.StatusNotFound, resp.StatusCode) + }) + + suite.Run("Deleting non-existing runner returns NotFound", func() { + resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", "n0n-3x1st1ng-1d"), nil) + suite.NoError(err) + suite.Equal(http.StatusNotFound, resp.StatusCode) + }) +} + +// HttpDelete sends a Delete Http Request with body to the passed url. +func httpDelete(url string, body io.Reader) (response *http.Response, err error) { + req, _ := http.NewRequest(http.MethodDelete, url, body) + client := &http.Client{} + return client.Do(req) +} diff --git a/tests/e2e/websocket_test.go b/tests/e2e/websocket_test.go new file mode 100644 index 0000000..f90b9d2 --- /dev/null +++ b/tests/e2e/websocket_test.go @@ -0,0 +1,198 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/suite" + "gitlab.hpi.de/codeocean/codemoon/poseidon/api" + "gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto" + "gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers" + "net/http" + "strings" + "time" +) + +func (suite *E2ETestSuite) TestExecuteCommandRoute() { + runnerId, err := ProvideRunner(&suite.Suite, &dto.RunnerRequest{}) + suite.Require().NoError(err) + + webSocketURL, err := ProvideWebSocketURL(&suite.Suite, runnerId, &dto.ExecutionRequest{Command: "true"}) + suite.Require().NoError(err) + suite.NotEqual("", webSocketURL) + + var connection *websocket.Conn + var connectionClosed bool + + connection, err = ConnectToWebSocket(webSocketURL) + suite.Require().NoError(err, "websocket connects") + closeHandler := connection.CloseHandler() + connection.SetCloseHandler(func(code int, text string) error { + connectionClosed = true + return closeHandler(code, text) + }) + + startMessage, err := helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, startMessage) + + exitMessage, err := helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, exitMessage) + + _, err = helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) + + _, _, _ = connection.ReadMessage() + suite.True(connectionClosed, "connection should be closed") +} + +func (suite *E2ETestSuite) TestOutputToStdout() { + connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "echo Hello World"}) + suite.Require().NoError(err) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) + + suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, messages[0]) + suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketOutputStdout, Data: "Hello World\r\n"}, messages[1]) + suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, messages[2]) +} + +func (suite *E2ETestSuite) TestOutputToStderr() { + suite.T().Skip("known bug causing all output to be written to stdout (even if it should be written to stderr)") + connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "cat -invalid"}) + suite.Require().NoError(err) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) + + controlMessages := helpers.WebSocketControlMessages(messages) + suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, controlMessages[0]) + suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, controlMessages[1]) + + stdout, stderr, errors := helpers.WebSocketOutputMessages(messages) + suite.NotContains(stdout, "cat: invalid option", "Stdout should not contain the error") + suite.Contains(stderr, "cat: invalid option", "Stderr should contain the error") + suite.Empty(errors) +} + +func (suite *E2ETestSuite) TestCommandHead() { + hello := "Hello World!" + connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "head -n 1"}) + suite.Require().NoError(err) + + startMessage, err := helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(dto.WebSocketMetaStart, startMessage.Type) + + err = connection.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("%s\n", hello))) + suite.Require().NoError(err) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure}) + stdout, _, _ := helpers.WebSocketOutputMessages(messages) + suite.Contains(stdout, fmt.Sprintf("%s\r\n%s\r\n", hello, hello)) +} + +func (suite *E2ETestSuite) TestCommandReturnsAfterTimeout() { + connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "sleep 4", TimeLimit: 1}) + suite.Require().NoError(err) + + c := make(chan bool) + var messages []*dto.WebSocketMessage + go func() { + messages, err = helpers.ReceiveAllWebSocketMessages(connection) + if !suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) { + suite.T().Fail() + } + close(c) + }() + + select { + case <-time.After(2 * time.Second): + suite.T().Fatal("The execution should have returned by now") + case <-c: + if suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaTimeout}, messages[len(messages)-1]) { + return + } + } + suite.T().Fail() +} + +func (suite *E2ETestSuite) TestEchoEnvironment() { + connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{ + Command: "echo $hello", + Environment: map[string]string{"hello": "world"}, + }) + suite.Require().NoError(err) + + startMessage, err := helpers.ReceiveNextWebSocketMessage(connection) + suite.Require().NoError(err) + suite.Equal(dto.WebSocketMetaStart, startMessage.Type) + + messages, err := helpers.ReceiveAllWebSocketMessages(connection) + suite.Require().Error(err) + suite.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure}) + stdout, _, _ := helpers.WebSocketOutputMessages(messages) + suite.Equal("world\r\n", stdout) +} + +// ProvideWebSocketConnection establishes a client WebSocket connection to run the passed ExecutionRequest. +// It requires a running Poseidon instance. +func ProvideWebSocketConnection(suite *suite.Suite, request *dto.ExecutionRequest) (connection *websocket.Conn, err error) { + runnerId, err := ProvideRunner(suite, &dto.RunnerRequest{}) + if err != nil { + return + } + webSocketURL, err := ProvideWebSocketURL(suite, runnerId, request) + if err != nil { + return + } + connection, err = ConnectToWebSocket(webSocketURL) + return +} + +// ProvideRunner creates a runner with the given RunnerRequest via an external request. +// It needs a running Poseidon instance to work. +func ProvideRunner(suite *suite.Suite, request *dto.RunnerRequest) (string, error) { + url := helpers.BuildURL(api.RouteBase, api.RouteRunners) + runnerRequestBytes, _ := json.Marshal(request) + reader := strings.NewReader(string(runnerRequestBytes)) + resp, err := http.Post(url, "application/json", reader) + suite.Require().NoError(err) + suite.Require().Equal(http.StatusOK, resp.StatusCode) + + runnerResponse := new(dto.RunnerResponse) + err = json.NewDecoder(resp.Body).Decode(runnerResponse) + suite.Require().NoError(err) + + return runnerResponse.Id, nil +} + +// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request. +// It requires a running Poseidon instance. +func ProvideWebSocketURL(suite *suite.Suite, runnerId string, request *dto.ExecutionRequest) (string, error) { + url := helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId, api.ExecutePath) + executionRequestBytes, _ := json.Marshal(request) + reader := strings.NewReader(string(executionRequestBytes)) + resp, err := http.Post(url, "application/json", reader) + suite.Require().NoError(err) + suite.Require().Equal(http.StatusOK, resp.StatusCode) + + executionResponse := new(dto.ExecutionResponse) + err = json.NewDecoder(resp.Body).Decode(executionResponse) + suite.Require().NoError(err) + return executionResponse.WebSocketUrl, nil +} + +// ConnectToWebSocket establish an external WebSocket connection to the provided url. +// It requires a running Poseidon instance. +func ConnectToWebSocket(url string) (conn *websocket.Conn, err error) { + conn, _, err = websocket.DefaultDialer.Dial(url, nil) + return +} diff --git a/tests/test_constants.go b/tests/test_constants.go index 6317586..25f4781 100644 --- a/tests/test_constants.go +++ b/tests/test_constants.go @@ -7,4 +7,5 @@ const ( AnotherJobId = "4n0th3r-j0b-1d" DefaultRunnerId = "s0m3-r4nd0m-1d" AnotherRunnerId = "4n0th3r-runn3r-1d" + DefaultExecutionId = "s0m3-3x3cu710n-1d" )