Add tests and end-to-end tests for websocket execution

For unit tests, this mocks the runners Execute method with a
customizable function that operates on the request, streams and exit
channel to simulate a real execution.

End-to-end tests are moved to the tests/e2e_tests folder. The tests
folder allows us to have shared helper functions for all tests in a
separate package (tests) that is not included in the non-test build.

This also adds one second of delay before each end-to-end test case by
using the TestSetup method of suite. By slowing down test execution,
this gives Nomad time to create new allocations when a test requested a
runner. Another solution could be to increase the scale of the job to
have enough allocations for all end-to-end tests.

Co-authored-by: Maximilian Paß <maximilian.pass@student.hpi.uni-potsdam.de>
This commit is contained in:
Konrad Hanff
2021-05-20 08:51:25 +02:00
parent 3afcdeaba8
commit 242d0175a2
18 changed files with 1078 additions and 140 deletions

44
tests/e2e/e2e_test.go Normal file
View File

@@ -0,0 +1,44 @@
package e2e
import (
"github.com/stretchr/testify/suite"
"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
"gitlab.hpi.de/codeocean/codemoon/poseidon/logging"
"os"
"testing"
"time"
)
/*
* # E2E Tests
*
* For the e2e tests a nomad cluster must be connected and poseidon must be running.
*/
var log = logging.GetLogger("e2e")
type E2ETestSuite struct {
suite.Suite
}
func (suite *E2ETestSuite) SetupTest() {
// Waiting one second before each test allows Nomad to rescale after tests requested runners.
<-time.After(time.Second)
}
func TestE2ETestSuite(t *testing.T) {
suite.Run(t, new(E2ETestSuite))
}
// Overwrite TestMain for custom setup.
func TestMain(m *testing.M) {
log.Info("Test Setup")
err := config.InitConfig()
if err != nil {
log.Warn("Could not initialize configuration")
}
// ToDo: Add Nomad job here when it is possible to create execution environments. See #26.
log.Info("Test Run")
code := m.Run()
os.Exit(code)
}

16
tests/e2e/health_test.go Normal file
View File

@@ -0,0 +1,16 @@
package e2e
import (
"github.com/stretchr/testify/assert"
"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
"net/http"
"testing"
)
func TestHealthRoute(t *testing.T) {
resp, err := http.Get(helpers.BuildURL(api.RouteBase, api.RouteHealth))
if assert.NoError(t, err) {
assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent")
}
}

View File

@@ -0,0 +1,134 @@
// Package helpers contains functions that help executing tests.
// The helper functions generally look from the client side - a Poseidon user.
package helpers
import (
"context"
"crypto/tls"
"encoding/json"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/mock"
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
"io"
"net/http/httptest"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// BuildURL joins multiple route paths.
func BuildURL(parts ...string) (url string) {
parts = append([]string{config.Config.PoseidonAPIURL().String()}, parts...)
return strings.Join(parts, "")
}
// WebSocketOutputMessages extracts all stdout, stderr and error messages from the passed messages.
// It is useful since Nomad splits the command output nondeterministic.
func WebSocketOutputMessages(messages []*dto.WebSocketMessage) (stdout, stderr string, errors []string) {
for _, msg := range messages {
switch msg.Type {
case dto.WebSocketOutputStdout:
stdout += msg.Data
case dto.WebSocketOutputStderr:
stderr += msg.Data
case dto.WebSocketOutputError:
errors = append(errors, msg.Data)
}
}
return
}
// WebSocketControlMessages extracts all meta (and exit) messages from the passed messages.
func WebSocketControlMessages(messages []*dto.WebSocketMessage) (controls []*dto.WebSocketMessage) {
for _, msg := range messages {
switch msg.Type {
case dto.WebSocketMetaStart, dto.WebSocketMetaTimeout, dto.WebSocketExit:
controls = append(controls, msg)
}
}
return
}
// ReceiveAllWebSocketMessages pulls all messages from the websocket connection without sending anything.
// This function does not return unless the server closes the connection or a readDeadline is set in the WebSocket connection.
func ReceiveAllWebSocketMessages(connection *websocket.Conn) (messages []*dto.WebSocketMessage, err error) {
for {
var message *dto.WebSocketMessage
message, err = ReceiveNextWebSocketMessage(connection)
if err != nil {
return
}
messages = append(messages, message)
}
}
// ReceiveNextWebSocketMessage pulls the next message from the websocket connection.
// This function does not return unless the server sends a message, closes the connection or a readDeadline is set in the WebSocket connection.
func ReceiveNextWebSocketMessage(connection *websocket.Conn) (*dto.WebSocketMessage, error) {
_, reader, err := connection.NextReader()
if err != nil {
return nil, err
}
message := new(dto.WebSocketMessage)
err = json.NewDecoder(reader).Decode(message)
if err != nil {
return nil, err
}
return message, nil
}
// StartTLSServer runs a httptest.Server with the passed mux.Router and TLS enabled.
func StartTLSServer(t *testing.T, router *mux.Router) (server *httptest.Server, err error) {
dir := t.TempDir()
keyOut := filepath.Join(dir, "poseidon-test.key")
certOut := filepath.Join(dir, "poseidon-test.crt")
err = exec.Command("openssl", "req", "-x509", "-nodes", "-newkey", "rsa:2048",
"-keyout", keyOut, "-out", certOut, "-days", "1",
"-subj", "/CN=Poseidon test", "-addext", "subjectAltName=IP:127.0.0.1,DNS:localhost").Run()
if err != nil {
return nil, err
}
cert, err := tls.LoadX509KeyPair(certOut, keyOut)
if err != nil {
return nil, err
}
server = httptest.NewUnstartedServer(router)
server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}}
server.StartTLS()
return
}
func NewNomadAllocationWithMockedApiClient(runnerId string) (r runner.Runner, mock *nomad.ExecutorApiMock) {
mock = &nomad.ExecutorApiMock{}
r = runner.NewNomadAllocation(runnerId, mock)
return
}
// MockApiExecute mocks the ExecuteCommand method of an ExecutorApi to call the given method run when the command
// corresponding to the given ExecutionRequest is called.
func MockApiExecute(api *nomad.ExecutorApiMock, request *dto.ExecutionRequest,
run func(runnerId string, ctx context.Context, command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error)) {
call := api.On("ExecuteCommand",
mock.AnythingOfType("string"),
mock.Anything,
request.FullCommand(),
mock.Anything,
mock.Anything,
mock.Anything)
call.Run(func(args mock.Arguments) {
exit, err := run(args.Get(0).(string),
args.Get(1).(context.Context),
args.Get(2).([]string),
args.Get(3).(io.Reader),
args.Get(4).(io.Writer),
args.Get(5).(io.Writer))
call.ReturnArguments = mock.Arguments{exit, err}
})
}

67
tests/e2e/runners_test.go Normal file
View File

@@ -0,0 +1,67 @@
package e2e
import (
"encoding/json"
"github.com/stretchr/testify/assert"
"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
"io"
"net/http"
"strings"
"testing"
)
func (suite *E2ETestSuite) TestProvideRunnerRoute() {
runnerRequestString, _ := json.Marshal(dto.RunnerRequest{})
reader := strings.NewReader(string(runnerRequestString))
resp, err := http.Post(helpers.BuildURL(api.RouteBase, api.RouteRunners), "application/json", reader)
suite.NoError(err)
suite.Equal(http.StatusOK, resp.StatusCode, "The response code should be ok")
runnerResponse := new(dto.RunnerResponse)
err = json.NewDecoder(resp.Body).Decode(runnerResponse)
suite.NoError(err)
suite.True(runnerResponse.Id != "", "The response contains a runner id")
}
func newRunnerId(t *testing.T) string {
runnerRequestString, _ := json.Marshal(dto.RunnerRequest{})
reader := strings.NewReader(string(runnerRequestString))
resp, err := http.Post(helpers.BuildURL(api.RouteBase, api.RouteRunners), "application/json", reader)
assert.NoError(t, err)
runnerResponse := new(dto.RunnerResponse)
_ = json.NewDecoder(resp.Body).Decode(runnerResponse)
return runnerResponse.Id
}
func (suite *E2ETestSuite) TestDeleteRunnerRoute() {
runnerId := newRunnerId(suite.T())
suite.NotEqual("", runnerId)
suite.Run("Deleting the runner returns NoContent", func() {
resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId), nil)
suite.NoError(err)
suite.Equal(http.StatusNoContent, resp.StatusCode)
})
suite.Run("Deleting it again returns NotFound", func() {
resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId), nil)
suite.NoError(err)
suite.Equal(http.StatusNotFound, resp.StatusCode)
})
suite.Run("Deleting non-existing runner returns NotFound", func() {
resp, err := httpDelete(helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", "n0n-3x1st1ng-1d"), nil)
suite.NoError(err)
suite.Equal(http.StatusNotFound, resp.StatusCode)
})
}
// HttpDelete sends a Delete Http Request with body to the passed url.
func httpDelete(url string, body io.Reader) (response *http.Response, err error) {
req, _ := http.NewRequest(http.MethodDelete, url, body)
client := &http.Client{}
return client.Do(req)
}

198
tests/e2e/websocket_test.go Normal file
View File

@@ -0,0 +1,198 @@
package e2e
import (
"encoding/json"
"fmt"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/suite"
"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
"net/http"
"strings"
"time"
)
func (suite *E2ETestSuite) TestExecuteCommandRoute() {
runnerId, err := ProvideRunner(&suite.Suite, &dto.RunnerRequest{})
suite.Require().NoError(err)
webSocketURL, err := ProvideWebSocketURL(&suite.Suite, runnerId, &dto.ExecutionRequest{Command: "true"})
suite.Require().NoError(err)
suite.NotEqual("", webSocketURL)
var connection *websocket.Conn
var connectionClosed bool
connection, err = ConnectToWebSocket(webSocketURL)
suite.Require().NoError(err, "websocket connects")
closeHandler := connection.CloseHandler()
connection.SetCloseHandler(func(code int, text string) error {
connectionClosed = true
return closeHandler(code, text)
})
startMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
suite.Require().NoError(err)
suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, startMessage)
exitMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
suite.Require().NoError(err)
suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, exitMessage)
_, err = helpers.ReceiveAllWebSocketMessages(connection)
suite.Require().Error(err)
suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
_, _, _ = connection.ReadMessage()
suite.True(connectionClosed, "connection should be closed")
}
func (suite *E2ETestSuite) TestOutputToStdout() {
connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "echo Hello World"})
suite.Require().NoError(err)
messages, err := helpers.ReceiveAllWebSocketMessages(connection)
suite.Require().Error(err)
suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, messages[0])
suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketOutputStdout, Data: "Hello World\r\n"}, messages[1])
suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, messages[2])
}
func (suite *E2ETestSuite) TestOutputToStderr() {
suite.T().Skip("known bug causing all output to be written to stdout (even if it should be written to stderr)")
connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "cat -invalid"})
suite.Require().NoError(err)
messages, err := helpers.ReceiveAllWebSocketMessages(connection)
suite.Require().Error(err)
suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err)
controlMessages := helpers.WebSocketControlMessages(messages)
suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, controlMessages[0])
suite.Require().Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, controlMessages[1])
stdout, stderr, errors := helpers.WebSocketOutputMessages(messages)
suite.NotContains(stdout, "cat: invalid option", "Stdout should not contain the error")
suite.Contains(stderr, "cat: invalid option", "Stderr should contain the error")
suite.Empty(errors)
}
func (suite *E2ETestSuite) TestCommandHead() {
hello := "Hello World!"
connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "head -n 1"})
suite.Require().NoError(err)
startMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
suite.Require().NoError(err)
suite.Equal(dto.WebSocketMetaStart, startMessage.Type)
err = connection.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("%s\n", hello)))
suite.Require().NoError(err)
messages, err := helpers.ReceiveAllWebSocketMessages(connection)
suite.Require().Error(err)
suite.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure})
stdout, _, _ := helpers.WebSocketOutputMessages(messages)
suite.Contains(stdout, fmt.Sprintf("%s\r\n%s\r\n", hello, hello))
}
func (suite *E2ETestSuite) TestCommandReturnsAfterTimeout() {
connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{Command: "sleep 4", TimeLimit: 1})
suite.Require().NoError(err)
c := make(chan bool)
var messages []*dto.WebSocketMessage
go func() {
messages, err = helpers.ReceiveAllWebSocketMessages(connection)
if !suite.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) {
suite.T().Fail()
}
close(c)
}()
select {
case <-time.After(2 * time.Second):
suite.T().Fatal("The execution should have returned by now")
case <-c:
if suite.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaTimeout}, messages[len(messages)-1]) {
return
}
}
suite.T().Fail()
}
func (suite *E2ETestSuite) TestEchoEnvironment() {
connection, err := ProvideWebSocketConnection(&suite.Suite, &dto.ExecutionRequest{
Command: "echo $hello",
Environment: map[string]string{"hello": "world"},
})
suite.Require().NoError(err)
startMessage, err := helpers.ReceiveNextWebSocketMessage(connection)
suite.Require().NoError(err)
suite.Equal(dto.WebSocketMetaStart, startMessage.Type)
messages, err := helpers.ReceiveAllWebSocketMessages(connection)
suite.Require().Error(err)
suite.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure})
stdout, _, _ := helpers.WebSocketOutputMessages(messages)
suite.Equal("world\r\n", stdout)
}
// ProvideWebSocketConnection establishes a client WebSocket connection to run the passed ExecutionRequest.
// It requires a running Poseidon instance.
func ProvideWebSocketConnection(suite *suite.Suite, request *dto.ExecutionRequest) (connection *websocket.Conn, err error) {
runnerId, err := ProvideRunner(suite, &dto.RunnerRequest{})
if err != nil {
return
}
webSocketURL, err := ProvideWebSocketURL(suite, runnerId, request)
if err != nil {
return
}
connection, err = ConnectToWebSocket(webSocketURL)
return
}
// ProvideRunner creates a runner with the given RunnerRequest via an external request.
// It needs a running Poseidon instance to work.
func ProvideRunner(suite *suite.Suite, request *dto.RunnerRequest) (string, error) {
url := helpers.BuildURL(api.RouteBase, api.RouteRunners)
runnerRequestBytes, _ := json.Marshal(request)
reader := strings.NewReader(string(runnerRequestBytes))
resp, err := http.Post(url, "application/json", reader)
suite.Require().NoError(err)
suite.Require().Equal(http.StatusOK, resp.StatusCode)
runnerResponse := new(dto.RunnerResponse)
err = json.NewDecoder(resp.Body).Decode(runnerResponse)
suite.Require().NoError(err)
return runnerResponse.Id, nil
}
// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request.
// It requires a running Poseidon instance.
func ProvideWebSocketURL(suite *suite.Suite, runnerId string, request *dto.ExecutionRequest) (string, error) {
url := helpers.BuildURL(api.RouteBase, api.RouteRunners, "/", runnerId, api.ExecutePath)
executionRequestBytes, _ := json.Marshal(request)
reader := strings.NewReader(string(executionRequestBytes))
resp, err := http.Post(url, "application/json", reader)
suite.Require().NoError(err)
suite.Require().Equal(http.StatusOK, resp.StatusCode)
executionResponse := new(dto.ExecutionResponse)
err = json.NewDecoder(resp.Body).Decode(executionResponse)
suite.Require().NoError(err)
return executionResponse.WebSocketUrl, nil
}
// ConnectToWebSocket establish an external WebSocket connection to the provided url.
// It requires a running Poseidon instance.
func ConnectToWebSocket(url string) (conn *websocket.Conn, err error) {
conn, _, err = websocket.DefaultDialer.Dial(url, nil)
return
}