htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ability to copy files to and delete files from runner

Browse Source
This commit is contained in:
Jan-Eric Hellenberg
2021-05-31 16:31:15 +02:00
parent 242d0175a2
commit 02b3f52a11
23 changed files with 757 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
api/api.go
View File

@ -12,9 +12,9 @@ import (
var log = logging.GetLogger("api")
const (
RouteBase = "/api/v1"
RouteHealth = "/health"
RouteRunners = "/runners"
BasePath = "/api/v1"
HealthPath = "/health"
RunnersPath = "/runners"
)
// NewRouter returns a *mux.Router which can be
@ -33,8 +33,8 @@ func NewRouter(runnerManager runner.Manager, environmentManager environment.Mana
// configureV1Router configures a given router with the routes of version 1 of the Poseidon API.
func configureV1Router(router *mux.Router, runnerManager runner.Manager, environmentManager environment.Manager) {
v1 := router.PathPrefix(RouteBase).Subrouter()
v1.HandleFunc(RouteHealth, Health).Methods(http.MethodGet)
v1 := router.PathPrefix(BasePath).Subrouter()
v1.HandleFunc(HealthPath, Health).Methods(http.MethodGet)
runnerController := &RunnerController{manager: runnerManager}

49
api/dto/dto.go
View File

@ -4,6 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"path"
"strings"
)
// RunnerRequest is the expected json structure of the request body for the ProvideRunner function.
@ -45,15 +47,54 @@ type RunnerResponse struct {
Id string `json:"runnerId"`
}
// FileCreation is the expected json structure of the request body for the copy files route.
// TODO: specify content of the struct
type FileCreation struct{}
// ExecutionResponse is the expected response when creating an execution for a runner.
type ExecutionResponse struct {
WebSocketUrl string `json:"websocketUrl"`
}
// UpdateFileSystemRequest is the expected json structure of the request body for the update file system route.
type UpdateFileSystemRequest struct {
Delete []FilePath `json:"delete"`
Copy []File `json:"copy"`
}
// FilePath specifies the path of a file and is part of the UpdateFileSystemRequest.
type FilePath string
// File is a DTO for transmitting file contents. It is part of the UpdateFileSystemRequest.
type File struct {
Path FilePath `json:"path"`
Content []byte `json:"content"`
}
// ToAbsolute returns the absolute path of the FilePath with respect to the given basePath. If the FilePath already is absolute, basePath will be ignored.
func (f FilePath) ToAbsolute(basePath string) string {
filePathString := string(f)
if path.IsAbs(filePathString) {
return path.Clean(filePathString)
}
return path.Clean(path.Join(basePath, filePathString))
}
// AbsolutePath returns the absolute path of the file. See FilePath.ToAbsolute for details.
func (f File) AbsolutePath(basePath string) string {
return f.Path.ToAbsolute(basePath)
}
// IsDirectory returns true iff the path of the File ends with a /.
func (f File) IsDirectory() bool {
return strings.HasSuffix(string(f.Path), "/")
}
// ByteContent returns the content of the File. If the File is a directory, the content will be empty.
func (f File) ByteContent() []byte {
if f.IsDirectory() {
return []byte("")
} else {
return f.Content
}
}
// WebSocketMessageType is the type for the messages from Poseidon to the client.
type WebSocketMessageType string

34
api/runners.go
View File

@ -12,11 +12,12 @@ import (
)
const (
ExecutePath = "/execute"
WebsocketPath = "/websocket"
DeleteRoute = "deleteRunner"
RunnerIdKey = "runnerId"
ExecutionIdKey = "executionId"
ExecutePath = "/execute"
WebsocketPath = "/websocket"
UpdateFileSystemPath = "/files"
DeleteRoute = "deleteRunner"
RunnerIdKey = "runnerId"
ExecutionIdKey = "executionId"
)
type RunnerController struct {
@ -26,10 +27,11 @@ type RunnerController struct {
// ConfigureRoutes configures a given router with the runner routes of our API.
func (r *RunnerController) ConfigureRoutes(router *mux.Router) {
runnersRouter := router.PathPrefix(RouteRunners).Subrouter()
runnersRouter := router.PathPrefix(RunnersPath).Subrouter()
runnersRouter.HandleFunc("", r.provide).Methods(http.MethodPost)
r.runnerRouter = runnersRouter.PathPrefix(fmt.Sprintf("/{%s}", RunnerIdKey)).Subrouter()
r.runnerRouter.Use(r.findRunnerMiddleware)
r.runnerRouter.HandleFunc(UpdateFileSystemPath, r.updateFileSystem).Methods(http.MethodPatch).Name(UpdateFileSystemPath)
r.runnerRouter.HandleFunc(ExecutePath, r.execute).Methods(http.MethodPost).Name(ExecutePath)
r.runnerRouter.HandleFunc(WebsocketPath, r.connectToRunner).Methods(http.MethodGet).Name(WebsocketPath)
r.runnerRouter.HandleFunc("", r.delete).Methods(http.MethodDelete).Name(DeleteRoute)
@ -37,7 +39,7 @@ func (r *RunnerController) ConfigureRoutes(router *mux.Router) {
// provide handles the provide runners API route.
// It tries to respond with the id of a unused runner.
// This runner is then reserved for future use
// This runner is then reserved for future use.
func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Request) {
runnerRequest := new(dto.RunnerRequest)
if err := parseJSONRequestBody(writer, request, runnerRequest); err != nil {
@ -59,6 +61,24 @@ func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Req
sendJson(writer, &dto.RunnerResponse{Id: nextRunner.Id()}, http.StatusOK)
}
// updateFileSystem handles the files API route.
// It takes an dto.UpdateFileSystemRequest and sends it to the runner for processing.
func (r *RunnerController) updateFileSystem(writer http.ResponseWriter, request *http.Request) {
fileCopyRequest := new(dto.UpdateFileSystemRequest)
if err := parseJSONRequestBody(writer, request, fileCopyRequest); err != nil {
return
}
targetRunner, _ := runner.FromContext(request.Context())
if err := targetRunner.UpdateFileSystem(fileCopyRequest); err != nil {
log.WithError(err).Error("Could not perform the requested updateFileSystem.")
writeInternalServerError(writer, err, dto.ErrorUnknown)
return
}
writer.WriteHeader(http.StatusNoContent)
}
// execute handles the execute API route.
// It takes an ExecutionRequest and stores it for a runner.
// It returns a url to connect to for a websocket connection to this execution in the corresponding runner.

68
api/runners_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/suite"
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
"net/http"
"net/http/httptest"
"net/url"
@ -159,6 +160,73 @@ func (suite *RunnerRouteTestSuite) TestExecuteRoute() {
})
}
func TestUpdateFileSystemRouteTestSuite(t *testing.T) {
suite.Run(t, new(UpdateFileSystemRouteTestSuite))
}
type UpdateFileSystemRouteTestSuite struct {
RunnerRouteTestSuite
path string
recorder *httptest.ResponseRecorder
runnerMock *runner.RunnerMock
}
func (s *UpdateFileSystemRouteTestSuite) SetupTest() {
s.RunnerRouteTestSuite.SetupTest()
routeUrl, err := s.router.Get(UpdateFileSystemPath).URL(RunnerIdKey, tests.DefaultMockId)
if err != nil {
s.T().Fatal(err)
}
s.path = routeUrl.String()
s.runnerMock = &runner.RunnerMock{}
s.runnerManager.On("Get", tests.DefaultMockId).Return(s.runnerMock, nil)
s.recorder = httptest.NewRecorder()
}
func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsNoContentOnValidRequest() {
s.runnerMock.On("UpdateFileSystem", mock.AnythingOfType("*dto.UpdateFileSystemRequest")).Return(nil)
copyRequest := dto.UpdateFileSystemRequest{}
body, _ := json.Marshal(copyRequest)
request, _ := http.NewRequest(http.MethodPatch, s.path, bytes.NewReader(body))
s.router.ServeHTTP(s.recorder, request)
s.Equal(http.StatusNoContent, s.recorder.Code)
s.runnerMock.AssertCalled(s.T(), "UpdateFileSystem", mock.AnythingOfType("*dto.UpdateFileSystemRequest"))
}
func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsBadRequestOnInvalidRequestBody() {
request, _ := http.NewRequest(http.MethodPatch, s.path, strings.NewReader(""))
s.router.ServeHTTP(s.recorder, request)
s.Equal(http.StatusBadRequest, s.recorder.Code)
}
func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemToNonExistingRunnerReturnsNotFound() {
invalidID := "some-invalid-runner-id"
s.runnerManager.On("Get", invalidID).Return(nil, runner.ErrRunnerNotFound)
path, _ := s.router.Get(UpdateFileSystemPath).URL(RunnerIdKey, invalidID)
copyRequest := dto.UpdateFileSystemRequest{}
body, _ := json.Marshal(copyRequest)
request, _ := http.NewRequest(http.MethodPatch, path.String(), bytes.NewReader(body))
s.router.ServeHTTP(s.recorder, request)
s.Equal(http.StatusNotFound, s.recorder.Code)
}
func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsInternalServerErrorWhenCopyFailed() {
s.runnerMock.
On("UpdateFileSystem", mock.AnythingOfType("*dto.UpdateFileSystemRequest")).
Return(runner.ErrorFileCopyFailed)
copyRequest := dto.UpdateFileSystemRequest{}
body, _ := json.Marshal(copyRequest)
request, _ := http.NewRequest(http.MethodPatch, s.path, bytes.NewReader(body))
s.router.ServeHTTP(s.recorder, request)
s.Equal(http.StatusInternalServerError, s.recorder.Code)
}
func TestDeleteRunnerRouteTestSuite(t *testing.T) {
suite.Run(t, new(DeleteRunnerRouteTestSuite))
}

3
api/websocket.go
View File

@ -168,7 +168,6 @@ func newWebSocketProxy(connection webSocketConnection) (*webSocketProxy, error)
// and handles WebSocket exit messages.
func (wp *webSocketProxy) waitForExit(exit <-chan runner.ExitInfo, cancelExecution context.CancelFunc) {
defer wp.close()
cancelInputLoop := wp.Stdin.readInputLoop()
var exitInfo runner.ExitInfo
select {
@ -258,7 +257,7 @@ func (r *RunnerController) connectToRunner(writer http.ResponseWriter, request *
}
log.WithField("runnerId", targetRunner.Id()).WithField("executionId", executionId).Info("Running execution")
exit, cancel := targetRunner.Execute(executionRequest, proxy.Stdin, proxy.Stdout, proxy.Stderr)
exit, cancel := targetRunner.ExecuteInteractively(executionRequest, proxy.Stdin, proxy.Stdout, proxy.Stderr)
proxy.waitForExit(exit, cancel)
}

32
api/websocket_test.go
View File

@ -16,7 +16,7 @@ import (
"gitlab.hpi.de/codeocean/codemoon/poseidon/api/dto"
"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/e2e/helpers"
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests/helpers"
"io"
"net/http"
"net/http/httptest"
@ -40,7 +40,7 @@ type WebSocketTestSuite struct {
func (suite *WebSocketTestSuite) SetupTest() {
runnerId := "runner-id"
suite.runner, suite.apiMock = helpers.NewNomadAllocationWithMockedApiClient(runnerId)
suite.runner, suite.apiMock = newNomadAllocationWithMockedApiClient(runnerId)
// default execution
suite.executionId = "execution-id"
@ -97,7 +97,7 @@ func (suite *WebSocketTestSuite) TestWebsocketConnection() {
suite.Run("Executes the request in the runner", func() {
<-time.After(100 * time.Millisecond)
suite.apiMock.AssertCalled(suite.T(), "ExecuteCommand",
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything)
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything)
})
suite.Run("Can send input", func() {
@ -137,7 +137,7 @@ func (suite *WebSocketTestSuite) TestCancelWebSocketConnection() {
select {
case <-canceled:
suite.Fail("Execute canceled unexpected")
suite.Fail("ExecuteInteractively canceled unexpected")
default:
}
@ -147,7 +147,7 @@ func (suite *WebSocketTestSuite) TestCancelWebSocketConnection() {
select {
case <-canceled:
case <-time.After(time.Second):
suite.Fail("Execute not canceled")
suite.Fail("ExecuteInteractively not canceled")
}
}
@ -169,7 +169,7 @@ func (suite *WebSocketTestSuite) TestWebSocketConnectionTimeout() {
select {
case <-canceled:
suite.Fail("Execute canceled unexpected")
suite.Fail("ExecuteInteractively canceled unexpected")
case <-time.After(time.Duration(limitExecution.TimeLimit-1) * time.Second):
<-time.After(time.Second)
}
@ -177,7 +177,7 @@ func (suite *WebSocketTestSuite) TestWebSocketConnectionTimeout() {
select {
case <-canceled:
case <-time.After(time.Second):
suite.Fail("Execute not canceled")
suite.Fail("ExecuteInteractively not canceled")
}
message, err = helpers.ReceiveNextWebSocketMessage(connection)
@ -245,7 +245,7 @@ func (suite *WebSocketTestSuite) TestWebsocketNonZeroExit() {
func TestWebsocketTLS(t *testing.T) {
runnerId := "runner-id"
r, apiMock := helpers.NewNomadAllocationWithMockedApiClient(runnerId)
r, apiMock := newNomadAllocationWithMockedApiClient(runnerId)
executionId := runner.ExecutionId("execution-id")
r.Add(executionId, &executionRequestLs)
@ -307,6 +307,12 @@ func TestRawToCodeOceanWriter(t *testing.T) {
// --- Test suite specific test helpers ---
func newNomadAllocationWithMockedApiClient(runnerId string) (r runner.Runner, mock *nomad.ExecutorApiMock) {
mock = &nomad.ExecutorApiMock{}
r = runner.NewNomadAllocation(runnerId, mock)
return
}
func webSocketUrl(scheme string, server *httptest.Server, router *mux.Router, runnerId string, executionId runner.ExecutionId) (*url.URL, error) {
websocketUrl, err := url.Parse(server.URL)
if err != nil {
@ -331,7 +337,7 @@ var executionRequestLs = dto.ExecutionRequest{Command: "ls"}
// mockApiExecuteLs mocks the ExecuteCommand of an ExecutorApi to act as if 'ls existing-file non-existing-file' was executed.
func mockApiExecuteLs(api *nomad.ExecutorApiMock) {
helpers.MockApiExecute(api, &executionRequestLs,
func(_ string, _ context.Context, _ []string, _ io.Reader, stdout, stderr io.Writer) (int, error) {
func(_ string, _ context.Context, _ []string, _ bool, _ io.Reader, stdout, stderr io.Writer) (int, error) {
_, _ = stdout.Write([]byte("existing-file\n"))
_, _ = stderr.Write([]byte("ls: cannot access 'non-existing-file': No such file or directory\n"))
return 0, nil
@ -343,7 +349,7 @@ var executionRequestHead = dto.ExecutionRequest{Command: "head -n 1"}
// mockApiExecuteHead mocks the ExecuteCommand of an ExecutorApi to act as if 'head -n 1' was executed.
func mockApiExecuteHead(api *nomad.ExecutorApiMock) {
helpers.MockApiExecute(api, &executionRequestHead,
func(_ string, _ context.Context, _ []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
func(_ string, _ context.Context, _ []string, _ bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
scanner := bufio.NewScanner(stdin)
for !scanner.Scan() {
scanner = bufio.NewScanner(stdin)
@ -359,7 +365,7 @@ var executionRequestSleep = dto.ExecutionRequest{Command: "sleep infinity"}
func mockApiExecuteSleep(api *nomad.ExecutorApiMock) <-chan bool {
canceled := make(chan bool, 1)
helpers.MockApiExecute(api, &executionRequestSleep,
func(_ string, ctx context.Context, _ []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
func(_ string, ctx context.Context, _ []string, _ bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
<-ctx.Done()
close(canceled)
return 0, ctx.Err()
@ -372,7 +378,7 @@ var executionRequestError = dto.ExecutionRequest{Command: "error"}
// mockApiExecuteError mocks the ExecuteCommand method of an ExecutorApi to return an error.
func mockApiExecuteError(api *nomad.ExecutorApiMock) {
helpers.MockApiExecute(api, &executionRequestError,
func(_ string, _ context.Context, _ []string, _ io.Reader, _, _ io.Writer) (int, error) {
func(_ string, _ context.Context, _ []string, _ bool, _ io.Reader, _, _ io.Writer) (int, error) {
return 0, errors.New("intended error")
})
}
@ -382,7 +388,7 @@ var executionRequestExitNonZero = dto.ExecutionRequest{Command: "exit 42"}
// mockApiExecuteExitNonZero mocks the ExecuteCommand method of an ExecutorApi to exit with exit status 42.
func mockApiExecuteExitNonZero(api *nomad.ExecutorApiMock) {
helpers.MockApiExecute(api, &executionRequestExitNonZero,
func(_ string, _ context.Context, _ []string, _ io.Reader, _, _ io.Writer) (int, error) {
func(_ string, _ context.Context, _ []string, _ bool, _ io.Reader, _, _ io.Writer) (int, error) {
return 42, nil
})
}