#136 Copy files back from Nomad runner.

internal/api/runners.go

@@ -20,9 +20,11 @@ const (
 	ExecutePath          = "/execute"
 	WebsocketPath        = "/websocket"
 	UpdateFileSystemPath = "/files"
+	FileContentRawPath   = UpdateFileSystemPath + "/raw"
 	DeleteRoute          = "deleteRunner"
 	RunnerIDKey          = "runnerId"
 	ExecutionIDKey       = "executionID"
+	PathKey              = "path"
 	ProvideRoute         = "provideRunner"
 )
 
@@ -39,6 +41,7 @@ func (r *RunnerController) ConfigureRoutes(router *mux.Router) {
 	r.runnerRouter.Use(r.findRunnerMiddleware)
 	r.runnerRouter.HandleFunc(UpdateFileSystemPath, r.updateFileSystem).Methods(http.MethodPatch).
 		Name(UpdateFileSystemPath)
+	r.runnerRouter.HandleFunc(FileContentRawPath, r.fileContent).Methods(http.MethodGet).Name(FileContentRawPath)
 	r.runnerRouter.HandleFunc(ExecutePath, r.execute).Methods(http.MethodPost).Name(ExecutePath)
 	r.runnerRouter.HandleFunc(WebsocketPath, r.connectToRunner).Methods(http.MethodGet).Name(WebsocketPath)
 	r.runnerRouter.HandleFunc("", r.delete).Methods(http.MethodDelete).Name(DeleteRoute)
@@ -92,6 +95,25 @@ func (r *RunnerController) updateFileSystem(writer http.ResponseWriter, request
 	writer.WriteHeader(http.StatusNoContent)
 }
 
+func (r *RunnerController) fileContent(writer http.ResponseWriter, request *http.Request) {
+	monitoring.AddRequestSize(request)
+	targetRunner, _ := runner.FromContext(request.Context())
+	path := request.URL.Query().Get(PathKey)
+
+	err := targetRunner.GetFileContent(path, writer, request.Context())
+	if errors.Is(err, runner.ErrFileNotFound) {
+		writeNotFound(writer, err)
+		return
+	} else if err != nil {
+		log.WithError(err).Error("Could not retrieve the requested file.")
+		writeInternalServerError(writer, err, dto.ErrorUnknown)
+		return
+	}
+
+	writer.Header().Set("Content-Type", "application/octet-stream")
+	writer.WriteHeader(http.StatusOK)
+}
+
 // execute handles the execute API route.
 // It takes an ExecutionRequest and stores it for a runner.
 // It returns a url to connect to for a websocket connection to this execution in the corresponding runner.

internal/api/runners_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/gorilla/mux"
+	"github.com/openHPI/poseidon/internal/nomad"
 	"github.com/openHPI/poseidon/internal/runner"
 	"github.com/openHPI/poseidon/pkg/dto"
 	"github.com/openHPI/poseidon/tests"
@@ -310,6 +311,39 @@ func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsInternalServ
 	s.Equal(http.StatusInternalServerError, s.recorder.Code)
 }
 
+func (s *UpdateFileSystemRouteTestSuite) TestFileContent() {
+	routeURL, err := s.router.Get(FileContentRawPath).URL(RunnerIDKey, tests.DefaultMockID)
+	s.Require().NoError(err)
+	mockCall := s.runnerMock.
+		On("GetFileContent", mock.AnythingOfType("string"), mock.Anything, mock.Anything)
+
+	s.Run("Not Found", func() {
+		mockCall.Return(runner.ErrFileNotFound)
+		request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader(""))
+		s.Require().NoError(err)
+		s.router.ServeHTTP(s.recorder, request)
+		s.Equal(http.StatusNotFound, s.recorder.Code)
+	})
+
+	s.recorder = httptest.NewRecorder()
+	s.Run("Unknown Error", func() {
+		mockCall.Return(nomad.ErrorExecutorCommunicationFailed)
+		request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader(""))
+		s.Require().NoError(err)
+		s.router.ServeHTTP(s.recorder, request)
+		s.Equal(http.StatusInternalServerError, s.recorder.Code)
+	})
+
+	s.recorder = httptest.NewRecorder()
+	s.Run("No Error", func() {
+		mockCall.Return(nil)
+		request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader(""))
+		s.Require().NoError(err)
+		s.router.ServeHTTP(s.recorder, request)
+		s.Equal(http.StatusOK, s.recorder.Code)
+	})
+}
+
 func TestDeleteRunnerRouteTestSuite(t *testing.T) {
 	suite.Run(t, new(DeleteRunnerRouteTestSuite))
 }

internal/runner/aws_runner.go

@@ -115,6 +115,13 @@ func (w *AWSFunctionWorkload) UpdateFileSystem(request *dto.UpdateFileSystemRequ
 	return nil
 }
 
+// GetFileContent is currently not supported with this aws serverless function.
+// This is because the function execution ends with the termination of the workload code.
+// So an on-demand file streaming after the termination is not possible. Also, we do not want to copy all files.
+func (w *AWSFunctionWorkload) GetFileContent(_ string, _ io.Writer, _ context.Context) error {
+	return dto.ErrNotSupported
+}
+
 func (w *AWSFunctionWorkload) Destroy() error {
 	for _, cancel := range w.runningExecutions {
 		cancel()

internal/runner/nomad_runner.go

@@ -12,6 +12,7 @@ import (
 	"github.com/openHPI/poseidon/internal/nomad"
 	"github.com/openHPI/poseidon/pkg/dto"
 	"github.com/openHPI/poseidon/pkg/monitoring"
+	"github.com/openHPI/poseidon/pkg/nullio"
 	"github.com/openHPI/poseidon/pkg/storage"
 	"io"
 	"strings"
@@ -31,6 +32,7 @@ const (
 var (
 	ErrorUnknownExecution = errors.New("unknown execution")
 	ErrorFileCopyFailed   = errors.New("file copy failed")
+	ErrFileNotFound       = errors.New("file not found")
 )
 
 // NomadJob is an abstraction to communicate with Nomad environments.
@@ -149,6 +151,23 @@ func (r *NomadJob) UpdateFileSystem(copyRequest *dto.UpdateFileSystemRequest) er
 	return nil
 }
 
+func (r *NomadJob) GetFileContent(path string, content io.Writer, ctx context.Context) error {
+	r.ResetTimeout()
+
+	retrieveCommand := (&dto.ExecutionRequest{Command: fmt.Sprintf("cat %s", path)}).FullCommand()
+	// Improve: Instead of using io.Discard use a **fixed-sized** buffer. With that we could improve the error message.
+	exitCode, err := r.api.ExecuteCommand(r.id, ctx, retrieveCommand, false, &nullio.Reader{}, content, io.Discard)
+
+	if err != nil {
+		return fmt.Errorf("%w: nomad error during retrieve file content copy: %v",
+			nomad.ErrorExecutorCommunicationFailed, err)
+	}
+	if exitCode != 0 {
+		return ErrFileNotFound
+	}
+	return nil
+}
+
 func (r *NomadJob) Destroy() error {
 	if err := r.onDestroy(r); err != nil {
 		return fmt.Errorf("error while destroying runner: %w", err)

internal/runner/nomad_runner_test.go

@@ -399,3 +399,10 @@ func NewRunner(id string, manager Accessor) Runner {
 	}
 	return NewNomadJob(id, nil, nil, handler)
 }
+
+func (s *UpdateFileSystemTestSuite) TestGetFileContentReturnsErrorIfExitCodeIsNotZero() {
+	s.mockedExecuteCommandCall.RunFn = nil
+	s.mockedExecuteCommandCall.Return(1, nil)
+	err := s.runner.GetFileContent("", &bytes.Buffer{}, context.Background())
+	s.ErrorIs(err, ErrFileNotFound)
+}

internal/runner/runner.go

@@ -48,6 +48,9 @@ type Runner interface {
 	// and then copying each given dto.File to the runner.
 	UpdateFileSystem(request *dto.UpdateFileSystemRequest) error
 
+	// GetFileContent streams the file content at the requested path into the Writer provided at content.
+	GetFileContent(path string, content io.Writer, ctx context.Context) error
+
 	// Destroy destroys the Runner in Nomad.
 	Destroy() error
 }

internal/runner/runner_mock.go

@@ -1,4 +1,4 @@
-// Code generated by mockery v2.10.4. DO NOT EDIT.
+// Code generated by mockery v2.14.0. DO NOT EDIT.
 
 package runner
 
@@ -92,6 +92,20 @@ func (_m *RunnerMock) ExecutionExists(id string) bool {
 	return r0
 }
 
+// GetFileContent provides a mock function with given fields: path, content, ctx
+func (_m *RunnerMock) GetFileContent(path string, content io.Writer, ctx context.Context) error {
+	ret := _m.Called(path, content, ctx)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, io.Writer, context.Context) error); ok {
+		r0 = rf(path, content, ctx)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // ID provides a mock function with given fields:
 func (_m *RunnerMock) ID() string {
 	ret := _m.Called()
@@ -169,3 +183,18 @@ func (_m *RunnerMock) UpdateFileSystem(request *dto.UpdateFileSystemRequest) err
 
 	return r0
 }
+
+type mockConstructorTestingTNewRunnerMock interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewRunnerMock creates a new instance of RunnerMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewRunnerMock(t mockConstructorTestingTNewRunnerMock) *RunnerMock {
+	mock := &RunnerMock{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}