Add listing of runners file system.

internal/api/runners.go

@@ -17,15 +17,17 @@ import (
 )
 
 const (
-	ExecutePath          = "/execute"
-	WebsocketPath        = "/websocket"
-	UpdateFileSystemPath = "/files"
-	FileContentRawPath   = UpdateFileSystemPath + "/raw"
-	DeleteRoute          = "deleteRunner"
-	RunnerIDKey          = "runnerId"
-	ExecutionIDKey       = "executionID"
-	PathKey              = "path"
-	ProvideRoute         = "provideRunner"
+	ExecutePath             = "/execute"
+	WebsocketPath           = "/websocket"
+	UpdateFileSystemPath    = "/files"
+	ListFileSystemRouteName = UpdateFileSystemPath + "_list"
+	FileContentRawPath      = UpdateFileSystemPath + "/raw"
+	ProvideRoute            = "provideRunner"
+	DeleteRoute             = "deleteRunner"
+	RunnerIDKey             = "runnerId"
+	ExecutionIDKey          = "executionID"
+	PathKey                 = "path"
+	RecursiveKey            = "recursive"
 )
 
 type RunnerController struct {
@@ -39,6 +41,8 @@ func (r *RunnerController) ConfigureRoutes(router *mux.Router) {
 	runnersRouter.HandleFunc("", r.provide).Methods(http.MethodPost).Name(ProvideRoute)
 	r.runnerRouter = runnersRouter.PathPrefix(fmt.Sprintf("/{%s}", RunnerIDKey)).Subrouter()
 	r.runnerRouter.Use(r.findRunnerMiddleware)
+	r.runnerRouter.HandleFunc(UpdateFileSystemPath, r.listFileSystem).Methods(http.MethodGet).
+		Name(ListFileSystemRouteName)
 	r.runnerRouter.HandleFunc(UpdateFileSystemPath, r.updateFileSystem).Methods(http.MethodPatch).
 		Name(UpdateFileSystemPath)
 	r.runnerRouter.HandleFunc(FileContentRawPath, r.fileContent).Methods(http.MethodGet).Name(FileContentRawPath)
@@ -75,6 +79,29 @@ func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Req
 	sendJSON(writer, &dto.RunnerResponse{ID: nextRunner.ID(), MappedPorts: nextRunner.MappedPorts()}, http.StatusOK)
 }
 
+// listFileSystem handles the files API route with the method GET.
+// It returns a listing of the file system of the provided runner.
+func (r *RunnerController) listFileSystem(writer http.ResponseWriter, request *http.Request) {
+	targetRunner, _ := runner.FromContext(request.Context())
+	monitoring.AddRunnerMonitoringData(request, targetRunner.ID(), targetRunner.Environment())
+
+	recursiveRaw := request.URL.Query().Get(RecursiveKey)
+	recursive, err := strconv.ParseBool(recursiveRaw)
+	recursive = err != nil || recursive
+
+	path := request.URL.Query().Get(PathKey)
+	if path == "" {
+		path = "./"
+	}
+
+	writer.Header().Set("Content-Type", "application/json")
+	if err := targetRunner.ListFileSystem(path, recursive, writer, request.Context()); err != nil {
+		log.WithError(err).Error("Could not perform the requested listFileSystem.")
+		writeInternalServerError(writer, err, dto.ErrorUnknown)
+		return
+	}
+}
+
 // updateFileSystem handles the files API route.
 // It takes an dto.UpdateFileSystemRequest and sends it to the runner for processing.
 func (r *RunnerController) updateFileSystem(writer http.ResponseWriter, request *http.Request) {
@@ -96,7 +123,6 @@ func (r *RunnerController) updateFileSystem(writer http.ResponseWriter, request
 }
 
 func (r *RunnerController) fileContent(writer http.ResponseWriter, request *http.Request) {
-	monitoring.AddRequestSize(request)
 	targetRunner, _ := runner.FromContext(request.Context())
 	path := request.URL.Query().Get(PathKey)
 

internal/api/runners_test.go

@@ -14,6 +14,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -311,6 +312,66 @@ func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsInternalServ
 	s.Equal(http.StatusInternalServerError, s.recorder.Code)
 }
 
+func (s *UpdateFileSystemRouteTestSuite) TestListFileSystem() {
+	routeURL, err := s.router.Get(UpdateFileSystemPath).URL(RunnerIDKey, tests.DefaultMockID)
+	s.Require().NoError(err)
+	mockCall := s.runnerMock.On("ListFileSystem",
+		mock.AnythingOfType("string"), mock.AnythingOfType("bool"), mock.Anything, mock.Anything)
+
+	s.Run("default parameters", func() {
+		mockCall.Run(func(args mock.Arguments) {
+			path, ok := args.Get(0).(string)
+			s.True(ok)
+			s.Equal("./", path)
+			recursive, ok := args.Get(1).(bool)
+			s.True(ok)
+			s.True(recursive)
+			mockCall.ReturnArguments = mock.Arguments{nil}
+		})
+		request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader(""))
+		s.Require().NoError(err)
+		s.router.ServeHTTP(s.recorder, request)
+		s.Equal(http.StatusOK, s.recorder.Code)
+	})
+
+	s.recorder = httptest.NewRecorder()
+	s.Run("passed parameters", func() {
+		expectedPath := "/flag"
+
+		mockCall.Run(func(args mock.Arguments) {
+			path, ok := args.Get(0).(string)
+			s.True(ok)
+			s.Equal(expectedPath, path)
+			recursive, ok := args.Get(1).(bool)
+			s.True(ok)
+			s.False(recursive)
+			mockCall.ReturnArguments = mock.Arguments{nil}
+		})
+
+		query := routeURL.Query()
+		query.Set(PathKey, expectedPath)
+		query.Set(RecursiveKey, strconv.FormatBool(false))
+		routeURL.RawQuery = query.Encode()
+
+		request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader(""))
+		s.Require().NoError(err)
+		s.router.ServeHTTP(s.recorder, request)
+		s.Equal(http.StatusOK, s.recorder.Code)
+	})
+
+	s.recorder = httptest.NewRecorder()
+	s.Run("Internal Server Error on failure", func() {
+		mockCall.Run(func(args mock.Arguments) {
+			mockCall.ReturnArguments = mock.Arguments{runner.ErrRunnerNotFound}
+		})
+
+		request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader(""))
+		s.Require().NoError(err)
+		s.router.ServeHTTP(s.recorder, request)
+		s.Equal(http.StatusInternalServerError, s.recorder.Code)
+	})
+}
+
 func (s *UpdateFileSystemRouteTestSuite) TestFileContent() {
 	routeURL, err := s.router.Get(FileContentRawPath).URL(RunnerIDKey, tests.DefaultMockID)
 	s.Require().NoError(err)

internal/runner/aws_runner.go

@@ -101,6 +101,13 @@ func (w *AWSFunctionWorkload) ExecuteInteractively(id string, _ io.ReadWriter, s
 	return exit, cancel, nil
 }
 
+// ListFileSystem is currently not supported with this aws serverless function.
+// This is because the function execution ends with the termination of the workload code.
+// So an on-demand file system listing after the termination is not possible. Also, we do not want to copy all files.
+func (w *AWSFunctionWorkload) ListFileSystem(_ string, _ bool, _ io.Writer, _ context.Context) error {
+	return dto.ErrNotSupported
+}
+
 // UpdateFileSystem copies Files into the executor.
 // Current limitation: No files can be deleted apart from the previously added files.
 // Future Work: Deduplication of the file systems, as the largest workload is likely to be used by additional

internal/runner/nomad_runner.go

@@ -118,6 +118,27 @@ func (r *NomadJob) ExecuteInteractively(
 	return exit, cancel, nil
 }
 
+func (r *NomadJob) ListFileSystem(path string, recursive bool, content io.Writer, ctx context.Context) error {
+	r.ResetTimeout()
+	command := "ls -l --time-style=+%s -1 --literal"
+	if recursive {
+		command += " --recursive"
+	}
+
+	ls2json := &nullio.Ls2JsonWriter{Target: content}
+	defer ls2json.Close()
+	retrieveCommand := (&dto.ExecutionRequest{Command: fmt.Sprintf("%s %s", command, path)}).FullCommand()
+	exitCode, err := r.api.ExecuteCommand(r.id, ctx, retrieveCommand, false, &nullio.Reader{}, ls2json, io.Discard)
+	if err != nil {
+		return fmt.Errorf("%w: nomad error during retrieve file headers: %v",
+			nomad.ErrorExecutorCommunicationFailed, err)
+	}
+	if exitCode != 0 {
+		return ErrFileNotFound
+	}
+	return nil
+}
+
 func (r *NomadJob) UpdateFileSystem(copyRequest *dto.UpdateFileSystemRequest) error {
 	r.ResetTimeout()
 

internal/runner/runner.go

@@ -44,11 +44,16 @@ type Runner interface {
 		stderr io.Writer,
 	) (exit <-chan ExitInfo, cancel context.CancelFunc, err error)
 
+	// ListFileSystem streams the listing of the file system of the requested directory into the Writer provided.
+	// The result is streamed via the io.Writer in order to not overload the memory with user input.
+	ListFileSystem(path string, recursive bool, result io.Writer, ctx context.Context) error
+
 	// UpdateFileSystem processes a dto.UpdateFileSystemRequest by first deleting each given dto.FilePath recursively
 	// and then copying each given dto.File to the runner.
 	UpdateFileSystem(request *dto.UpdateFileSystemRequest) error
 
 	// GetFileContent streams the file content at the requested path into the Writer provided at content.
+	// The result is streamed via the io.Writer in order to not overload the memory with user input.
 	GetFileContent(path string, content io.Writer, ctx context.Context) error
 
 	// Destroy destroys the Runner in Nomad.

internal/runner/runner_mock.go

@@ -120,6 +120,20 @@ func (_m *RunnerMock) ID() string {
 	return r0
 }
 
+// ListFileSystem provides a mock function with given fields: path, recursive, result, ctx
+func (_m *RunnerMock) ListFileSystem(path string, recursive bool, result io.Writer, ctx context.Context) error {
+	ret := _m.Called(path, recursive, result, ctx)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, bool, io.Writer, context.Context) error); ok {
+		r0 = rf(path, recursive, result, ctx)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // MappedPorts provides a mock function with given fields:
 func (_m *RunnerMock) MappedPorts() []*dto.MappedPort {
 	ret := _m.Called()