Explicitly switch user for code execution.

Co-authored-by: Maximilian Pass <maximilian.pass@student.hpi.uni-potsdam.de>
This commit is contained in:
Sebastian Serth
2022-09-18 01:52:15 +02:00
committed by Sebastian Serth
parent 69237fb415
commit 1a5a49d7c8
13 changed files with 144 additions and 67 deletions

View File

@ -100,7 +100,8 @@ func (s *WebSocketTestSuite) TestWebsocketConnection() {
s.Run("Executes the request in the runner", func() {
<-time.After(tests.ShortTimeout)
s.apiMock.AssertCalled(s.T(), "ExecuteCommand",
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything)
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.AnythingOfType("bool"),
mock.Anything, mock.Anything, mock.Anything)
})
s.Run("Can send input", func() {
@ -405,6 +406,7 @@ var executionRequestSleep = dto.ExecutionRequest{Command: "sleep infinity"}
// until the execution receives a SIGQUIT.
func mockAPIExecuteSleep(api *nomad.ExecutorAPIMock) <-chan bool {
canceled := make(chan bool, 1)
mockAPIExecute(api, &executionRequestSleep,
func(_ string, ctx context.Context, _ []string, _ bool,
stdin io.Reader, stdout io.Writer, stderr io.Writer,
@ -446,13 +448,13 @@ func mockAPIExecuteExitNonZero(api *nomad.ExecutorAPIMock) {
// corresponding to the given ExecutionRequest is called.
func mockAPIExecute(api *nomad.ExecutorAPIMock, request *dto.ExecutionRequest,
run func(runnerId string, ctx context.Context, command []string, tty bool,
stdin io.Reader, stdout, stderr io.Writer) (int, error),
) {
stdin io.Reader, stdout, stderr io.Writer) (int, error)) {
call := api.On("ExecuteCommand",
mock.AnythingOfType("string"),
mock.Anything,
request.FullCommand(),
mock.AnythingOfType("bool"),
mock.AnythingOfType("bool"),
mock.Anything,
mock.Anything,
mock.Anything)
@ -461,9 +463,9 @@ func mockAPIExecute(api *nomad.ExecutorAPIMock, request *dto.ExecutionRequest,
args.Get(1).(context.Context),
args.Get(2).([]string),
args.Get(3).(bool),
args.Get(4).(io.Reader),
args.Get(5).(io.Writer),
args.Get(6).(io.Writer))
args.Get(5).(io.Reader),
args.Get(6).(io.Writer),
args.Get(7).(io.Writer))
call.ReturnArguments = mock.Arguments{exit, err}
})
}

View File

@ -78,20 +78,20 @@ func (_m *ExecutorAPIMock) Execute(jobID string, ctx context.Context, command []
return r0, r1
}
// ExecuteCommand provides a mock function with given fields: allocationID, ctx, command, tty, stdin, stdout, stderr
func (_m *ExecutorAPIMock) ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
ret := _m.Called(allocationID, ctx, command, tty, stdin, stdout, stderr)
// ExecuteCommand provides a mock function with given fields: allocationID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr
func (_m *ExecutorAPIMock) ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool, privilegedExecution bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
ret := _m.Called(allocationID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr)
var r0 int
if rf, ok := ret.Get(0).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) int); ok {
r0 = rf(allocationID, ctx, command, tty, stdin, stdout, stderr)
if rf, ok := ret.Get(0).(func(string, context.Context, []string, bool, bool, io.Reader, io.Writer, io.Writer) int); ok {
r0 = rf(allocationID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr)
} else {
r0 = ret.Get(0).(int)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, context.Context, []string, bool, io.Reader, io.Writer, io.Writer) error); ok {
r1 = rf(allocationID, ctx, command, tty, stdin, stdout, stderr)
if rf, ok := ret.Get(1).(func(string, context.Context, []string, bool, bool, io.Reader, io.Writer, io.Writer) error); ok {
r1 = rf(allocationID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr)
} else {
r1 = ret.Error(1)
}

View File

@ -70,7 +70,8 @@ type ExecutorAPI interface {
// ExecuteCommand executes the given command in the allocation with the given id.
// It writes the output of the command to stdout/stderr and reads input from stdin.
// If tty is true, the command will run with a tty.
ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool,
// Iff privilegedExecution is true, the command will be executed privileged.
ExecuteCommand(allocationID string, ctx context.Context, command []string, tty bool, privilegedExecution bool,
stdin io.Reader, stdout, stderr io.Writer) (int, error)
// MarkRunnerAsUsed marks the runner with the given ID as used. It also stores the timeout duration in the metadata.
@ -390,12 +391,13 @@ func (a *APIClient) LoadEnvironmentJobs() ([]*nomadApi.Job, error) {
// In order for the stderr splitting to work, the command must have the structure
// []string{..., "sh", "-c", "my-command"}.
func (a *APIClient) ExecuteCommand(allocationID string,
ctx context.Context, command []string, tty bool,
ctx context.Context, command []string, tty bool, privilegedExecution bool,
stdin io.Reader, stdout, stderr io.Writer) (int, error) {
if tty && config.Config.Server.InteractiveStderr {
return a.executeCommandInteractivelyWithStderr(allocationID, ctx, command, stdin, stdout, stderr)
return a.executeCommandInteractivelyWithStderr(allocationID, ctx, command, privilegedExecution, stdin, stdout, stderr)
}
exitCode, err := a.apiQuerier.Execute(allocationID, ctx, command, tty, stdin, stdout, stderr)
exitCode, err := a.apiQuerier.
Execute(allocationID, ctx, setUserCommand(command, privilegedExecution), tty, stdin, stdout, stderr)
if err != nil {
return 1, fmt.Errorf("error executing command in API: %w", err)
}
@ -408,7 +410,7 @@ func (a *APIClient) ExecuteCommand(allocationID string,
// to be served both over stdout. This function circumvents this by creating a fifo for the stderr
// of the command and starting a second execution that reads the stderr from that fifo.
func (a *APIClient) executeCommandInteractivelyWithStderr(allocationID string, ctx context.Context,
command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) {
command []string, privilegedExecution bool, stdin io.Reader, stdout, stderr io.Writer) (int, error) {
// Use current nano time to make the stderr fifo kind of unique.
currentNanoTime := time.Now().UnixNano()
// We expect the command to be like []string{..., "sh", "-c", "my-command"}.
@ -422,7 +424,8 @@ func (a *APIClient) executeCommandInteractivelyWithStderr(allocationID string, c
defer cancel()
// Catch stderr in separate execution.
exit, err := a.Execute(allocationID, ctx, stderrFifoCommand(currentNanoTime), true,
stdErrCommand := setUserCommand(stderrFifoCommand(currentNanoTime), privilegedExecution)
exit, err := a.Execute(allocationID, ctx, stdErrCommand, true,
nullio.Reader{Ctx: readingContext}, stderr, io.Discard)
if err != nil {
log.WithError(err).WithField("runner", allocationID).Warn("Stderr task finished with error")
@ -430,7 +433,8 @@ func (a *APIClient) executeCommandInteractivelyWithStderr(allocationID string, c
stderrExitChan <- exit
}()
exit, err := a.Execute(allocationID, ctx, command, true, stdin, stdout, io.Discard)
exit, err := a.
Execute(allocationID, ctx, setUserCommand(command, privilegedExecution), true, stdin, stdout, io.Discard)
// Wait until the stderr catch command finished to make sure we receive all output.
<-stderrExitChan
@ -450,8 +454,25 @@ const (
// redirected to the fifo.
// Example: "until [ -e my.fifo ]; do sleep 0.01; done; (echo \"my.fifo exists\") 2> my.fifo".
stderrWrapperCommandFormat = "until [ -e %s ]; do sleep 0.01; done; (%s) 2> %s"
// setUserBinaryPath is due to Poseidon requires the setuser script for Nomad environments.
setUserBinaryPath = "/sbin/setuser"
// setUserBinaryUser is the user that is used and required by Poseidon for Nomad environments.
setUserBinaryUser = "user"
// PrivilegedExecution is to indicate the privileged execution of the passed command.
PrivilegedExecution = true
// UnprivilegedExecution is to indicate the unprivileged execution of the passed command.
UnprivilegedExecution = false
)
func setUserCommand(command []string, privilegedExecution bool) []string {
if privilegedExecution {
return command
} else {
return append([]string{setUserBinaryPath, setUserBinaryUser}, command...)
}
}
func stderrFifoCommand(id int64) []string {
stderrFifoPath := stderrFifo(id)
return []string{"sh", "-c", fmt.Sprintf(stderrFifoCommandFormat, stderrFifoPath, stderrFifoPath, stderrFifoPath)}

View File

@ -676,28 +676,31 @@ func (s *ExecuteCommandTestSuite) TestWithSeparateStderr() {
var calledStdoutCommand, calledStderrCommand []string
// mock regular call
s.mockExecute(s.testCommandArray, commandExitCode, nil, func(args mock.Arguments) {
call := s.mockExecute(mock.AnythingOfType("[]string"), 0, nil, func(_ mock.Arguments) {})
call.Run(func(args mock.Arguments) {
var ok bool
calledStdoutCommand, ok = args.Get(2).([]string)
calledCommand, ok := args.Get(2).([]string)
s.Require().True(ok)
writer, ok := args.Get(5).(io.Writer)
s.Require().True(ok)
_, err := writer.Write([]byte(s.expectedStdout))
s.Require().Equal(5, len(calledCommand))
isStderrCommand, err := regexp.MatchString("mkfifo.*", calledCommand[4])
s.Require().NoError(err)
})
// mock stderr call
s.mockExecute(mock.AnythingOfType("[]string"), stderrExitCode, nil, func(args mock.Arguments) {
var ok bool
calledStderrCommand, ok = args.Get(2).([]string)
s.Require().True(ok)
writer, ok := args.Get(5).(io.Writer)
s.Require().True(ok)
_, err := writer.Write([]byte(s.expectedStderr))
if isStderrCommand {
calledStderrCommand = calledCommand
_, err = writer.Write([]byte(s.expectedStderr))
call.ReturnArguments = mock.Arguments{stderrExitCode, nil}
} else {
calledStdoutCommand = calledCommand
_, err = writer.Write([]byte(s.expectedStdout))
call.ReturnArguments = mock.Arguments{commandExitCode, nil}
}
s.Require().NoError(err)
})
exitCode, err := s.nomadAPIClient.
ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullio.Reader{}, &stdout, &stderr)
exitCode, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY,
UnprivilegedExecution, nullio.Reader{}, &stdout, &stderr)
s.Require().NoError(err)
s.apiMock.AssertNumberOfCalls(s.T(), "Execute", 2)
@ -725,10 +728,26 @@ func (s *ExecuteCommandTestSuite) TestWithSeparateStderr() {
func (s *ExecuteCommandTestSuite) TestWithSeparateStderrReturnsCommandError() {
config.Config.Server.InteractiveStderr = true
s.mockExecute(s.testCommandArray, 1, tests.ErrDefault, func(args mock.Arguments) {})
s.mockExecute(mock.AnythingOfType("[]string"), 1, nil, func(args mock.Arguments) {})
_, err := s.nomadAPIClient.
ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullio.Reader{}, io.Discard, io.Discard)
call := s.mockExecute(mock.AnythingOfType("[]string"), 0, nil, func(_ mock.Arguments) {})
call.Run(func(args mock.Arguments) {
var ok bool
calledCommand, ok := args.Get(2).([]string)
s.Require().True(ok)
s.Require().Equal(5, len(calledCommand))
isStderrCommand, err := regexp.MatchString("mkfifo.*", calledCommand[4])
s.Require().NoError(err)
if isStderrCommand {
call.ReturnArguments = mock.Arguments{1, nil}
} else {
call.ReturnArguments = mock.Arguments{1, tests.ErrDefault}
}
s.Require().NoError(err)
})
_, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, UnprivilegedExecution,
nullio.Reader{}, io.Discard, io.Discard)
s.Equal(tests.ErrDefault, err)
}
@ -738,7 +757,8 @@ func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderr() {
commandExitCode := 42
// mock regular call
s.mockExecute(s.testCommandArray, commandExitCode, nil, func(args mock.Arguments) {
expectedCommand := setUserCommand(s.testCommandArray, UnprivilegedExecution)
s.mockExecute(expectedCommand, commandExitCode, nil, func(args mock.Arguments) {
stdout, ok := args.Get(5).(io.Writer)
s.Require().True(ok)
_, err := stdout.Write([]byte(s.expectedStdout))
@ -749,8 +769,8 @@ func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderr() {
s.Require().NoError(err)
})
exitCode, err := s.nomadAPIClient.
ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullio.Reader{}, &stdout, &stderr)
exitCode, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY,
UnprivilegedExecution, nullio.Reader{}, &stdout, &stderr)
s.Require().NoError(err)
s.apiMock.AssertNumberOfCalls(s.T(), "Execute", 1)
@ -761,15 +781,16 @@ func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderr() {
func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderrReturnsCommandError() {
config.Config.Server.InteractiveStderr = false
s.mockExecute(s.testCommandArray, 1, tests.ErrDefault, func(args mock.Arguments) {})
_, err := s.nomadAPIClient.
ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, nullio.Reader{}, io.Discard, io.Discard)
expectedCommand := setUserCommand(s.testCommandArray, UnprivilegedExecution)
s.mockExecute(expectedCommand, 1, tests.ErrDefault, func(args mock.Arguments) {})
_, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommandArray, withTTY, UnprivilegedExecution,
nullio.Reader{}, io.Discard, io.Discard)
s.ErrorIs(err, tests.ErrDefault)
}
func (s *ExecuteCommandTestSuite) mockExecute(command interface{}, exitCode int,
err error, runFunc func(arguments mock.Arguments)) {
s.apiMock.On("Execute", s.allocationID, s.ctx, command, withTTY,
err error, runFunc func(arguments mock.Arguments)) *mock.Call {
return s.apiMock.On("Execute", s.allocationID, s.ctx, command, withTTY,
mock.Anything, mock.Anything, mock.Anything).
Run(runFunc).
Return(exitCode, err)

View File

@ -90,6 +90,7 @@ func (w *AWSFunctionWorkload) ExecuteInteractively(id string, _ io.ReadWriter, s
return nil, nil, ErrorUnknownExecution
}
hideEnvironmentVariables(request, "AWS")
request.PrivilegedExecution = true // AWS does not support multiple users at this moment.
command, ctx, cancel := prepareExecution(request)
exitInternal := make(chan ExitInfo)
exit := make(chan ExitInfo, 1)

View File

@ -110,7 +110,7 @@ func (r *NomadJob) ExecuteInteractively(
exit := make(chan ExitInfo, 1)
ctxExecute, cancelExecute := context.WithCancel(context.Background())
go r.executeCommand(ctxExecute, command, stdin, stdout, stderr, exitInternal)
go r.executeCommand(ctxExecute, command, request.PrivilegedExecution, stdin, stdout, stderr, exitInternal)
go r.handleExitOrContextDone(ctx, cancelExecute, exitInternal, exit, stdin)
return exit, cancel, nil
@ -130,6 +130,7 @@ func (r *NomadJob) UpdateFileSystem(copyRequest *dto.UpdateFileSystemRequest) er
stdOut := bytes.Buffer{}
stdErr := bytes.Buffer{}
exitCode, err := r.api.ExecuteCommand(r.id, context.Background(), updateFileCommand, false,
nomad.PrivilegedExecution, // All files should be written and owned by a privileged user #211.
&tarBuffer, &stdOut, &stdErr)
if err != nil {
@ -167,10 +168,10 @@ func prepareExecution(request *dto.ExecutionRequest) (
return command, ctx, cancel
}
func (r *NomadJob) executeCommand(ctx context.Context, command []string,
func (r *NomadJob) executeCommand(ctx context.Context, command []string, privilegedExecution bool,
stdin io.ReadWriter, stdout, stderr io.Writer, exit chan<- ExitInfo,
) {
exitCode, err := r.api.ExecuteCommand(r.id, ctx, command, true, stdin, stdout, stderr)
exitCode, err := r.api.ExecuteCommand(r.id, ctx, command, true, privilegedExecution, stdin, stdout, stderr)
if err == nil && r.TimeoutPassed() {
err = ErrorRunnerInactivityTimeout
}

View File

@ -111,8 +111,8 @@ type ExecuteInteractivelyTestSuite struct {
func (s *ExecuteInteractivelyTestSuite) SetupTest() {
s.apiMock = &nomad.ExecutorAPIMock{}
s.mockedExecuteCommandCall = s.apiMock.
On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, true, mock.Anything, mock.Anything, mock.Anything).
s.mockedExecuteCommandCall = s.apiMock.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything,
true, false, mock.Anything, mock.Anything, mock.Anything).
Return(0, nil)
s.timer = &InactivityTimerMock{}
s.timer.On("ResetTimeout").Return()
@ -142,7 +142,7 @@ func (s *ExecuteInteractivelyTestSuite) TestCallsApi() {
time.Sleep(tests.ShortTimeout)
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", tests.DefaultRunnerID, mock.Anything, request.FullCommand(),
true, mock.Anything, mock.Anything, mock.Anything)
true, false, mock.Anything, mock.Anything, mock.Anything)
}
func (s *ExecuteInteractivelyTestSuite) TestReturnsAfterTimeout() {
@ -173,7 +173,7 @@ func (s *ExecuteInteractivelyTestSuite) TestReturnsAfterTimeout() {
func (s *ExecuteInteractivelyTestSuite) TestSendsSignalAfterTimeout() {
quit := make(chan struct{})
s.mockedExecuteCommandCall.Run(func(args mock.Arguments) {
stdin, ok := args.Get(4).(io.Reader)
stdin, ok := args.Get(5).(io.Reader)
s.Require().True(ok)
buffer := make([]byte, 1) //nolint:makezero,lll // If the length is zero, the Read call never reads anything. gofmt want this alignment.
for n := 0; !(n == 1 && buffer[0] == SIGQUIT); {
@ -257,12 +257,12 @@ func (s *UpdateFileSystemTestSuite) SetupTest() {
api: s.apiMock,
}
s.mockedExecuteCommandCall = s.apiMock.On("ExecuteCommand", tests.DefaultRunnerID, mock.Anything,
mock.Anything, false, mock.Anything, mock.Anything, mock.Anything).
mock.Anything, false, mock.AnythingOfType("bool"), mock.Anything, mock.Anything, mock.Anything).
Run(func(args mock.Arguments) {
var ok bool
s.command, ok = args.Get(2).([]string)
s.Require().True(ok)
s.stdin, ok = args.Get(4).(*bytes.Buffer)
s.stdin, ok = args.Get(5).(*bytes.Buffer)
s.Require().True(ok)
}).Return(0, nil)
}
@ -274,7 +274,7 @@ func (s *UpdateFileSystemTestSuite) TestUpdateFileSystemForRunnerPerformsTarExtr
err := s.runner.UpdateFileSystem(copyRequest)
s.NoError(err)
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything,
false, mock.Anything, mock.Anything, mock.Anything)
false, mock.AnythingOfType("bool"), mock.Anything, mock.Anything, mock.Anything)
s.Regexp("tar --extract --absolute-names", s.command)
}
@ -297,7 +297,7 @@ func (s *UpdateFileSystemTestSuite) TestFilesToCopyAreIncludedInTarArchive() {
{Path: tests.DefaultFileName, Content: []byte(tests.DefaultFileContent)}}}
err := s.runner.UpdateFileSystem(copyRequest)
s.NoError(err)
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false,
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false, true,
mock.Anything, mock.Anything, mock.Anything)
tarFiles := s.readFilesFromTarArchive(s.stdin)
@ -348,7 +348,7 @@ func (s *UpdateFileSystemTestSuite) TestFilesToRemoveGetRemoved() {
copyRequest := &dto.UpdateFileSystemRequest{Delete: []dto.FilePath{tests.DefaultFileName}}
err := s.runner.UpdateFileSystem(copyRequest)
s.NoError(err)
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false,
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false, true,
mock.Anything, mock.Anything, mock.Anything)
s.Regexp(fmt.Sprintf("rm[^;]+%s' *;", regexp.QuoteMeta(tests.DefaultFileName)), s.command)
}
@ -357,7 +357,7 @@ func (s *UpdateFileSystemTestSuite) TestFilesToRemoveGetEscaped() {
copyRequest := &dto.UpdateFileSystemRequest{Delete: []dto.FilePath{"/some/potentially/harmful'filename"}}
err := s.runner.UpdateFileSystem(copyRequest)
s.NoError(err)
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false,
s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false, true,
mock.Anything, mock.Anything, mock.Anything)
s.Contains(strings.Join(s.command, " "), "'/some/potentially/harmful'\\''filename'")
}