htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Sentry Spans for Bash execution.

Browse Source
This commit is contained in:
Maximilian Paß
2023-02-17 13:25:12 +00:00
committed by Sebastian Serth
parent 526e938985
commit cc0c425197
4 changed files with 144 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
internal/nomad/nomad.go
View File

@ -398,7 +398,8 @@ func (a *APIClient) ExecuteCommand(jobID string,
return a.executeCommandInteractivelyWithStderr(jobID, ctx, command, privilegedExecution, stdin, stdout, stderr)
}
command = prepareCommandWithoutTTY(command, privilegedExecution)
exitCode, err := a.apiQuerier.Execute(jobID, ctx, command, tty, stdin, stdout, stderr)
debugWriter := &SentryDebugWriter{Target: stdout, Ctx: ctx}
exitCode, err := a.apiQuerier.Execute(jobID, ctx, command, tty, stdin, debugWriter, stderr)
if err != nil {
return 1, fmt.Errorf("error executing command in job %s: %w", jobID, err)
}
@ -422,8 +423,9 @@ func (a *APIClient) executeCommandInteractivelyWithStderr(allocationID string, c
// Catch stderr in separate execution.
logging.StartSpan("nomad.execute.stderr", "Execution for separate StdErr", ctx, func(ctx context.Context) {
debugWriterStderr := &SentryDebugWriter{Target: stderr, Ctx: ctx}
exit, err := a.Execute(allocationID, ctx, prepareCommandTTYStdErr(currentNanoTime, privilegedExecution), true,
nullio.Reader{Ctx: readingContext}, stderr, io.Discard)
nullio.Reader{Ctx: readingContext}, debugWriterStderr, io.Discard)
if err != nil {
log.WithError(err).WithField("runner", allocationID).Warn("Stderr task finished with error")
}
@ -435,7 +437,8 @@ func (a *APIClient) executeCommandInteractivelyWithStderr(allocationID string, c
var exit int
var err error
logging.StartSpan("nomad.execute.tty", "Interactive Execution", ctx, func(ctx context.Context) {
exit, err = a.Execute(allocationID, ctx, command, true, stdin, stdout, io.Discard)
debugWriter := &SentryDebugWriter{Target: stdout, Ctx: ctx}
exit, err = a.Execute(allocationID, ctx, command, true, stdin, debugWriter, io.Discard)
})
// Wait until the stderr catch command finished to make sure we receive all output.
@ -474,27 +477,42 @@ const (
UnprivilegedExecution = false
)
func prepareCommandWithoutTTY(commands []string, privilegedExecution bool) []string {
func prepareCommandWithoutTTY(srcCommands []string, privilegedExecution bool) []string {
commands := make([]string, len(srcCommands)) // nozero The size is required for the copy.
copy(commands, srcCommands)
commands[len(commands)-1] = setInnerDebugMessages(commands[len(commands)-1])
commands = setUserCommand(commands, privilegedExecution)
commands[len(commands)-1] = fmt.Sprintf("'%s'", commands[len(commands)-1])
cmd := strings.Join(commands, " ")
return []string{"bash", "-c", fmt.Sprintf(unsetEnvironmentVariablesFormat, unsetEnvironmentVariablesShell, cmd)}
cmd = fmt.Sprintf(unsetEnvironmentVariablesFormat, unsetEnvironmentVariablesShell, cmd)
// Debug Message
cmd = fmt.Sprintf(timeDebugMessageFormatStart, "SetUserAndUnsetEnv", cmd)
return []string{"bash", "-c", cmd}
}
func prepareCommandTTY(commands []string, currentNanoTime int64, privilegedExecution bool) []string {
func prepareCommandTTY(srcCommands []string, currentNanoTime int64, privilegedExecution bool) []string {
commands := make([]string, len(srcCommands)) // nozero The size is required for the copy.
copy(commands, srcCommands)
commands[len(commands)-1] = setInnerDebugMessages(commands[len(commands)-1])
commands = setUserCommand(commands, privilegedExecution)
// Take the last command which is the one to be executed and wrap it to redirect stderr.
stderrFifoPath := stderrFifo(currentNanoTime)
commands[len(commands)-1] =
fmt.Sprintf(stderrWrapperCommandFormat, stderrFifoPath, commands[len(commands)-1], stderrFifoPath)
cmd := strings.Join(commands, " ")
return []string{"bash", "-c", fmt.Sprintf(unsetEnvironmentVariablesFormat, unsetEnvironmentVariablesShell, cmd)}
cmd = fmt.Sprintf(unsetEnvironmentVariablesFormat, unsetEnvironmentVariablesShell, cmd)
// Debug Message
cmd = fmt.Sprintf(timeDebugMessageFormatStart, "SetUserAndUnsetEnv", cmd)
return []string{"bash", "-c", cmd}
}
func prepareCommandTTYStdErr(currentNanoTime int64, privilegedExecution bool) []string {
stderrFifoPath := stderrFifo(currentNanoTime)
command := []string{"bash", "-c", fmt.Sprintf(stderrFifoCommandFormat, stderrFifoPath, stderrFifoPath, stderrFifoPath)}
return setUserCommand(command, privilegedExecution)
cmd := fmt.Sprintf(stderrFifoCommandFormat, stderrFifoPath, stderrFifoPath, stderrFifoPath)
cmd = setInnerDebugMessages(cmd)
return setUserCommand([]string{"bash", "-c", cmd}, privilegedExecution)
}
// setUserCommand prefixes the passed command with the setUser command.
@ -510,3 +528,9 @@ func setUserCommand(command []string, privilegedExecution bool) []string {
func stderrFifo(id int64) string {
return fmt.Sprintf(stderrFifoFormat, id)
}
func setInnerDebugMessages(command string) (result string) {
result = fmt.Sprintf(timeDebugMessageFormatStart, "innerCommand", command)
result = strings.TrimSuffix(result, ";")
return fmt.Sprintf(timeDebugMessageFormatEnd, result, "End")
}