htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Runner Destroy Reason Masking

Browse Source 
and ignore expected reasons such when the runner got destroyed by an API request.
This commit is contained in:
Maximilian Paß
2023-07-21 17:42:49 +01:00
parent 102b3f0701
commit eb818f92f7
3 changed files with 11 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/runner/nomad_manager.go
View File

@ -189,15 +189,6 @@ func (m *NomadRunnerManager) onAllocationStopped(runnerID string, reason error)
r, stillActive := m.usedRunners.Get(runnerID)
if stillActive {
// Mask the internal stop reason because the runner might disclose/forward it to CodeOcean/externally.
switch {
case errors.Is(reason, nomad.ErrorOOMKilled):
reason = ErrOOMKilled
default:
log.WithField(dto.KeyRunnerID, runnerID).WithField("reason", reason).Debug("Internal reason for allocation stop")
reason = ErrAllocationStopped
}
m.usedRunners.Delete(runnerID)
if err := r.Destroy(reason); err != nil {
log.WithError(err).Warn("Runner of stopped allocation cannot be destroyed")

5
internal/runner/nomad_runner.go
View File

@ -41,7 +41,6 @@ var (
ErrorUnknownExecution = errors.New("unknown execution")
ErrorFileCopyFailed = errors.New("file copy failed")
ErrFileNotFound = errors.New("file not found or insufficient permissions")
ErrAllocationStopped DestroyReason = errors.New("the allocation stopped")
ErrOOMKilled DestroyReason = nomad.ErrorOOMKilled
ErrDestroyedByAPIRequest DestroyReason = errors.New("the client wants to stop the runner")
ErrCannotStopExecution DestroyReason = errors.New("the execution did not stop after SIGQUIT")
@ -244,7 +243,7 @@ func (r *NomadJob) Destroy(reason DestroyReason) (err error) {
err = r.onDestroy(r)
}
if err == nil && (!errors.Is(reason, ErrAllocationStopped) || !errors.Is(reason, ErrOOMKilled)) {
if err == nil && !errors.Is(reason, ErrOOMKilled) {
err = util.RetryExponential(time.Second, func() (err error) {
if err = r.api.DeleteJob(r.ID()); err != nil {
err = fmt.Errorf("error deleting runner in Nomad: %w", err)
@ -331,7 +330,7 @@ func (r *NomadJob) handleContextDone(exitInternal <-chan ExitInfo, exit chan<- E
exit <- ExitInfo{255, err}
// This condition prevents further interaction with a stopped / dead allocation.
if errors.Is(err, ErrAllocationStopped) || errors.Is(err, ErrOOMKilled) {
if errors.Is(err, nomad.ErrorOOMKilled) {
return
}