htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix rescheduled used runners being removed.

Browse Source 
As they are already rescheduled and therefore recreated they do not need to be removed, but can be handled as a new runner.
This commit is contained in:
Maximilian Paß
2023-09-17 20:03:33 +02:00
committed by Sebastian Serth
parent 6dc83ca7b5
commit 3d252492fe
1 changed files with 18 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
internal/runner/nomad_runner.go
View File

@ -246,23 +246,30 @@ func (r *NomadJob) Destroy(reason DestroyReason) (err error) {
r.StopTimeout() r.StopTimeout()
if r.onDestroy != nil { if r.onDestroy != nil {
err = r.onDestroy(r) err = r.onDestroy(r)
if err != nil {
log.WithContext(r.ctx).WithError(err).Warn("runner onDestroy callback failed")
}
} }
if err == nil && !errors.Is(reason, ErrOOMKilled) { // local determines if a reason is present that the runner should only be removed locally (without requesting Nomad).
err = util.RetryExponential(func() (err error) { local := errors.Is(reason, nomad.ErrorAllocationRescheduled) ||
if err = r.api.DeleteJob(r.ID()); err != nil { errors.Is(reason, ErrOOMKilled)
err = fmt.Errorf("error deleting runner in Nomad: %w", err) if local {
} log.WithContext(r.ctx).Debug("Runner destroyed locally")
return return nil
})
} }
err = util.RetryExponential(func() (err error) {
if err = r.api.DeleteJob(r.ID()); err != nil {
err = fmt.Errorf("error deleting runner in Nomad: %w", err)
}
return
})
if err != nil { if err != nil {
err = fmt.Errorf("cannot destroy runner: %w", err) return fmt.Errorf("cannot destroy runner: %w", err)
} else {
log.WithContext(r.ctx).Trace("Runner destroyed")
} }
return err log.WithContext(r.ctx).Trace("Runner destroyed")
return nil
} }
func prepareExecution(request *dto.ExecutionRequest, environmentCtx context.Context) ( func prepareExecution(request *dto.ExecutionRequest, environmentCtx context.Context) (