htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add inactivity timeout for runners.

Browse Source 
By removing runners after a specified timeout they no longer stay
around indefinitely and block Nomads capacities. The timeout can be set
individually per runner when requesting the provide route. If it is set
to 0, the runner is never removed automatically.

The timeout is reset when activity is detected. Currently that is when
something gets executed or the filesystem gets modified.
This commit is contained in:
Konrad Hanff
2021-06-17 10:16:50 +02:00
parent c7ed54942d
commit 4b2cae0bd1
6 changed files with 212 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
api/runners.go
View File

@ -9,6 +9,7 @@ import (
"gitlab.hpi.de/codeocean/codemoon/poseidon/runner"
"net/http"
"net/url"
"time"
)
const (
@ -49,16 +50,19 @@ func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Req
environmentId := runner.EnvironmentID(runnerRequest.ExecutionEnvironmentId)
nextRunner, err := r.manager.Claim(environmentId)
if err != nil {
if err == runner.ErrUnknownExecutionEnvironment {
switch err {
case runner.ErrUnknownExecutionEnvironment:
writeNotFound(writer, err)
} else if err == runner.ErrNoRunnersAvailable {
case runner.ErrNoRunnersAvailable:
log.WithField("environment", environmentId).Warn("No runners available")
writeInternalServerError(writer, err, dto.ErrorNomadOverload)
} else {
default:
writeInternalServerError(writer, err, dto.ErrorUnknown)
}
return
}
timeout := time.Duration(runnerRequest.InactivityTimeout) * time.Second
nextRunner.SetupTimeout(timeout, nextRunner, r.manager)
sendJson(writer, &dto.RunnerResponse{Id: nextRunner.Id()}, http.StatusOK)
}

7
api/websocket.go
View File

@ -186,11 +186,8 @@ func (wp *webSocketProxy) waitForExit(exit <-chan runner.ExitInfo, cancelExecuti
return
}
if exitInfo.Err == context.DeadlineExceeded {
err := wp.sendToClient(dto.WebSocketMessage{Type: dto.WebSocketMetaTimeout})
if err != nil {
return
}
if errors.Is(exitInfo.Err, context.DeadlineExceeded) || errors.Is(exitInfo.Err, runner.ErrorRunnerInactivityTimeout) {
_ = wp.sendToClient(dto.WebSocketMessage{Type: dto.WebSocketMetaTimeout})
return
} else if exitInfo.Err != nil {
errorMessage := "Error executing the request"