htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add inactivity timeout for runners.

Browse Source 
By removing runners after a specified timeout they no longer stay
around indefinitely and block Nomads capacities. The timeout can be set
individually per runner when requesting the provide route. If it is set
to 0, the runner is never removed automatically.

The timeout is reset when activity is detected. Currently that is when
something gets executed or the filesystem gets modified.
This commit is contained in:
Konrad Hanff
2021-06-17 10:16:50 +02:00
parent c7ed54942d
commit 4b2cae0bd1
6 changed files with 212 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
runner/manager.go
View File

@@ -185,6 +185,8 @@ func (m *NomadRunnerManager) Claim(environmentID EnvironmentID) (Runner, error)
return nil, fmt.Errorf("can't mark runner as used: %w", err)
}
runner.SetupTimeout(time.Duration(duration) * time.Second)
err = m.scaleEnvironment(environmentID)
if err != nil {
log.WithError(err).WithField("environmentID", environmentID).Error("Couldn't scale environment")
@@ -202,6 +204,7 @@ func (m *NomadRunnerManager) Get(runnerID string) (Runner, error) {
}
func (m *NomadRunnerManager) Return(r Runner) (err error) {
r.StopTimeout()
err = m.apiClient.DeleteRunner(r.Id())
if err != nil {
return
@@ -234,9 +237,15 @@ func (m *NomadRunnerManager) Load() {
continue
}
isUsed := configTaskGroup.Meta[nomad.ConfigMetaUsedKey] == nomad.ConfigMetaUsedValue
newJob := NewNomadJob(*job.ID, m.apiClient)
newJob := NewNomadJob(*job.ID, m.apiClient, m)
if isUsed {
m.usedRunners.Add(newJob)
timeout, err := strconv.Atoi(configTaskGroup.Meta[nomad.ConfigMetaTimeoutKey])
if err != nil {
log.WithError(err).Warn("Error loading timeout from meta values")
} else {
newJob.SetupTimeout(time.Duration(timeout) * time.Second)
}
} else {
environment.idleRunners.Add(newJob)
}
@@ -273,7 +282,7 @@ func (m *NomadRunnerManager) onAllocationAdded(alloc *nomadApi.Allocation) {
job, ok := m.environments.Get(environmentID)
if ok {
job.idleRunners.Add(NewNomadJob(alloc.JobID, m.apiClient))
job.idleRunners.Add(NewNomadJob(alloc.JobID, m.apiClient, m))
}
}