htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce Allocation State Tracking

Browse Source 
in order to break down the current state and evaluate if it is invalid.
This commit is contained in:
Maximilian Paß
2023-06-01 01:40:37 +01:00
committed by Sebastian Serth
parent bcab46d746
commit b620d0fad7
5 changed files with 266 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/runner/nomad_manager.go
View File

@ -133,7 +133,7 @@ func (m *NomadRunnerManager) keepRunnersSynced(ctx context.Context) {
retries := 0
for ctx.Err() == nil {
err := m.apiClient.WatchEventStream(ctx,
&nomad.AllocationProcessoring{OnNew: m.onAllocationAdded, OnDeleted: m.onAllocationStopped})
&nomad.AllocationProcessing{OnNew: m.onAllocationAdded, OnDeleted: m.onAllocationStopped})
retries += 1
log.WithContext(ctx).WithError(err).Errorf("Stopped updating the runners! Retry %v", retries)
<-time.After(time.Second)
@ -177,22 +177,22 @@ func monitorAllocationStartupDuration(startup time.Duration, runnerID string, en
monitoring.WriteInfluxPoint(p)
}
func (m *NomadRunnerManager) onAllocationStopped(alloc *nomadApi.Allocation) {
log.WithField("id", alloc.JobID).Debug("Runner stopped")
func (m *NomadRunnerManager) onAllocationStopped(runnerID string) {
log.WithField("id", runnerID).Debug("Runner stopped")
if nomad.IsEnvironmentTemplateID(alloc.JobID) {
if nomad.IsEnvironmentTemplateID(runnerID) {
return
}
environmentID, err := nomad.EnvironmentIDFromRunnerID(alloc.JobID)
environmentID, err := nomad.EnvironmentIDFromRunnerID(runnerID)
if err != nil {
log.WithError(err).Warn("Stopped allocation can not be handled")
return
}
m.usedRunners.Delete(alloc.JobID)
m.usedRunners.Delete(runnerID)
environment, ok := m.environments.Get(environmentID.ToString())
if ok {
environment.DeleteRunner(alloc.JobID)
environment.DeleteRunner(runnerID)
}
}