htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Job ID to Nomad Allocation monitoring.

Browse Source
This commit is contained in:
Maximilian Paß
2023-04-23 12:43:20 +01:00
parent 63878e7715
commit d8d9abbddd
2 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/nomad/nomad.go
View File

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
nomadApi "github.com/hashicorp/nomad/api" nomadApi "github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs"
"github.com/influxdata/influxdb-client-go/v2/api/write"
"github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/internal/config"
"github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/pkg/logging" "github.com/openHPI/poseidon/pkg/logging"
@ -41,6 +42,7 @@ type AllocationProcessorMonitored func(*nomadApi.Allocation, time.Duration)
type allocationData struct { type allocationData struct {
// allocClientStatus defines the state defined by Nomad. // allocClientStatus defines the state defined by Nomad.
allocClientStatus string allocClientStatus string
jobID string
start time.Time start time.Time
} }
@ -104,7 +106,10 @@ func NewExecutorAPI(nomadConfig *config.Nomad) (ExecutorAPI, error) {
client := &APIClient{ client := &APIClient{
apiQuerier: &nomadAPIClient{}, apiQuerier: &nomadAPIClient{},
evaluations: map[string]chan error{}, evaluations: map[string]chan error{},
allocations: storage.NewMonitoredLocalStorage[*allocationData](monitoring.MeasurementNomadAllocations, nil, 0, nil), allocations: storage.NewMonitoredLocalStorage[*allocationData](monitoring.MeasurementNomadAllocations,
func(p *write.Point, object *allocationData, _ storage.EventType) {
p.AddTag(monitoring.InfluxKeyJobID, object.jobID)
}, 0, nil),
} }
err := client.init(nomadConfig) err := client.init(nomadConfig)
return client, err return client, err
@ -230,7 +235,7 @@ func (a *APIClient) initializeAllocations() {
stub.ClientStatus == structs.AllocClientStatusRunning { stub.ClientStatus == structs.AllocClientStatusRunning {
log.WithField("jobID", stub.JobID).WithField("status", stub.ClientStatus).Debug("Recovered Runner") log.WithField("jobID", stub.JobID).WithField("status", stub.ClientStatus).Debug("Recovered Runner")
a.allocations.Add(stub.ID, a.allocations.Add(stub.ID,
&allocationData{allocClientStatus: stub.ClientStatus, start: time.Unix(0, stub.CreateTime)}) &allocationData{allocClientStatus: stub.ClientStatus, start: time.Unix(0, stub.CreateTime), jobID: stub.JobID})
} }
} }
} }
@ -338,7 +343,8 @@ func handlePendingAllocationEvent(alloc *nomadApi.Allocation,
callbacks.OnDeleted(alloc) callbacks.OnDeleted(alloc)
} }
// Store Pending Allocation - Allocation gets started, wait until it runs. // Store Pending Allocation - Allocation gets started, wait until it runs.
allocations.Add(alloc.ID, &allocationData{allocClientStatus: structs.AllocClientStatusPending, start: time.Now()}) allocations.Add(alloc.ID,
&allocationData{allocClientStatus: structs.AllocClientStatusPending, start: time.Now(), jobID: alloc.JobID})
} else { } else {
log.WithField("alloc", alloc).Warn("Other Desired Status") log.WithField("alloc", alloc).Warn("Other Desired Status")
} }

1
pkg/monitoring/influxdb2_middleware.go
View File

@ -35,6 +35,7 @@ const (
InfluxKeyRunnerID = "runner_id" InfluxKeyRunnerID = "runner_id"
InfluxKeyEnvironmentID = "environment_id" InfluxKeyEnvironmentID = "environment_id"
InfluxKeyJobID = "job_id"
InfluxKeyActualContentLength = "actual_length" InfluxKeyActualContentLength = "actual_length"
InfluxKeyExpectedContentLength = "expected_length" InfluxKeyExpectedContentLength = "expected_length"
InfluxKeyDuration = "duration" InfluxKeyDuration = "duration"