#110 Refactor influxdb monitoring
to use it as singleton. This enables the possibility to monitor processes that are independent of an incoming request.
This commit is contained in:
Maximilian Paß
@ -8,69 +8,77 @@ import (
|
||||
influxdb2API "github.com/influxdata/influxdb-client-go/v2/api"
|
||||
"github.com/influxdata/influxdb-client-go/v2/api/write"
|
||||
"github.com/openHPI/poseidon/internal/config"
|
||||
"github.com/openHPI/poseidon/internal/environment"
|
||||
"github.com/openHPI/poseidon/internal/runner"
|
||||
"github.com/openHPI/poseidon/pkg/dto"
|
||||
"github.com/openHPI/poseidon/pkg/logging"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var log = logging.GetLogger("monitoring")
|
||||
|
||||
const (
|
||||
// influxdbContextKey is a key to reference the influxdb data point in the request context.
|
||||
influxdbContextKey runner.ContextKey = "influxdb data point"
|
||||
// influxdbMeasurementPrefix allows easier filtering in influxdb.
|
||||
influxdbMeasurementPrefix = "poseidon_"
|
||||
// influxdbContextKey is a key (runner.ContextKey) to reference the influxdb data point in the request context.
|
||||
influxdbContextKey dto.ContextKey = "influxdb data point"
|
||||
// measurementPrefix allows easier filtering in influxdb.
|
||||
measurementPrefix = "poseidon_"
|
||||
measurementPoolSize = measurementPrefix + "poolsize"
|
||||
MeasurementIdleRunnerNomad = measurementPrefix + "nomad_idle_runners"
|
||||
MeasurementExecutionsAWS = measurementPrefix + "aws_executions"
|
||||
MeasurementExecutionsNomad = measurementPrefix + "nomad_executions"
|
||||
MeasurementEnvironments = measurementPrefix + "environments"
|
||||
MeasurementUsedRunner = measurementPrefix + "used_runners"
|
||||
|
||||
// The keys for the monitored tags and fields.
|
||||
influxKeyRunnerID = "runner_id"
|
||||
influxKeyEnvironmentID = "environment_id"
|
||||
influxKeyEnvironmentType = "environment_type"
|
||||
influxKeyEnvironmentIdleRunner = "idle_runner"
|
||||
influxKeyEnvironmentPrewarmingPoolSize = "prewarming_pool_size"
|
||||
influxKeyRequestSize = "request_size"
|
||||
)
|
||||
|
||||
// InfluxDB2Middleware is a middleware to send events to an influx database.
|
||||
func InfluxDB2Middleware(influxClient influxdb2API.WriteAPI, manager environment.Manager) mux.MiddlewareFunc {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
route := mux.CurrentRoute(r).GetName()
|
||||
p := influxdb2.NewPointWithMeasurement(influxdbMeasurementPrefix + route)
|
||||
p.AddTag("stage", config.Config.InfluxDB.Stage)
|
||||
var (
|
||||
log = logging.GetLogger("monitoring")
|
||||
influxClient influxdb2API.WriteAPI
|
||||
)
|
||||
|
||||
start := time.Now().UTC()
|
||||
p.SetTime(time.Now())
|
||||
|
||||
ctx := context.WithValue(r.Context(), influxdbContextKey, p)
|
||||
requestWithPoint := r.WithContext(ctx)
|
||||
lrw := logging.NewLoggingResponseWriter(w)
|
||||
next.ServeHTTP(lrw, requestWithPoint)
|
||||
|
||||
p.AddField("duration", time.Now().UTC().Sub(start).Nanoseconds())
|
||||
p.AddTag("status", strconv.Itoa(lrw.StatusCode))
|
||||
|
||||
environmentID, err := strconv.Atoi(getEnvironmentID(p))
|
||||
if err == nil && manager != nil {
|
||||
addEnvironmentData(p, manager, dto.EnvironmentID(environmentID))
|
||||
}
|
||||
|
||||
if influxClient != nil {
|
||||
influxClient.WritePoint(p)
|
||||
}
|
||||
})
|
||||
func InitializeInfluxDB(db *config.InfluxDB) (cancel func()) {
|
||||
if db.URL == "" {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
client := influxdb2.NewClient(db.URL, db.Token)
|
||||
influxClient = client.WriteAPI(db.Organization, db.Bucket)
|
||||
cancel = func() {
|
||||
influxClient.Flush()
|
||||
client.Close()
|
||||
}
|
||||
return cancel
|
||||
}
|
||||
|
||||
// InfluxDB2Middleware is a middleware to send events to an influx database.
|
||||
func InfluxDB2Middleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
route := mux.CurrentRoute(r).GetName()
|
||||
p := influxdb2.NewPointWithMeasurement(measurementPrefix + route)
|
||||
|
||||
start := time.Now().UTC()
|
||||
p.SetTime(time.Now())
|
||||
|
||||
ctx := context.WithValue(r.Context(), influxdbContextKey, p)
|
||||
requestWithPoint := r.WithContext(ctx)
|
||||
lrw := logging.NewLoggingResponseWriter(w)
|
||||
next.ServeHTTP(lrw, requestWithPoint)
|
||||
|
||||
p.AddField("duration", time.Now().UTC().Sub(start).Nanoseconds())
|
||||
p.AddTag("status", strconv.Itoa(lrw.StatusCode))
|
||||
|
||||
WriteInfluxPoint(p)
|
||||
})
|
||||
}
|
||||
|
||||
// AddRunnerMonitoringData adds the data of the runner we want to monitor.
|
||||
func AddRunnerMonitoringData(request *http.Request, r runner.Runner) {
|
||||
addRunnerID(request, r.ID())
|
||||
addEnvironmentID(request, r.Environment())
|
||||
func AddRunnerMonitoringData(request *http.Request, runnerID string, environmentID dto.EnvironmentID) {
|
||||
addRunnerID(request, runnerID)
|
||||
addEnvironmentID(request, environmentID)
|
||||
}
|
||||
|
||||
// addRunnerID adds the runner id to the influx data point for the current request.
|
||||
@ -99,6 +107,23 @@ func AddRequestSize(r *http.Request) {
|
||||
addInfluxDBField(r, influxKeyRequestSize, len(body))
|
||||
}
|
||||
|
||||
func ChangedPrewarmingPoolSize(id dto.EnvironmentID, count uint) {
|
||||
p := influxdb2.NewPointWithMeasurement(measurementPoolSize)
|
||||
|
||||
p.AddTag(influxKeyEnvironmentID, strconv.Itoa(int(id)))
|
||||
p.AddField(influxKeyEnvironmentPrewarmingPoolSize, count)
|
||||
|
||||
WriteInfluxPoint(p)
|
||||
}
|
||||
|
||||
// WriteInfluxPoint schedules the indlux data point to be sent.
|
||||
func WriteInfluxPoint(p *write.Point) {
|
||||
if influxClient != nil {
|
||||
p.AddTag("stage", config.Config.InfluxDB.Stage)
|
||||
influxClient.WritePoint(p)
|
||||
}
|
||||
}
|
||||
|
||||
// addInfluxDBTag adds a tag to the influxdb data point in the request.
|
||||
func addInfluxDBTag(r *http.Request, key, value string) {
|
||||
dataPointFromRequest(r).AddTag(key, value)
|
||||
@ -117,32 +142,3 @@ func dataPointFromRequest(r *http.Request) *write.Point {
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// getEnvironmentID tries to find the environment id in the influxdb data point.
|
||||
func getEnvironmentID(p *write.Point) string {
|
||||
for _, tag := range p.TagList() {
|
||||
if tag.Key == influxKeyEnvironmentID {
|
||||
return tag.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// addEnvironmentData adds environment specific data to the influxdb data point.
|
||||
func addEnvironmentData(p *write.Point, manager environment.Manager, id dto.EnvironmentID) {
|
||||
e, err := manager.Get(id, false)
|
||||
if err == nil {
|
||||
p.AddTag(influxKeyEnvironmentType, getType(e))
|
||||
p.AddField(influxKeyEnvironmentIdleRunner, e.IdleRunnerCount())
|
||||
p.AddField(influxKeyEnvironmentPrewarmingPoolSize, e.PrewarmingPoolSize())
|
||||
}
|
||||
}
|
||||
|
||||
// Get type returns the type of the passed execution environment.
|
||||
func getType(e runner.ExecutionEnvironment) string {
|
||||
if t := reflect.TypeOf(e); t.Kind() == reflect.Ptr {
|
||||
return t.Elem().Name()
|
||||
} else {
|
||||
return t.Name()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user