From 8feffdae3ab303cb29ed7ef3165e9ea10998d381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Pa=C3=9F?= <22845248+mpass99@users.noreply.github.com> Date: Sat, 9 Apr 2022 17:51:34 +0200 Subject: [PATCH] Add initial structure of influxdb monitoring. --- cmd/poseidon/main.go | 25 +++++++- configuration.example.yaml | 12 ++++ go.mod | 17 ++++-- go.sum | 52 +++++++++++++++++ internal/api/api.go | 5 +- internal/api/environments_test.go | 2 +- internal/api/runners.go | 8 +++ internal/api/runners_test.go | 3 +- internal/api/websocket.go | 2 + internal/api/websocket_test.go | 6 +- internal/config/config.go | 25 ++++++-- pkg/logging/influxdb2_middleware.go | 88 +++++++++++++++++++++++++++++ 12 files changed, 225 insertions(+), 20 deletions(-) create mode 100644 pkg/logging/influxdb2_middleware.go diff --git a/cmd/poseidon/main.go b/cmd/poseidon/main.go index 5fb20df..9235c25 100644 --- a/cmd/poseidon/main.go +++ b/cmd/poseidon/main.go @@ -4,6 +4,8 @@ import ( "context" "errors" "github.com/getsentry/sentry-go" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + influxdb2API "github.com/influxdata/influxdb-client-go/v2/api" "github.com/openHPI/poseidon/internal/api" "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/internal/environment" @@ -35,6 +37,20 @@ func shutdownSentry() { } } +func initInfluxDB(db *config.InfluxDB) (writeAPI influxdb2API.WriteAPI, cancel func()) { + if db.Url == "" { + return nil, func() {} + } + + client := influxdb2.NewClient(db.Url, db.Token) + writeAPI = client.WriteAPI(db.Organization, db.Bucket) + cancel = func() { + writeAPI.Flush() + client.Close() + } + return writeAPI, cancel +} + func runServer(server *http.Server) { log.WithField("address", server.Addr).Info("Starting server") var err error @@ -95,7 +111,7 @@ func createAWSManager() (runnerManager runner.Manager, environmentManager enviro } // initServer builds the http server and configures it with the chain of responsibility for multiple managers. -func initServer() *http.Server { +func initServer(influxClient influxdb2API.WriteAPI) *http.Server { runnerManager, environmentManager := createManagerHandler(createNomadManager, config.Config.Nomad.Enabled, runner.NewAbstractManager(), &environment.AbstractManager{}) runnerManager, environmentManager = createManagerHandler(createAWSManager, config.Config.AWS.Enabled, @@ -105,7 +121,7 @@ func initServer() *http.Server { Addr: config.Config.Server.URL().Host, ReadTimeout: time.Second * 15, IdleTimeout: time.Second * 60, - Handler: api.NewRouter(runnerManager, environmentManager), + Handler: api.NewRouter(runnerManager, environmentManager, influxClient), } } @@ -133,7 +149,10 @@ func main() { initSentry(&config.Config.Sentry) defer shutdownSentry() - server := initServer() + influxDB, cancel := initInfluxDB(&config.Config.InfluxDB) + defer cancel() + + server := initServer(influxDB) go runServer(server) shutdownOnOSSignal(server) } diff --git a/configuration.example.yaml b/configuration.example.yaml index fcc18b5..fe32106 100644 --- a/configuration.example.yaml +++ b/configuration.example.yaml @@ -66,3 +66,15 @@ sentry: # This release information is used by Poseidon to provide the version route. # Normally it is set by the deployment process. # release: this is replaced in the deployment process + +# Configuration of the influxdb monitoring +influxdb: + # The url of the influxdb endpoint. + # If the url is not set, the influxdb monitoring is disabled. + url: http://localhost:8086 + # The token to secure the influxdb endpoint. + token: SecretBase64Token== + # The organisation set in your influxdb configuration. + organisation: PoseidonOrg + # The influxdb bucket to store the data in. + bucket: poseidon diff --git a/go.mod b/go.mod index 0b92fb6..5800192 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/hashicorp/nomad v1.2.6 github.com/hashicorp/nomad/api v0.0.0-20220318175809-7ff71ad19cf6 + github.com/influxdata/influxdb-client-go/v2 v2.8.1 github.com/mitchellh/mapstructure v1.4.3 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.1 @@ -22,6 +23,7 @@ require ( github.com/armon/go-metrics v0.3.10 // indirect github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.7 // indirect @@ -44,9 +46,10 @@ require ( github.com/hashicorp/raft v1.3.6 // indirect github.com/hashicorp/serf v0.9.7 // indirect github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect + github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/miekg/dns v1.1.47 // indirect + github.com/miekg/dns v1.1.48 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -54,19 +57,21 @@ require ( github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/zclconf/go-cty v1.10.0 // indirect github.com/zclconf/go-cty-yaml v1.0.2 // indirect - golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect + golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect - golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect + golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect + golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e // indirect + google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect google.golang.org/grpc v1.45.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect ) diff --git a/go.sum b/go.sum index cbe1025..916bb8d 100644 --- a/go.sum +++ b/go.sum @@ -46,9 +46,13 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -60,14 +64,18 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -91,6 +99,7 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -174,6 +183,8 @@ github.com/hashicorp/nomad v1.2.6 h1:8pZhmmtmaVHDRzLCkPvPkTKeZKDQY2MAHEoQOowJ9OY github.com/hashicorp/nomad v1.2.6/go.mod h1:tD1cklXLXFb9Z/THdmahBbmxrloqvTkQW8J7xqvnODc= github.com/hashicorp/nomad/api v0.0.0-20220318175809-7ff71ad19cf6 h1:JMr63Usze8EtYjzoG+L60TFnH+tQwZqSWyl1lI1kRNo= github.com/hashicorp/nomad/api v0.0.0-20220318175809-7ff71ad19cf6/go.mod h1:OcvjEDY+xMVaMl2/ev+Esz/IdJ529Qyrhc5ZQ6fq7AE= +github.com/hashicorp/nomad/api v0.0.0-20220407221241-ccaaadf49321 h1:t6GDfyy/lHbs61FxmTGAYXyaaeRzUesECCV0Nx8LmG4= +github.com/hashicorp/nomad/api v0.0.0-20220407221241-ccaaadf49321/go.mod h1:b/AoT79m3PEpb6tKCFKva/M+q1rKJNUk5mdu1S8DymM= github.com/hashicorp/raft v1.3.6 h1:v5xW5KzByoerQlN/o31VJrFNiozgzGyDoMgDJgXpsto= github.com/hashicorp/raft v1.3.6/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= @@ -182,6 +193,10 @@ github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/influxdata/influxdb-client-go/v2 v2.8.1 h1:DahMl2iYvr9hEtGZcV4Ud8Z7AQ8cQVbPlX6gEOBBCrE= +github.com/influxdata/influxdb-client-go/v2 v2.8.1/go.mod h1:x7Jo5UHHl+w8wu8UnGiNobDDHygojXwJX4mx7rXGKMk= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -197,14 +212,23 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -215,6 +239,8 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.47 h1:J9bWiXbqMbnZPcY8Qi2E3EWIBsIm6MZzzJB9VRg5gL8= github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= +github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -253,6 +279,7 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -299,6 +326,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -320,8 +350,12 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= +golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -345,12 +379,15 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 h1:EN5+DfgmRMvRUrMGERW2gQl3Vc+Z7ZMnI/xdEpPSf0c= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -369,6 +406,7 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -378,7 +416,9 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -387,6 +427,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 h1:QyVthZKMsyaQwBTJE04jdNN0Pp5Fn9Qga0mrgxyERQM= +golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -395,6 +438,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -403,6 +448,7 @@ golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= @@ -421,6 +467,8 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg= google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac h1:qSNTkEN+L2mvWcLgJOR+8bdHX9rN/IdU3A1Ghpfb1Rg= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -444,6 +492,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -455,6 +505,8 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/api.go b/internal/api/api.go index 0d61011..229c4fd 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -2,6 +2,7 @@ package api import ( "github.com/gorilla/mux" + influxdb2API "github.com/influxdata/influxdb-client-go/v2/api" "github.com/openHPI/poseidon/internal/api/auth" "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/internal/environment" @@ -27,12 +28,14 @@ const ( // always returns a router for the newest version of our API. We // use gorilla/mux because it is more convenient than net/http, e.g. // when extracting path parameters. -func NewRouter(runnerManager runner.Manager, environmentManager environment.ManagerHandler) *mux.Router { +func NewRouter(runnerManager runner.Manager, environmentManager environment.ManagerHandler, + influxClient influxdb2API.WriteAPI) *mux.Router { router := mux.NewRouter() // this can later be restricted to a specific host with // `router.Host(...)` and to HTTPS with `router.Schemes("https")` configureV1Router(router, runnerManager, environmentManager) router.Use(logging.HTTPLoggingMiddleware) + router.Use(logging.InfluxDB2Middleware(influxClient)) return router } diff --git a/internal/api/environments_test.go b/internal/api/environments_test.go index 6babb4a..1d71814 100644 --- a/internal/api/environments_test.go +++ b/internal/api/environments_test.go @@ -31,7 +31,7 @@ func TestEnvironmentControllerTestSuite(t *testing.T) { func (s *EnvironmentControllerTestSuite) SetupTest() { s.manager = &environment.ManagerHandlerMock{} - s.router = NewRouter(nil, s.manager) + s.router = NewRouter(nil, s.manager, nil) } func (s *EnvironmentControllerTestSuite) TestList() { diff --git a/internal/api/runners.go b/internal/api/runners.go index 888d5ea..d1e0f48 100644 --- a/internal/api/runners.go +++ b/internal/api/runners.go @@ -8,6 +8,7 @@ import ( "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/logging" "io" "net/http" "net/url" @@ -50,6 +51,8 @@ func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Req return } environmentID := dto.EnvironmentID(runnerRequest.ExecutionEnvironmentID) + logging.AddEnvironmentID(request, environmentID) + nextRunner, err := r.manager.Claim(environmentID, runnerRequest.InactivityTimeout) if err != nil { switch { @@ -63,18 +66,21 @@ func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Req } return } + logging.AddRunnerID(request, nextRunner.ID()) sendJSON(writer, &dto.RunnerResponse{ID: nextRunner.ID(), MappedPorts: nextRunner.MappedPorts()}, http.StatusOK) } // updateFileSystem handles the files API route. // It takes an dto.UpdateFileSystemRequest and sends it to the runner for processing. func (r *RunnerController) updateFileSystem(writer http.ResponseWriter, request *http.Request) { + logging.AddRequestSize(request) fileCopyRequest := new(dto.UpdateFileSystemRequest) if err := parseJSONRequestBody(writer, request, fileCopyRequest); err != nil { return } targetRunner, _ := runner.FromContext(request.Context()) + logging.AddRunnerID(request, targetRunner.ID()) if err := targetRunner.UpdateFileSystem(fileCopyRequest); err != nil { log.WithError(err).Error("Could not perform the requested updateFileSystem.") writeInternalServerError(writer, err, dto.ErrorUnknown) @@ -100,6 +106,7 @@ func (r *RunnerController) execute(writer http.ResponseWriter, request *http.Req scheme = "ws" } targetRunner, _ := runner.FromContext(request.Context()) + logging.AddRunnerID(request, targetRunner.ID()) path, err := r.runnerRouter.Get(WebsocketPath).URL(RunnerIDKey, targetRunner.ID()) if err != nil { @@ -151,6 +158,7 @@ func (r *RunnerController) findRunnerMiddleware(next http.Handler) http.Handler // It destroys the given runner on the executor and removes it from the used runners list. func (r *RunnerController) delete(writer http.ResponseWriter, request *http.Request) { targetRunner, _ := runner.FromContext(request.Context()) + logging.AddRunnerID(request, targetRunner.ID()) err := r.manager.Return(targetRunner) if err != nil { diff --git a/internal/api/runners_test.go b/internal/api/runners_test.go index c4b823e..f7836de 100644 --- a/internal/api/runners_test.go +++ b/internal/api/runners_test.go @@ -107,7 +107,7 @@ type RunnerRouteTestSuite struct { func (s *RunnerRouteTestSuite) SetupTest() { s.runnerManager = &runner.ManagerMock{} - s.router = NewRouter(s.runnerManager, nil) + s.router = NewRouter(s.runnerManager, nil, nil) s.runner = runner.NewNomadJob("some-id", nil, nil, nil) s.executionID = "execution" s.runner.StoreExecution(s.executionID, &dto.ExecutionRequest{}) @@ -253,6 +253,7 @@ func (s *UpdateFileSystemRouteTestSuite) SetupTest() { s.Require().NoError(err) s.path = routeURL.String() s.runnerMock = &runner.RunnerMock{} + s.runnerMock.On("ID").Return(tests.DefaultMockID) s.runnerManager.On("Get", tests.DefaultMockID).Return(s.runnerMock, nil) s.recorder = httptest.NewRecorder() } diff --git a/internal/api/websocket.go b/internal/api/websocket.go index 9e39bba..a99ff87 100644 --- a/internal/api/websocket.go +++ b/internal/api/websocket.go @@ -8,6 +8,7 @@ import ( "github.com/gorilla/websocket" "github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/logging" "io" "net/http" "sync" @@ -336,6 +337,7 @@ func (wp *webSocketProxy) writeMessage(messageType int, data []byte) error { // connectToRunner is the endpoint for websocket connections. func (r *RunnerController) connectToRunner(writer http.ResponseWriter, request *http.Request) { targetRunner, _ := runner.FromContext(request.Context()) + logging.AddRunnerID(request, targetRunner.ID()) executionID := request.URL.Query().Get(ExecutionIDKey) if !targetRunner.ExecutionExists(executionID) { writeNotFound(writer, ErrUnknownExecutionID) diff --git a/internal/api/websocket_test.go b/internal/api/websocket_test.go index d6db468..894a77a 100644 --- a/internal/api/websocket_test.go +++ b/internal/api/websocket_test.go @@ -53,7 +53,7 @@ func (s *WebSocketTestSuite) SetupTest() { runnerManager := &runner.ManagerMock{} runnerManager.On("Get", s.runner.ID()).Return(s.runner, nil) - s.router = NewRouter(runnerManager, nil) + s.router = NewRouter(runnerManager, nil, nil) s.server = httptest.NewServer(s.router) } @@ -259,7 +259,7 @@ func TestWebsocketTLS(t *testing.T) { runnerManager := &runner.ManagerMock{} runnerManager.On("Get", r.ID()).Return(r, nil) - router := NewRouter(runnerManager, nil) + router := NewRouter(runnerManager, nil, nil) server, err := helpers.StartTLSServer(t, router) require.NoError(t, err) @@ -426,7 +426,7 @@ func newRunnerWithNotMockedRunnerManager(t *testing.T, apiMock *nomad.ExecutorAP call.ReturnArguments = mock.Arguments{nil} }) runnerManager := runner.NewNomadRunnerManager(apiMock, context.Background()) - router := NewRouter(runnerManager, nil) + router := NewRouter(runnerManager, nil, nil) server := httptest.NewServer(router) runnerID := tests.DefaultRunnerID diff --git a/internal/config/config.go b/internal/config/config.go index 802d326..a02faf0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -53,6 +53,12 @@ var ( Level: "INFO", }, Sentry: sentry.ClientOptions{}, + InfluxDB: InfluxDB{ + Url: "", + Token: "", + Organization: "", + Bucket: "", + }, } configurationFilePath = "./configuration.yaml" configurationInitialized = false @@ -115,13 +121,22 @@ type logger struct { Level string } +// InfluxDB configures the usage of an Influx db monitoring. +type InfluxDB struct { + Url string + Token string + Organization string + Bucket string +} + // configuration contains the complete configuration of Poseidon. type configuration struct { - Server server - Nomad Nomad - AWS AWS - Logger logger - Sentry sentry.ClientOptions + Server server + Nomad Nomad + AWS AWS + Logger logger + Sentry sentry.ClientOptions + InfluxDB InfluxDB } // InitConfig merges configuration options from environment variables and diff --git a/pkg/logging/influxdb2_middleware.go b/pkg/logging/influxdb2_middleware.go new file mode 100644 index 0000000..3f921b4 --- /dev/null +++ b/pkg/logging/influxdb2_middleware.go @@ -0,0 +1,88 @@ +package logging + +import ( + "bytes" + "context" + "github.com/gorilla/mux" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + influxdb2API "github.com/influxdata/influxdb-client-go/v2/api" + "github.com/influxdata/influxdb-client-go/v2/api/write" + "github.com/openHPI/poseidon/pkg/dto" + "io" + "net/http" + "strconv" + "time" +) + +// InfluxdbContextKey is a key to reference the influxdb data point in the request context. +const InfluxdbContextKey = "influxdb data point" + +// InfluxdbMeasurementPrefix allows easier filtering in influxdb. +const InfluxdbMeasurementPrefix = "poseidon_" + +// InfluxDB2Middleware is a middleware to send events to an influx database. +func InfluxDB2Middleware(influxClient influxdb2API.WriteAPI) mux.MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + route := mux.CurrentRoute(r).GetName() + p := influxdb2.NewPointWithMeasurement(InfluxdbMeasurementPrefix + route) + + start := time.Now().UTC() + p.SetTime(time.Now()) + + requestWithPoint := r.WithContext(newContextWithPoint(r.Context(), p)) + lrw := NewLoggingResponseWriter(w) + next.ServeHTTP(lrw, requestWithPoint) + + p.AddField("duration", time.Now().UTC().Sub(start).Nanoseconds()) + p.AddTag("status", strconv.Itoa(lrw.statusCode)) + + if influxClient != nil { + influxClient.WritePoint(p) + } + }) + } +} + +// AddEnvironmentID adds the environment id to the influx data point for the current request. +func AddEnvironmentID(r *http.Request, id dto.EnvironmentID) { + p := pointFromContext(r.Context()) + p.AddTag("environment_id", strconv.Itoa(int(id))) +} + +// AddRunnerID adds the runner id to the influx data point for the current request. +func AddRunnerID(r *http.Request, id string) { + p := pointFromContext(r.Context()) + p.AddTag("runner_id", id) +} + +// AddRequestSize adds the size of the request body to the influx data point for the current request. +func AddRequestSize(r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + log.WithError(err).Warn("Failed to read request body") + } + + err = r.Body.Close() + if err != nil { + log.WithError(err).Warn("Failed to close request body") + } + r.Body = io.NopCloser(bytes.NewBuffer(body)) + + p := pointFromContext(r.Context()) + p.AddField("request_size", strconv.Itoa(len(body))) +} + +// newContextWithPoint creates a context containing an InfluxDB data point. +func newContextWithPoint(ctx context.Context, p *write.Point) context.Context { + return context.WithValue(ctx, InfluxdbContextKey, p) +} + +// pointFromContext returns an InfluxDB data point from a context. +func pointFromContext(ctx context.Context) *write.Point { + p, ok := ctx.Value(InfluxdbContextKey).(*write.Point) + if !ok { + log.Errorf("InfluxDB data point not stored in context.") + } + return p +}