htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#110 Refactor influxdb monitoring

Browse Source 
to use it as singleton.
This enables the possibility to monitor processes that are independent of an incoming request.
This commit is contained in:
Maximilian Paß
2022-06-29 20:05:19 +02:00
parent eafc01e69a
commit 498e8f5ff5
19 changed files with 174 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/api/api.go
View File

@ -2,7 +2,6 @@ package api
import (
"github.com/gorilla/mux"
influxdb2API "github.com/influxdata/influxdb-client-go/v2/api"
"github.com/openHPI/poseidon/internal/api/auth"
"github.com/openHPI/poseidon/internal/config"
"github.com/openHPI/poseidon/internal/environment"
@ -29,14 +28,13 @@ const (
// always returns a router for the newest version of our API. We
// use gorilla/mux because it is more convenient than net/http, e.g.
// when extracting path parameters.
func NewRouter(runnerManager runner.Manager, environmentManager environment.ManagerHandler,
influxClient influxdb2API.WriteAPI) *mux.Router {
func NewRouter(runnerManager runner.Manager, environmentManager environment.ManagerHandler) *mux.Router {
router := mux.NewRouter()
// this can later be restricted to a specific host with
// `router.Host(...)` and to HTTPS with `router.Schemes("https")`
configureV1Router(router, runnerManager, environmentManager)
router.Use(logging.HTTPLoggingMiddleware)
router.Use(monitoring.InfluxDB2Middleware(influxClient, environmentManager))
router.Use(monitoring.InfluxDB2Middleware)
return router
}

2
internal/api/environments_test.go
View File

@ -31,7 +31,7 @@ func TestEnvironmentControllerTestSuite(t *testing.T) {
func (s *EnvironmentControllerTestSuite) SetupTest() {
s.manager = &environment.ManagerHandlerMock{}
s.router = NewRouter(nil, s.manager, nil)
s.router = NewRouter(nil, s.manager)
}
func (s *EnvironmentControllerTestSuite) TestList() {

8
internal/api/runners.go
View File

@ -68,7 +68,7 @@ func (r *RunnerController) provide(writer http.ResponseWriter, request *http.Req
}
return
}
monitoring.AddRunnerMonitoringData(request, nextRunner)
monitoring.AddRunnerMonitoringData(request, nextRunner.ID(), nextRunner.Environment())
sendJSON(writer, &dto.RunnerResponse{ID: nextRunner.ID(), MappedPorts: nextRunner.MappedPorts()}, http.StatusOK)
}
@ -82,7 +82,7 @@ func (r *RunnerController) updateFileSystem(writer http.ResponseWriter, request
}
targetRunner, _ := runner.FromContext(request.Context())
monitoring.AddRunnerMonitoringData(request, targetRunner)
monitoring.AddRunnerMonitoringData(request, targetRunner.ID(), targetRunner.Environment())
if err := targetRunner.UpdateFileSystem(fileCopyRequest); err != nil {
log.WithError(err).Error("Could not perform the requested updateFileSystem.")
writeInternalServerError(writer, err, dto.ErrorUnknown)
@ -108,7 +108,7 @@ func (r *RunnerController) execute(writer http.ResponseWriter, request *http.Req
scheme = "ws"
}
targetRunner, _ := runner.FromContext(request.Context())
monitoring.AddRunnerMonitoringData(request, targetRunner)
monitoring.AddRunnerMonitoringData(request, targetRunner.ID(), targetRunner.Environment())
path, err := r.runnerRouter.Get(WebsocketPath).URL(RunnerIDKey, targetRunner.ID())
if err != nil {
@ -160,7 +160,7 @@ func (r *RunnerController) findRunnerMiddleware(next http.Handler) http.Handler
// It destroys the given runner on the executor and removes it from the used runners list.
func (r *RunnerController) delete(writer http.ResponseWriter, request *http.Request) {
targetRunner, _ := runner.FromContext(request.Context())
monitoring.AddRunnerMonitoringData(request, targetRunner)
monitoring.AddRunnerMonitoringData(request, targetRunner.ID(), targetRunner.Environment())
err := r.manager.Return(targetRunner)
if err != nil {

2
internal/api/runners_test.go
View File

@ -107,7 +107,7 @@ type RunnerRouteTestSuite struct {
func (s *RunnerRouteTestSuite) SetupTest() {
s.runnerManager = &runner.ManagerMock{}
s.router = NewRouter(s.runnerManager, nil, nil)
s.router = NewRouter(s.runnerManager, nil)
s.runner = runner.NewNomadJob("some-id", nil, nil, nil)
s.executionID = "execution"
s.runner.StoreExecution(s.executionID, &dto.ExecutionRequest{})

2
internal/api/websocket.go
View File

@ -76,7 +76,7 @@ func (wp *webSocketProxy) waitForExit(exit <-chan runner.ExitInfo, cancelExecuti
// connectToRunner is the endpoint for websocket connections.
func (r *RunnerController) connectToRunner(writer http.ResponseWriter, request *http.Request) {
targetRunner, _ := runner.FromContext(request.Context())
monitoring.AddRunnerMonitoringData(request, targetRunner)
monitoring.AddRunnerMonitoringData(request, targetRunner.ID(), targetRunner.Environment())
executionID := request.URL.Query().Get(ExecutionIDKey)
if !targetRunner.ExecutionExists(executionID) {

6
internal/api/websocket_test.go
View File

@ -51,7 +51,7 @@ func (s *WebSocketTestSuite) SetupTest() {
runnerManager := &runner.ManagerMock{}
runnerManager.On("Get", s.runner.ID()).Return(s.runner, nil)
s.router = NewRouter(runnerManager, nil, nil)
s.router = NewRouter(runnerManager, nil)
s.server = httptest.NewServer(s.router)
}
@ -257,7 +257,7 @@ func TestWebsocketTLS(t *testing.T) {
runnerManager := &runner.ManagerMock{}
runnerManager.On("Get", r.ID()).Return(r, nil)
router := NewRouter(runnerManager, nil, nil)
router := NewRouter(runnerManager, nil)
server, err := helpers.StartTLSServer(t, router)
require.NoError(t, err)
@ -327,7 +327,7 @@ func newRunnerWithNotMockedRunnerManager(t *testing.T, apiMock *nomad.ExecutorAP
call.ReturnArguments = mock.Arguments{nil}
})
runnerManager := runner.NewNomadRunnerManager(apiMock, context.Background())
router := NewRouter(runnerManager, nil, nil)
router := NewRouter(runnerManager, nil)
server := httptest.NewServer(router)
runnerID := tests.DefaultRunnerID

4
internal/environment/aws_environment.go
View File

@ -87,7 +87,7 @@ func (a *AWSEnvironment) CPULimit() uint {
func (a *AWSEnvironment) SetCPULimit(_ uint) {}
func (a *AWSEnvironment) MemoryLimit() uint {
panic("not supported")
return 0
}
func (a *AWSEnvironment) SetMemoryLimit(_ uint) {
@ -95,7 +95,7 @@ func (a *AWSEnvironment) SetMemoryLimit(_ uint) {
}
func (a *AWSEnvironment) NetworkAccess() (enabled bool, mappedPorts []uint16) {
panic("not supported")
return false, nil
}
func (a *AWSEnvironment) SetNetworkAccess(_ bool, _ []uint16) {

5
internal/environment/nomad_environment.go
View File

@ -11,6 +11,7 @@ import (
"github.com/openHPI/poseidon/internal/nomad"
"github.com/openHPI/poseidon/internal/runner"
"github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/pkg/monitoring"
"github.com/openHPI/poseidon/pkg/storage"
"strconv"
"sync"
@ -35,7 +36,8 @@ func NewNomadEnvironment(apiClient nomad.ExecutorAPI, jobHCL string) (*NomadEnvi
return nil, fmt.Errorf("error parsing Nomad job: %w", err)
}
return &NomadEnvironment{apiClient, jobHCL, job, storage.NewLocalStorage[runner.Runner]()}, nil
return &NomadEnvironment{apiClient, jobHCL, job,
storage.NewMonitoredLocalStorage[runner.Runner](monitoring.MeasurementIdleRunnerNomad, nil)}, nil
}
func NewNomadEnvironmentFromRequest(
@ -80,6 +82,7 @@ func (n *NomadEnvironment) PrewarmingPoolSize() uint {
}
func (n *NomadEnvironment) SetPrewarmingPoolSize(count uint) {
monitoring.ChangedPrewarmingPoolSize(n.ID(), count)
taskGroup := nomad.FindAndValidateConfigTaskGroup(n.job)
if taskGroup.Meta == nil {

5
internal/environment/nomad_manager.go
View File

@ -8,6 +8,7 @@ import (
"github.com/openHPI/poseidon/internal/runner"
"github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/pkg/logging"
"github.com/openHPI/poseidon/pkg/monitoring"
"github.com/openHPI/poseidon/pkg/storage"
"os"
)
@ -142,7 +143,7 @@ func (m *NomadEnvironmentManager) Load() error {
apiClient: m.api,
jobHCL: templateEnvironmentJobHCL,
job: job,
idleRunners: storage.NewLocalStorage[runner.Runner](),
idleRunners: storage.NewMonitoredLocalStorage[runner.Runner](monitoring.MeasurementIdleRunnerNomad, nil),
}
m.runnerManager.StoreEnvironment(environment)
jobLogger.Info("Successfully recovered environment")
@ -181,7 +182,7 @@ func fetchEnvironment(id dto.EnvironmentID, apiClient nomad.ExecutorAPI) (runner
apiClient: apiClient,
jobHCL: templateEnvironmentJobHCL,
job: job,
idleRunners: storage.NewLocalStorage[runner.Runner](),
idleRunners: storage.NewMonitoredLocalStorage[runner.Runner](monitoring.MeasurementIdleRunnerNomad, nil),
}
}
}

6
internal/runner/abstract_manager.go
View File

@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/pkg/monitoring"
"github.com/openHPI/poseidon/pkg/storage"
)
@ -21,8 +22,9 @@ type AbstractManager struct {
// NewAbstractManager creates a new abstract runner manager that keeps track of all runners of one kind.
func NewAbstractManager() *AbstractManager {
return &AbstractManager{
environments: storage.NewLocalStorage[ExecutionEnvironment](),
usedRunners: storage.NewLocalStorage[Runner](),
environments: storage.NewMonitoredLocalStorage[ExecutionEnvironment](
monitoring.MeasurementEnvironments, monitorEnvironmentData),
usedRunners: storage.NewMonitoredLocalStorage[Runner](monitoring.MeasurementUsedRunner, nil),
}
}

19
internal/runner/aws_manager_test.go
View File

@ -14,8 +14,7 @@ func TestAWSRunnerManager_EnvironmentAccessor(t *testing.T) {
environments := m.ListEnvironments()
assert.Empty(t, environments)
environment := &ExecutionEnvironmentMock{}
environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger))
environment := createBasicEnvironmentMock(defaultEnvironmentID)
m.StoreEnvironment(environment)
environments = m.ListEnvironments()
@ -32,8 +31,7 @@ func TestAWSRunnerManager_EnvironmentAccessor(t *testing.T) {
func TestAWSRunnerManager_Claim(t *testing.T) {
m := NewAWSRunnerManager()
environment := &ExecutionEnvironmentMock{}
environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger))
environment := createBasicEnvironmentMock(defaultEnvironmentID)
r, err := NewAWSFunctionWorkload(environment, nil)
assert.NoError(t, err)
environment.On("Sample").Return(r, true)
@ -59,8 +57,7 @@ func TestAWSRunnerManager_Claim(t *testing.T) {
func TestAWSRunnerManager_Return(t *testing.T) {
m := NewAWSRunnerManager()
environment := &ExecutionEnvironmentMock{}
environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger))
environment := createBasicEnvironmentMock(defaultEnvironmentID)
m.StoreEnvironment(environment)
r, err := NewAWSFunctionWorkload(environment, nil)
assert.NoError(t, err)
@ -85,3 +82,13 @@ func TestAWSRunnerManager_Return(t *testing.T) {
nextHandler.AssertCalled(t, "Return", nonAWSRunner)
})
}
func createBasicEnvironmentMock(id dto.EnvironmentID) *ExecutionEnvironmentMock {
environment := &ExecutionEnvironmentMock{}
environment.On("ID").Return(id)
environment.On("Image").Return("")
environment.On("CPULimit").Return(uint(0))
environment.On("MemoryLimit").Return(uint(0))
environment.On("NetworkAccess").Return(false, nil)
return environment
}

3
internal/runner/aws_runner.go
View File

@ -10,6 +10,7 @@ import (
"github.com/openHPI/poseidon/internal/config"
"github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/pkg/execution"
"github.com/openHPI/poseidon/pkg/monitoring"
"github.com/openHPI/poseidon/pkg/storage"
"io"
)
@ -47,7 +48,7 @@ func NewAWSFunctionWorkload(
workload := &AWSFunctionWorkload{
id: newUUID.String(),
fs: make(map[dto.FilePath][]byte),
executions: storage.NewLocalStorage[*dto.ExecutionRequest](),
executions: storage.NewMonitoredLocalStorage[*dto.ExecutionRequest](monitoring.MeasurementExecutionsAWS, nil),
runningExecutions: make(map[execution.ID]context.CancelFunc),
onDestroy: onDestroy,
environment: environment,

13
internal/runner/execution_environment.go
View File

@ -2,7 +2,9 @@ package runner
import (
"encoding/json"
"github.com/influxdata/influxdb-client-go/v2/api/write"
"github.com/openHPI/poseidon/pkg/dto"
"strconv"
)
// ExecutionEnvironment are groups of runner that share the configuration stored in the environment.
@ -47,3 +49,14 @@ type ExecutionEnvironment interface {
// IdleRunnerCount returns the number of idle runners of the environment.
IdleRunnerCount() uint
}
// monitorEnvironmentData passes the configuration of the environment e into the monitoring Point p.
func monitorEnvironmentData(p *write.Point, e ExecutionEnvironment, isDeletion bool) {
if !isDeletion && e != nil {
p.AddTag("image", e.Image())
p.AddTag("cpu_limit", strconv.Itoa(int(e.CPULimit())))
p.AddTag("memory_limit", strconv.Itoa(int(e.MemoryLimit())))
hasNetworkAccess, _ := e.NetworkAccess()
p.AddTag("network_access", strconv.FormatBool(hasNetworkAccess))
}
}

12
internal/runner/nomad_manager_test.go
View File

@ -36,8 +36,8 @@ func (s *ManagerTestSuite) SetupTest() {
s.nomadRunnerManager = NewNomadRunnerManager(s.apiMock, ctx)
s.exerciseRunner = NewRunner(tests.DefaultRunnerID, s.nomadRunnerManager)
s.exerciseEnvironment = &ExecutionEnvironmentMock{}
s.setDefaultEnvironment()
s.exerciseEnvironment = createBasicEnvironmentMock(defaultEnvironmentID)
s.nomadRunnerManager.StoreEnvironment(s.exerciseEnvironment)
}
func mockRunnerQueries(apiMock *nomad.ExecutorAPIMock, returnedRunnerIds []string) {
@ -81,18 +81,12 @@ func mockIdleRunners(environmentMock *ExecutionEnvironmentMock) {
})
}
func (s *ManagerTestSuite) setDefaultEnvironment() {
s.exerciseEnvironment.On("ID").Return(defaultEnvironmentID)
s.nomadRunnerManager.StoreEnvironment(s.exerciseEnvironment)
}
func (s *ManagerTestSuite) waitForRunnerRefresh() {
<-time.After(100 * time.Millisecond)
}
func (s *ManagerTestSuite) TestSetEnvironmentAddsNewEnvironment() {
anotherEnvironment := &ExecutionEnvironmentMock{}
anotherEnvironment.On("ID").Return(anotherEnvironmentID)
anotherEnvironment := createBasicEnvironmentMock(anotherEnvironmentID)
s.nomadRunnerManager.StoreEnvironment(anotherEnvironment)
job, ok := s.nomadRunnerManager.environments.Get(anotherEnvironmentID.ToString())

8
internal/runner/nomad_runner.go
View File

@ -11,18 +11,16 @@ import (
nomadApi "github.com/hashicorp/nomad/api"
"github.com/openHPI/poseidon/internal/nomad"
"github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/pkg/monitoring"
"github.com/openHPI/poseidon/pkg/storage"
"io"
"strings"
"time"
)
// ContextKey is the type for keys in a request context.
type ContextKey string
const (
// runnerContextKey is the key used to store runners in context.Context.
runnerContextKey ContextKey = "runner"
runnerContextKey dto.ContextKey = "runner"
// SIGQUIT is the character that causes a tty to send the SIGQUIT signal to the controlled process.
SIGQUIT = 0x1c
// executionTimeoutGracePeriod is the time to wait after sending a SIGQUIT signal to a timed out execution.
@ -55,7 +53,7 @@ func NewNomadJob(id string, portMappings []nomadApi.PortMapping,
id: id,
portMappings: portMappings,
api: apiClient,
executions: storage.NewLocalStorage[*dto.ExecutionRequest](),
executions: storage.NewMonitoredLocalStorage[*dto.ExecutionRequest](monitoring.MeasurementExecutionsNomad, nil),
onDestroy: onDestroy,
}
job.InactivityTimer = NewInactivityTimer(job, onDestroy)