htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Block Webserver during first Nomad recovery.

Browse Source 
No requests are accepted while Poseidon is recovering Nomad environments and runners.
This commit is contained in:
Maximilian Paß
2023-12-04 13:31:56 +01:00
parent 0e00957adb
commit eaa022282c
4 changed files with 69 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
cmd/poseidon/main.go
View File

@@ -3,6 +3,7 @@ package main
import (
"context"
"errors"
"fmt"
"github.com/getsentry/sentry-go"
sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/openHPI/poseidon/internal/api"
@@ -197,7 +198,7 @@ func createNomadManager(ctx context.Context) (runner.Manager, environment.Manage
// API initialization
nomadAPIClient, err := nomad.NewExecutorAPI(&config.Config.Nomad)
if err != nil {
log.WithError(err).WithField("nomad config", config.Config.Nomad).Fatal("Error creating Nomad API client")
log.WithError(err).WithField("nomad_config", config.Config.Nomad).Fatal("Error creating Nomad API client")
}
runnerManager := runner.NewNomadRunnerManager(nomadAPIClient, ctx)
@@ -206,10 +207,35 @@ func createNomadManager(ctx context.Context) (runner.Manager, environment.Manage
if err != nil {
log.WithError(err).Fatal("Error initializing environment manager")
}
go environmentManager.KeepEnvironmentsSynced(runnerManager.SynchronizeRunners, ctx)
synchronizeNomad(ctx, environmentManager, runnerManager)
return runnerManager, environmentManager
}
// synchronizeNomad starts the asynchronous synchronization background task and waits for the first environment and runner recovery.
func synchronizeNomad(ctx context.Context, environmentManager *environment.NomadEnvironmentManager, runnerManager *runner.NomadRunnerManager) {
firstRecoveryDone := make(chan struct{})
go environmentManager.KeepEnvironmentsSynced(func(ctx context.Context) error {
runnerManager.Load()
select {
case firstRecoveryDone <- struct{}{}:
log.Info("First Recovery Done")
default:
}
if err := runnerManager.SynchronizeRunners(ctx); err != nil {
return fmt.Errorf("synchronize runners failed: %w", err)
}
return nil
}, ctx)
select {
case <-firstRecoveryDone:
case <-ctx.Done():
}
}
func createAWSManager(ctx context.Context) (
runnerManager runner.Manager, environmentManager environment.ManagerHandler) {
runnerManager = runner.NewAWSRunnerManager(ctx)

18
cmd/poseidon/main_test.go
View File

@@ -2,9 +2,12 @@ package main
import (
"context"
"github.com/hashicorp/nomad/api"
"github.com/openHPI/poseidon/internal/environment"
"github.com/openHPI/poseidon/internal/nomad"
"github.com/openHPI/poseidon/internal/runner"
"github.com/openHPI/poseidon/tests"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"golang.org/x/sys/unix"
"testing"
@@ -63,3 +66,18 @@ func (s *MainTestSuite) TestShutdownOnOSSignal_Profiling() {
s.True(called)
}
func (s *MainTestSuite) TestLoadNomadEnvironmentsBeforeStartingWebserver() {
apiMock := &nomad.ExecutorAPIMock{}
apiMock.On("LoadEnvironmentJobs").Return([]*api.Job{}, nil)
apiMock.On("WatchEventStream", mock.Anything, mock.Anything).Run(func(_ mock.Arguments) {
<-s.TestCtx.Done()
}).Return(nil).Maybe()
runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx)
environmentManager, err := environment.NewNomadEnvironmentManager(runnerManager, apiMock, "")
s.Require().NoError(err)
synchronizeNomad(s.TestCtx, environmentManager, runnerManager)
apiMock.AssertExpectations(s.T())
}