htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce a context for RetryExponential

Browse Source 
as second criteria (next to the maximum number of attempts) for canceling the retrying. This is required as we started with the previous commit to retry the nomad environment recovery. This always fails for unit tests (as they are not connected to an Nomad cluster). Before, we ignored the one error but the retrying leads to unit test timeouts.
Additionally, we now stop retrying to create a runner when the environment got deleted.
This commit is contained in:
Maximilian Paß
2023-08-16 18:22:44 +02:00
committed by Sebastian Serth
parent 73759f8a3c
commit 13a9da95e5
7 changed files with 68 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cmd/poseidon/main.go
View File

@@ -135,49 +135,51 @@ func runServer(server *http.Server, cancel context.CancelFunc) {
}
}
type managerCreator func() (runnerManager runner.Manager, environmentManager environment.ManagerHandler)
type managerCreator func(ctx context.Context) (
runnerManager runner.Manager, environmentManager environment.ManagerHandler)
// createManagerHandler adds the managers of the passed managerCreator to the chain of responsibility.
func createManagerHandler(handler managerCreator, enabled bool,
nextRunnerManager runner.Manager, nextEnvironmentManager environment.ManagerHandler) (
nextRunnerManager runner.Manager, nextEnvironmentManager environment.ManagerHandler, ctx context.Context) (
runnerManager runner.Manager, environmentManager environment.ManagerHandler) {
if !enabled {
return nextRunnerManager, nextEnvironmentManager
}
runnerManager, environmentManager = handler()
runnerManager, environmentManager = handler(ctx)
runnerManager.SetNextHandler(nextRunnerManager)
environmentManager.SetNextHandler(nextEnvironmentManager)
return runnerManager, environmentManager
}
func createNomadManager() (runnerManager runner.Manager, environmentManager environment.ManagerHandler) {
func createNomadManager(ctx context.Context) (
runnerManager runner.Manager, environmentManager environment.ManagerHandler) {
// API initialization
nomadAPIClient, err := nomad.NewExecutorAPI(&config.Config.Nomad)
if err != nil {
log.WithError(err).WithField("nomad config", config.Config.Nomad).Fatal("Error creating Nomad API client")
}
runnerManager = runner.NewNomadRunnerManager(nomadAPIClient, context.Background())
runnerManager = runner.NewNomadRunnerManager(nomadAPIClient, ctx)
environmentManager, err = environment.
NewNomadEnvironmentManager(runnerManager, nomadAPIClient, config.Config.Server.TemplateJobFile)
NewNomadEnvironmentManager(runnerManager, nomadAPIClient, config.Config.Server.TemplateJobFile, ctx)
if err != nil {
log.WithError(err).Fatal("Error initializing environment manager")
}
return runnerManager, environmentManager
}
func createAWSManager() (runnerManager runner.Manager, environmentManager environment.ManagerHandler) {
func createAWSManager(_ context.Context) (runnerManager runner.Manager, environmentManager environment.ManagerHandler) {
runnerManager = runner.NewAWSRunnerManager()
return runnerManager, environment.NewAWSEnvironmentManager(runnerManager)
}
// initServer builds the http server and configures it with the chain of responsibility for multiple managers.
func initServer() *http.Server {
func initServer(ctx context.Context) *http.Server {
runnerManager, environmentManager := createManagerHandler(createNomadManager, config.Config.Nomad.Enabled,
nil, nil)
nil, nil, ctx)
runnerManager, environmentManager = createManagerHandler(createAWSManager, config.Config.AWS.Enabled,
runnerManager, environmentManager)
runnerManager, environmentManager, ctx)
handler := api.NewRouter(runnerManager, environmentManager)
sentryHandler := sentryhttp.New(sentryhttp.Options{}).Handle(handler)
@@ -239,7 +241,7 @@ func main() {
stopProfiling := initProfiling(config.Config.Profiling)
ctx, cancel := context.WithCancel(context.Background())
server := initServer()
server := initServer(ctx)
go runServer(server, cancel)
shutdownOnOSSignal(server, ctx, stopProfiling)
}

18
cmd/poseidon/main_test.go
View File

@@ -13,27 +13,35 @@ import (
)
func TestAWSDisabledUsesNomadManager(t *testing.T) {
disableRecovery, cancel := context.WithCancel(context.Background())
cancel()
runnerManager, environmentManager := createManagerHandler(createNomadManager, true,
runner.NewAbstractManager(), &environment.AbstractManager{})
runner.NewAbstractManager(), &environment.AbstractManager{}, disableRecovery)
awsRunnerManager, awsEnvironmentManager := createManagerHandler(createAWSManager, false,
runnerManager, environmentManager)
runnerManager, environmentManager, disableRecovery)
assert.Equal(t, runnerManager, awsRunnerManager)
assert.Equal(t, environmentManager, awsEnvironmentManager)
}
func TestAWSEnabledWrappesNomadManager(t *testing.T) {
disableRecovery, cancel := context.WithCancel(context.Background())
cancel()
runnerManager, environmentManager := createManagerHandler(createNomadManager, true,
runner.NewAbstractManager(), &environment.AbstractManager{})
runner.NewAbstractManager(), &environment.AbstractManager{}, disableRecovery)
awsRunnerManager, awsEnvironmentManager := createManagerHandler(createAWSManager,
true, runnerManager, environmentManager)
true, runnerManager, environmentManager, disableRecovery)
assert.NotEqual(t, runnerManager, awsRunnerManager)
assert.NotEqual(t, environmentManager, awsEnvironmentManager)
}
func TestShutdownOnOSSignal_Profiling(t *testing.T) {
called := false
disableRecovery, cancel := context.WithCancel(context.Background())
cancel()
server := initServer()
server := initServer(disableRecovery)
go shutdownOnOSSignal(server, context.Background(), func() {
called = true
})