Fix Goroutine Leak of Environment Get

that was caused by creating an intermediate environment `fetchedEnvironment` when fetching the environments but not removing it in case that we just copy its configuration to the existing environment.
This commit is contained in:
Maximilian Paß
2023-09-11 10:22:19 +02:00
parent 460b8b2065
commit 59da36303c
8 changed files with 48 additions and 33 deletions

View File

@ -58,7 +58,7 @@ func (n *AbstractManager) Delete(id dto.EnvironmentID) (bool, error) {
} }
n.runnerManager.DeleteEnvironment(id) n.runnerManager.DeleteEnvironment(id)
if err := e.Delete(); err != nil { if err := e.Delete(false); err != nil {
return true, fmt.Errorf("could not delete environment: %w", err) return true, fmt.Errorf("could not delete environment: %w", err)
} }
return true, nil return true, nil

View File

@ -45,7 +45,7 @@ func (a *AWSEnvironment) SetImage(awsEndpoint string) {
a.awsEndpoint = awsEndpoint a.awsEndpoint = awsEndpoint
} }
func (a *AWSEnvironment) Delete() error { func (a *AWSEnvironment) Delete(_ bool) error {
return nil return nil
} }

View File

@ -219,15 +219,17 @@ func (n *NomadEnvironment) Register() error {
return nil return nil
} }
func (n *NomadEnvironment) Delete() error { func (n *NomadEnvironment) Delete(local bool) error {
n.cancel() n.cancel()
err := n.removeRunners() if !local {
if err != nil { err := n.removeRunners()
return err if err != nil {
} return err
err = n.apiClient.DeleteJob(*n.job.ID) }
if err != nil { err = n.apiClient.DeleteJob(*n.job.ID)
return fmt.Errorf("couldn't delete environment job: %w", err) if err != nil {
return fmt.Errorf("couldn't delete environment job: %w", err)
}
} }
return nil return nil
} }

View File

@ -165,7 +165,7 @@ func (s *MainTestSuite) TestParseJob() {
environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL)
s.NoError(err) s.NoError(err)
s.NotNil(environment.job) s.NotNil(environment.job)
s.NoError(environment.Delete()) s.NoError(environment.Delete(false))
}) })
s.Run("returns error when given wrong job", func() { s.Run("returns error when given wrong job", func() {
@ -216,7 +216,7 @@ func (s *MainTestSuite) TestSampleDoesNotSetForcePullFlag() {
_, job := helpers.CreateTemplateJob() _, job := helpers.CreateTemplateJob()
environment := &NomadEnvironment{apiMock, templateEnvironmentJobHCL, job, environment := &NomadEnvironment{apiMock, templateEnvironmentJobHCL, job,
storage.NewLocalStorage[runner.Runner](), context.Background(), nil} storage.NewLocalStorage[runner.Runner](), s.TestCtx, nil}
runner1 := &runner.RunnerMock{} runner1 := &runner.RunnerMock{}
runner1.On("ID").Return(tests.DefaultRunnerID) runner1.On("ID").Return(tests.DefaultRunnerID)
environment.AddRunner(runner1) environment.AddRunner(runner1)
@ -225,3 +225,13 @@ func (s *MainTestSuite) TestSampleDoesNotSetForcePullFlag() {
s.Require().True(ok) s.Require().True(ok)
<-time.After(tests.ShortTimeout) // New Runners are requested asynchronously <-time.After(tests.ShortTimeout) // New Runners are requested asynchronously
} }
func (s *MainTestSuite) TestNomadEnvironment_DeleteLocally() {
apiMock := &nomad.ExecutorAPIMock{}
environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL)
s.Require().NoError(err)
err = environment.Delete(true)
s.NoError(err)
apiMock.AssertExpectations(s.T())
}

View File

@ -72,6 +72,10 @@ func (m *NomadEnvironmentManager) Get(id dto.EnvironmentID, fetch bool) (
ok = true ok = true
default: default:
executionEnvironment.SetConfigFrom(fetchedEnvironment) executionEnvironment.SetConfigFrom(fetchedEnvironment)
err = fetchedEnvironment.Delete(true)
if err != nil {
log.WithError(err).Warn("Failed to remove environment locally")
}
} }
} }
@ -98,7 +102,7 @@ func (m *NomadEnvironmentManager) CreateOrUpdate(
if isExistingEnvironment { if isExistingEnvironment {
// Remove existing environment to force downloading the newest Docker image. // Remove existing environment to force downloading the newest Docker image.
// See https://github.com/openHPI/poseidon/issues/69 // See https://github.com/openHPI/poseidon/issues/69
err = environment.Delete() err = environment.Delete(false)
if err != nil { if err != nil {
return false, fmt.Errorf("failed to remove the environment: %w", err) return false, fmt.Errorf("failed to remove the environment: %w", err)
} }

View File

@ -127,7 +127,7 @@ func (s *MainTestSuite) TestNewNomadEnvironmentManager() {
s.NotNil(m) s.NotNil(m)
s.Equal(templateJobHCL, m.templateEnvironmentHCL) s.Equal(templateJobHCL, m.templateEnvironmentHCL)
s.NoError(environment.Delete()) s.NoError(environment.Delete(false))
}) })
s.Run("returns error if template file is invalid", func() { s.Run("returns error if template file is invalid", func() {
@ -145,8 +145,6 @@ func (s *MainTestSuite) TestNewNomadEnvironmentManager() {
} }
func (s *MainTestSuite) TestNomadEnvironmentManager_Get() { func (s *MainTestSuite) TestNomadEnvironmentManager_Get() {
s.T().Skip("ToDo: Get does not delete the replaced environment") // ToDo
disableRecovery, cancel := context.WithCancel(context.Background()) disableRecovery, cancel := context.WithCancel(context.Background())
cancel() cancel()
@ -179,7 +177,7 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_Get() {
s.NoError(err) s.NoError(err)
s.Equal(expectedEnvironment, environment) s.Equal(expectedEnvironment, environment)
err = environment.Delete() err = environment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
}) })
@ -213,11 +211,11 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_Get() {
s.NoError(err) s.NoError(err)
s.Equal(fetchedEnvironment.Image(), environment.Image()) s.Equal(fetchedEnvironment.Image(), environment.Image())
err = fetchedEnvironment.Delete() err = fetchedEnvironment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
err = environment.Delete() err = environment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
err = localEnvironment.Delete() err = localEnvironment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
}) })
runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger) runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger)
@ -239,9 +237,9 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_Get() {
s.NoError(err) s.NoError(err)
s.Equal(fetchedEnvironment.Image(), environment.Image()) s.Equal(fetchedEnvironment.Image(), environment.Image())
err = fetchedEnvironment.Delete() err = fetchedEnvironment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
err = environment.Delete() err = environment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
}) })
}) })
@ -281,7 +279,7 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_List() {
s.Equal(1, len(environments)) s.Equal(1, len(environments))
s.Equal(localEnvironment, environments[0]) s.Equal(localEnvironment, environments[0])
err = localEnvironment.Delete() err = localEnvironment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
}) })
runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger) runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger)
@ -308,9 +306,9 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_List() {
s.True(ok) s.True(ok)
s.Equal(fetchedEnvironment.job, nomadEnvironment.job) s.Equal(fetchedEnvironment.job, nomadEnvironment.job)
err = fetchedEnvironment.Delete() err = fetchedEnvironment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
err = nomadEnvironment.Delete() err = nomadEnvironment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
}) })
} }
@ -340,7 +338,7 @@ func (s *MainTestSuite) TestNomadEnvironmentManager_Load() {
s.Require().True(ok) s.Require().True(ok)
s.Equal("python:latest", environment.Image()) s.Equal("python:latest", environment.Image())
err = environment.Delete() err = environment.Delete(false)
s.Require().NoError(err) s.Require().NoError(err)
}) })

View File

@ -38,7 +38,8 @@ type ExecutionEnvironment interface {
// Register saves this environment at the executor. // Register saves this environment at the executor.
Register() error Register() error
// Delete removes this environment and all it's runner from the executor and Poseidon itself. // Delete removes this environment and all it's runner from the executor and Poseidon itself.
Delete() error // Iff local the environment is just removed from Poseidon without external escalation.
Delete(local bool) error
// Sample returns and removes an arbitrary available runner. // Sample returns and removes an arbitrary available runner.
// ok is true iff a runner was returned. // ok is true iff a runner was returned.

View File

@ -1,4 +1,4 @@
// Code generated by mockery v2.33.1. DO NOT EDIT. // Code generated by mockery v2.33.2. DO NOT EDIT.
package runner package runner
@ -45,13 +45,13 @@ func (_m *ExecutionEnvironmentMock) CPULimit() uint {
return r0 return r0
} }
// Delete provides a mock function with given fields: // Delete provides a mock function with given fields: local
func (_m *ExecutionEnvironmentMock) Delete() error { func (_m *ExecutionEnvironmentMock) Delete(local bool) error {
ret := _m.Called() ret := _m.Called(local)
var r0 error var r0 error
if rf, ok := ret.Get(0).(func() error); ok { if rf, ok := ret.Get(0).(func(bool) error); ok {
r0 = rf() r0 = rf(local)
} else { } else {
r0 = ret.Error(0) r0 = ret.Error(0)
} }