diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8735e2..5dbc7b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,3 +202,7 @@ jobs: ./poseidon & until curl -s --fail http://localhost:7200/api/v1/health ; do sleep 1; done make e2e-test + - name: Run e2e recovery tests + run: | + killall poseidon + make e2e-test-recovery diff --git a/Makefile b/Makefile index 6903105..45b8bfe 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PROJECT_NAME := "poseidon" REPOSITORY_OWNER = "openHPI" PKG := "github.com/$(REPOSITORY_OWNER)/$(PROJECT_NAME)/cmd/$(PROJECT_NAME)" -UNIT_TESTS = $(shell go list ./... | grep -v /e2e) +UNIT_TESTS = $(shell go list ./... | grep -v /e2e | grep -v /recovery) DOCKER_TAG := "poseidon:latest" DOCKER_OPTS := -v $(shell pwd)/configuration.yaml:/configuration.yaml @@ -109,6 +109,10 @@ e2e-test: deps ## Run e2e tests @[ -z "$(docker images -q $(E2E_TEST_DOCKER_IMAGE))" ] || docker pull $(E2E_TEST_DOCKER_IMAGE) @go test -count=1 ./tests/e2e -v -args -dockerImage="$(E2E_TEST_DOCKER_IMAGE)" +.PHONY: e2e-test-recovery +e2e-test-recovery: deps ## Run recovery e2e tests + @go test -count=1 ./tests/recovery -v -args -poseidonPath="../../poseidon" -dockerImage="$(E2E_TEST_DOCKER_IMAGE)" + .PHONY: e2e-docker e2e-docker: docker ## Run e2e tests against the Docker container docker run --rm -p 127.0.0.1:7200:7200 \ diff --git a/deploy/codeocean-terraform b/deploy/codeocean-terraform index 52adffb..b2e7989 160000 --- a/deploy/codeocean-terraform +++ b/deploy/codeocean-terraform @@ -1 +1 @@ -Subproject commit 52adffb1b5d73a27e7d40294559863d7b2dfb882 +Subproject commit b2e7989e5a99cd8203be16e1010c89fcac5739c8 diff --git a/tests/e2e/e2e_helpers.go b/tests/e2e/e2e_helpers.go new file mode 100644 index 0000000..87180b8 --- /dev/null +++ b/tests/e2e/e2e_helpers.go @@ -0,0 +1,102 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "github.com/openHPI/poseidon/internal/api" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/logging" + "github.com/openHPI/poseidon/tests" + "github.com/openHPI/poseidon/tests/helpers" + "net/http" + "time" +) + +var log = logging.GetLogger("e2e-helpers") + +func CreateDefaultEnvironment(prewarmingPoolSize uint, image string) dto.ExecutionEnvironmentRequest { + path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString) + defaultNomadEnvironment := dto.ExecutionEnvironmentRequest{ + PrewarmingPoolSize: prewarmingPoolSize, + CPULimit: 20, + MemoryLimit: 100, + Image: image, + NetworkAccess: false, + ExposedPorts: nil, + } + + resp, err := helpers.HTTPPutJSON(path, defaultNomadEnvironment) + if err != nil || resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent { + log.WithError(err).Fatal("Couldn't create default environment for e2e tests") + } + err = resp.Body.Close() + if err != nil { + log.Fatal("Failed closing body") + } + return defaultNomadEnvironment +} + +func WaitForDefaultEnvironment() { + path := helpers.BuildURL(api.BasePath, api.RunnersPath) + body := &dto.RunnerRequest{ + ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, + InactivityTimeout: 1, + } + var code int + const maxRetries = 60 + for count := 0; count < maxRetries && code != http.StatusOK; count++ { + <-time.After(time.Second) + resp, err := helpers.HTTPPostJSON(path, body) + if err == nil { + code = resp.StatusCode + log.WithField("count", count).WithField("statusCode", code).Info("Waiting for idle runners") + } else { + log.WithField("count", count).WithError(err).Warn("Waiting for idle runners") + } + _ = resp.Body.Close() + } + if code != http.StatusOK { + log.Fatal("Failed to provide a runner") + } +} + +// ProvideRunner creates a runner with the given RunnerRequest via an external request. +// It needs a running Poseidon instance to work. +func ProvideRunner(request *dto.RunnerRequest) (string, error) { + runnerURL := helpers.BuildURL(api.BasePath, api.RunnersPath) + resp, err := helpers.HTTPPostJSON(runnerURL, request) + if err != nil { + return "", fmt.Errorf("cannot post provide runner: %w", err) + } + if resp.StatusCode != http.StatusOK { + //nolint:goerr113 // dynamic error is ok in here, as it is a test + return "", fmt.Errorf("expected response code 200 when getting runner, got %v", resp.StatusCode) + } + runnerResponse := new(dto.RunnerResponse) + err = json.NewDecoder(resp.Body).Decode(runnerResponse) + if err != nil { + return "", fmt.Errorf("cannot decode runner response: %w", err) + } + _ = resp.Body.Close() + return runnerResponse.ID, nil +} + +// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request. +// It requires a running Poseidon instance. +func ProvideWebSocketURL(runnerID string, request *dto.ExecutionRequest) (string, error) { + url := helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.ExecutePath) + resp, err := helpers.HTTPPostJSON(url, request) + if err != nil { + return "", fmt.Errorf("cannot post provide websocket url: %w", err) + } else if resp.StatusCode != http.StatusOK { + return "", dto.ErrMissingData + } + + executionResponse := new(dto.ExecutionResponse) + err = json.NewDecoder(resp.Body).Decode(executionResponse) + if err != nil { + return "", fmt.Errorf("cannot parse execution response: %w", err) + } + _ = resp.Body.Close() + return executionResponse.WebSocketURL, nil +} diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 99548a9..9b36203 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -6,7 +6,6 @@ import ( "github.com/openHPI/poseidon/internal/api" "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/logging" "github.com/openHPI/poseidon/tests" "github.com/openHPI/poseidon/tests/helpers" "github.com/stretchr/testify/suite" @@ -23,7 +22,6 @@ import ( */ var ( - log = logging.GetLogger("e2e") testDockerImage = flag.String("dockerImage", "", "Docker image to use in E2E tests") nomadClient *nomadApi.Client nomadNamespace string @@ -94,56 +92,15 @@ func initNomad() { return } createDefaultEnvironment() - waitForDefaultEnvironment() + WaitForDefaultEnvironment() } func createDefaultEnvironment() { if *testDockerImage == "" { log.Fatal("You must specify the -dockerImage flag!") } - - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString) - - defaultNomadEnvironment = dto.ExecutionEnvironmentRequest{ - PrewarmingPoolSize: 10, - CPULimit: 20, - MemoryLimit: 100, - Image: *testDockerImage, - NetworkAccess: false, - ExposedPorts: nil, - } - - resp, err := helpers.HTTPPutJSON(path, defaultNomadEnvironment) - if err != nil || resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent { - log.WithError(err).Fatal("Couldn't create default environment for e2e tests") - } + defaultNomadEnvironment = CreateDefaultEnvironment(10, *testDockerImage) environmentIDs = append(environmentIDs, tests.DefaultEnvironmentIDAsInteger) - err = resp.Body.Close() - if err != nil { - log.Fatal("Failed closing body") - } -} - -func waitForDefaultEnvironment() { - path := helpers.BuildURL(api.BasePath, api.RunnersPath) - body := &dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - InactivityTimeout: 1, - } - var code int - const maxRetries = 60 - for count := 0; count < maxRetries && code != http.StatusOK; count++ { - <-time.After(time.Second) - if resp, err := helpers.HTTPPostJSON(path, body); err == nil { - code = resp.StatusCode - log.WithField("count", count).WithField("statusCode", code).Info("Waiting for idle runners") - } else { - log.WithField("count", count).WithError(err).Warn("Waiting for idle runners") - } - } - if code != http.StatusOK { - log.Fatal("Failed to provide a runner") - } } func deleteE2EEnvironments() { diff --git a/tests/e2e/runners_test.go b/tests/e2e/runners_test.go index 4ef817b..5f8f93d 100644 --- a/tests/e2e/runners_test.go +++ b/tests/e2e/runners_test.go @@ -56,31 +56,6 @@ func (s *E2ETestSuite) TestProvideRunnerRoute() { } } -// ProvideRunner creates a runner with the given RunnerRequest via an external request. -// It needs a running Poseidon instance to work. -func ProvideRunner(request *dto.RunnerRequest) (string, error) { - runnerURL := helpers.BuildURL(api.BasePath, api.RunnersPath) - runnerRequestByteString, err := json.Marshal(request) - if err != nil { - return "", err - } - reader := strings.NewReader(string(runnerRequestByteString)) - resp, err := http.Post(runnerURL, "application/json", reader) //nolint:gosec // runnerURL is not influenced by a user - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - //nolint:goerr113 // dynamic error is ok in here, as it is a test - return "", fmt.Errorf("expected response code 200 when getting runner, got %v", resp.StatusCode) - } - runnerResponse := new(dto.RunnerResponse) - err = json.NewDecoder(resp.Body).Decode(runnerResponse) - if err != nil { - return "", err - } - return runnerResponse.ID, nil -} - // CopyFiles sends the dto.UpdateFileSystemRequest to Poseidon. // It needs a running Poseidon instance to work. func CopyFiles(runnerID string, request *dto.UpdateFileSystemRequest) (*http.Response, error) { @@ -347,7 +322,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() { // User manipulates protected folder s.Run("User can create files", func() { - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{ + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ Command: fmt.Sprintf("touch %s/userfile", protectedFolderPath), TimeLimit: int(tests.DefaultTestTimeout.Seconds()), PrivilegedExecution: false, @@ -364,7 +339,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() { }) s.Run("User can not delete protected folder", func() { - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{ + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ Command: fmt.Sprintf("rm -fr %s", protectedFolderPath), TimeLimit: int(tests.DefaultTestTimeout.Seconds()), PrivilegedExecution: false, @@ -381,7 +356,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() { }) s.Run("User can not delete protected file", func() { - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{ + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ Command: fmt.Sprintf("rm -f %s", protectedFolderPath+tests.DefaultFileName), TimeLimit: int(tests.DefaultTestTimeout.Seconds()), PrivilegedExecution: false, @@ -398,7 +373,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() { }) s.Run("User can not write protected file", func() { - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{ + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ Command: fmt.Sprintf("echo Hi >> %s", protectedFolderPath+tests.DefaultFileName), TimeLimit: int(tests.DefaultTestTimeout.Seconds()), PrivilegedExecution: false, @@ -469,7 +444,7 @@ func (s *E2ETestSuite) TestRunnerGetsDestroyedAfterInactivityTimeout() { executionTerminated := make(chan bool) var lastMessage *dto.WebSocketMessage go func() { - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{Command: "sleep infinity"}) + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "sleep infinity"}) s.Require().NoError(err) connection, err := ConnectToWebSocket(webSocketURL) s.Require().NoError(err) @@ -497,7 +472,7 @@ func (s *E2ETestSuite) assertFileContent(runnerID, fileName, expectedContent str } func (s *E2ETestSuite) PrintContentOfFileOnRunner(runnerID, filename string) (stdout, stderr string) { - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{ + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ Command: fmt.Sprintf("cat %s", filename), TimeLimit: int(tests.DefaultTestTimeout.Seconds()), }) diff --git a/tests/e2e/websocket_test.go b/tests/e2e/websocket_test.go index 419d54f..c6fb551 100644 --- a/tests/e2e/websocket_test.go +++ b/tests/e2e/websocket_test.go @@ -3,10 +3,8 @@ package e2e import ( "bytes" "context" - "encoding/json" "fmt" "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/internal/api" "github.com/openHPI/poseidon/internal/nomad" "github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/tests" @@ -25,7 +23,7 @@ func (s *E2ETestSuite) TestExecuteCommandRoute() { runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)}) s.Require().NoError(err) - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{Command: "true"}) + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "true"}) s.Require().NoError(err) s.NotEqual("", webSocketURL) @@ -222,7 +220,7 @@ func (s *E2ETestSuite) TestNomadStderrFifoIsRemoved() { }) s.Require().NoError(err) - webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{Command: "ls -a /tmp/"}) + webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "ls -a /tmp/"}) s.Require().NoError(err) connection, err := ConnectToWebSocket(webSocketURL) s.Require().NoError(err) @@ -290,10 +288,8 @@ func ProvideWebSocketConnection(s *suite.Suite, environmentID dto.EnvironmentID, s.Require().NoError(err) s.Require().Equal(http.StatusNoContent, resp.StatusCode) } - webSocketURL, err := ProvideWebSocketURL(s, runnerID, executionRequest) - if err != nil { - return nil, fmt.Errorf("error providing WebSocket URL: %w", err) - } + webSocketURL, err := ProvideWebSocketURL(runnerID, executionRequest) + s.Require().NoError(err) connection, err := ConnectToWebSocket(webSocketURL) if err != nil { return nil, fmt.Errorf("error connecting to WebSocket: %w", err) @@ -301,23 +297,6 @@ func ProvideWebSocketConnection(s *suite.Suite, environmentID dto.EnvironmentID, return connection, nil } -// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request. -// It requires a running Poseidon instance. -func ProvideWebSocketURL(s *suite.Suite, runnerID string, request *dto.ExecutionRequest) (string, error) { - url := helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.ExecutePath) - executionRequestByteString, err := json.Marshal(request) - s.Require().NoError(err) - reader := strings.NewReader(string(executionRequestByteString)) - resp, err := http.Post(url, "application/json", reader) //nolint:gosec // url is not influenced by a user - s.Require().NoError(err) - s.Require().Equal(http.StatusOK, resp.StatusCode) - - executionResponse := new(dto.ExecutionResponse) - err = json.NewDecoder(resp.Body).Decode(executionResponse) - s.Require().NoError(err) - return executionResponse.WebSocketURL, nil -} - // ConnectToWebSocket establish an external WebSocket connection to the provided url. // It requires a running Poseidon instance. func ConnectToWebSocket(url string) (conn *websocket.Conn, err error) { diff --git a/tests/recovery/e2e_recovery_test.go b/tests/recovery/e2e_recovery_test.go new file mode 100644 index 0000000..2774a7f --- /dev/null +++ b/tests/recovery/e2e_recovery_test.go @@ -0,0 +1,136 @@ +package recovery + +import ( + "context" + "encoding/json" + "flag" + nomadApi "github.com/hashicorp/nomad/api" + "github.com/openHPI/poseidon/internal/api" + "github.com/openHPI/poseidon/internal/config" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/logging" + "github.com/openHPI/poseidon/tests" + "github.com/openHPI/poseidon/tests/e2e" + "github.com/openHPI/poseidon/tests/helpers" + "github.com/stretchr/testify/suite" + "net/http" + "os" + "testing" + "time" +) + +/* +* # E2E Recovery Tests +* +* For the e2e tests a nomad cluster must be connected and poseidon must be running. +* These cases test the behavior of Poseidon when restarting / recovering. + */ + +var ( + log = logging.GetLogger("e2e-recovery") + testDockerImage = flag.String("dockerImage", "", "Docker image to use in E2E tests") + poseidonBinary = flag.String("poseidonPath", "", "The path to the Poseidon binary") + nomadClient *nomadApi.Client + nomadNamespace string +) + +// InactivityTimeout of the created runner in seconds. +const ( + InactivityTimeout = 1 + PrewarmingPoolSize = 2 +) + +type E2ERecoveryTestSuite struct { + suite.Suite + runnerID string + poseidonCancel context.CancelFunc +} + +// Overwrite TestMain for custom setup. +func TestMain(m *testing.M) { + if err := config.InitConfig(); err != nil { + log.WithError(err).Fatal("Could not initialize configuration") + } + + if *poseidonBinary == "" { + log.Fatal("You must specify the -path to the Poseidon binary!") + } + if *testDockerImage == "" { + log.Fatal("You must specify the -dockerImage flag!") + } + + nomadNamespace = config.Config.Nomad.Namespace + var err error + nomadClient, err = nomadApi.NewClient(&nomadApi.Config{ + Address: config.Config.Nomad.URL().String(), + TLSConfig: &nomadApi.TLSConfig{}, + Namespace: nomadNamespace, + }) + if err != nil { + log.WithError(err).Fatal("Could not create Nomad client") + return + } + + os.Exit(m.Run()) +} + +func TestE2ERecoveryTests(t *testing.T) { + testSuite := new(E2ERecoveryTestSuite) + + ctx, cancelPoseidon := context.WithCancel(context.Background()) + testSuite.poseidonCancel = cancelPoseidon + + startPoseidon(ctx, cancelPoseidon) + waitForPoseidon() + + e2e.CreateDefaultEnvironment(PrewarmingPoolSize, *testDockerImage) + e2e.WaitForDefaultEnvironment() + + suite.Run(t, testSuite) + + TearDown() + testSuite.poseidonCancel() + <-time.After(tests.ShortTimeout) +} + +func (s *E2ERecoveryTestSuite) TestInactivityTimer_Valid() { + _, err := e2e.ProvideWebSocketURL(s.runnerID, &dto.ExecutionRequest{Command: "true"}) + s.NoError(err) +} + +func (s *E2ERecoveryTestSuite) TestInactivityTimer_Expired() { + <-time.After(InactivityTimeout * time.Second) + _, err := e2e.ProvideWebSocketURL(s.runnerID, &dto.ExecutionRequest{Command: "true"}) + s.Error(err) +} + +// We expect the runner count to be equal to the prewarming pool size plus the one provided runner. +// If the count does not include the provided runner, the evaluation of the runner status may be wrong. +func (s *E2ERecoveryTestSuite) TestRunnerCount() { + jobListStubs, _, err := nomadClient.Jobs().List(&nomadApi.QueryOptions{ + Prefix: tests.DefaultEnvironmentIDAsString, + Namespace: nomadNamespace, + }) + s.Require().NoError(err) + s.Equal(PrewarmingPoolSize+1, len(jobListStubs)) +} + +func (s *E2ERecoveryTestSuite) TestEnvironmentStatistics() { + url := helpers.BuildURL(api.BasePath, api.StatisticsPath, api.EnvironmentsPath) + response, err := http.Get(url) //nolint:gosec // The variability of this url is limited by our configurations. + s.Require().NoError(err) + s.Require().Equal(http.StatusOK, response.StatusCode) + + statistics := make(map[string]*dto.StatisticalExecutionEnvironmentData) + err = json.NewDecoder(response.Body).Decode(&statistics) + s.Require().NoError(err) + err = response.Body.Close() + s.Require().NoError(err) + + environmentStatistics, ok := statistics[tests.DefaultEnvironmentIDAsString] + s.Require().True(ok) + s.Equal(tests.DefaultEnvironmentIDAsInteger, environmentStatistics.ID) + s.Equal(uint(PrewarmingPoolSize), environmentStatistics.PrewarmingPoolSize) + s.Equal(uint(PrewarmingPoolSize), environmentStatistics.IdleRunners) + s.Equal(uint(1), environmentStatistics.UsedRunners) +} diff --git a/tests/recovery/setup_test.go b/tests/recovery/setup_test.go new file mode 100644 index 0000000..8db8de3 --- /dev/null +++ b/tests/recovery/setup_test.go @@ -0,0 +1,63 @@ +package recovery + +import ( + "context" + "github.com/openHPI/poseidon/internal/api" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/tests" + "github.com/openHPI/poseidon/tests/e2e" + "github.com/openHPI/poseidon/tests/helpers" + "net/http" + "os" + "os/exec" + "time" +) + +func (s *E2ERecoveryTestSuite) SetupTest() { + <-time.After(InactivityTimeout * time.Second) + // We do not want runner from the previous tests + + var err error + s.runnerID, err = e2e.ProvideRunner(&dto.RunnerRequest{ + ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, + InactivityTimeout: InactivityTimeout, + }) + if err != nil { + log.WithError(err).Fatal("Could not provide runner") + } + + <-time.After(tests.ShortTimeout) + s.poseidonCancel() + <-time.After(tests.ShortTimeout) + ctx, cancelPoseidon := context.WithCancel(context.Background()) + s.poseidonCancel = cancelPoseidon + startPoseidon(ctx, cancelPoseidon) + waitForPoseidon() +} + +func TearDown() { + path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString) + _, err := helpers.HTTPDelete(path, nil) + if err != nil { + log.WithError(err).Fatal("Could not remove default environment") + } +} + +func startPoseidon(ctx context.Context, cancelPoseidon context.CancelFunc) { + poseidon := exec.CommandContext(ctx, *poseidonBinary) //nolint:gosec // We accept that another binary can be executed. + poseidon.Stdout = os.Stdout + poseidon.Stderr = os.Stderr + if err := poseidon.Start(); err != nil { + cancelPoseidon() + log.WithError(err).Fatal("Failed to start Poseidon") + } +} + +func waitForPoseidon() { + done := false + for !done { + <-time.After(time.Second) + resp, err := http.Get(helpers.BuildURL(api.BasePath, api.HealthPath)) + done = err == nil && resp.StatusCode == http.StatusNoContent + } +}