#190 Add recovery e2e tests.
This commit is contained in:
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
@ -202,3 +202,7 @@ jobs:
|
|||||||
./poseidon &
|
./poseidon &
|
||||||
until curl -s --fail http://localhost:7200/api/v1/health ; do sleep 1; done
|
until curl -s --fail http://localhost:7200/api/v1/health ; do sleep 1; done
|
||||||
make e2e-test
|
make e2e-test
|
||||||
|
- name: Run e2e recovery tests
|
||||||
|
run: |
|
||||||
|
killall poseidon
|
||||||
|
make e2e-test-recovery
|
||||||
|
6
Makefile
6
Makefile
@ -1,7 +1,7 @@
|
|||||||
PROJECT_NAME := "poseidon"
|
PROJECT_NAME := "poseidon"
|
||||||
REPOSITORY_OWNER = "openHPI"
|
REPOSITORY_OWNER = "openHPI"
|
||||||
PKG := "github.com/$(REPOSITORY_OWNER)/$(PROJECT_NAME)/cmd/$(PROJECT_NAME)"
|
PKG := "github.com/$(REPOSITORY_OWNER)/$(PROJECT_NAME)/cmd/$(PROJECT_NAME)"
|
||||||
UNIT_TESTS = $(shell go list ./... | grep -v /e2e)
|
UNIT_TESTS = $(shell go list ./... | grep -v /e2e | grep -v /recovery)
|
||||||
|
|
||||||
DOCKER_TAG := "poseidon:latest"
|
DOCKER_TAG := "poseidon:latest"
|
||||||
DOCKER_OPTS := -v $(shell pwd)/configuration.yaml:/configuration.yaml
|
DOCKER_OPTS := -v $(shell pwd)/configuration.yaml:/configuration.yaml
|
||||||
@ -109,6 +109,10 @@ e2e-test: deps ## Run e2e tests
|
|||||||
@[ -z "$(docker images -q $(E2E_TEST_DOCKER_IMAGE))" ] || docker pull $(E2E_TEST_DOCKER_IMAGE)
|
@[ -z "$(docker images -q $(E2E_TEST_DOCKER_IMAGE))" ] || docker pull $(E2E_TEST_DOCKER_IMAGE)
|
||||||
@go test -count=1 ./tests/e2e -v -args -dockerImage="$(E2E_TEST_DOCKER_IMAGE)"
|
@go test -count=1 ./tests/e2e -v -args -dockerImage="$(E2E_TEST_DOCKER_IMAGE)"
|
||||||
|
|
||||||
|
.PHONY: e2e-test-recovery
|
||||||
|
e2e-test-recovery: deps ## Run recovery e2e tests
|
||||||
|
@go test -count=1 ./tests/recovery -v -args -poseidonPath="../../poseidon" -dockerImage="$(E2E_TEST_DOCKER_IMAGE)"
|
||||||
|
|
||||||
.PHONY: e2e-docker
|
.PHONY: e2e-docker
|
||||||
e2e-docker: docker ## Run e2e tests against the Docker container
|
e2e-docker: docker ## Run e2e tests against the Docker container
|
||||||
docker run --rm -p 127.0.0.1:7200:7200 \
|
docker run --rm -p 127.0.0.1:7200:7200 \
|
||||||
|
Submodule deploy/codeocean-terraform updated: 52adffb1b5...b2e7989e5a
102
tests/e2e/e2e_helpers.go
Normal file
102
tests/e2e/e2e_helpers.go
Normal file
@ -0,0 +1,102 @@
|
|||||||
|
package e2e
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"github.com/openHPI/poseidon/internal/api"
|
||||||
|
"github.com/openHPI/poseidon/pkg/dto"
|
||||||
|
"github.com/openHPI/poseidon/pkg/logging"
|
||||||
|
"github.com/openHPI/poseidon/tests"
|
||||||
|
"github.com/openHPI/poseidon/tests/helpers"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var log = logging.GetLogger("e2e-helpers")
|
||||||
|
|
||||||
|
func CreateDefaultEnvironment(prewarmingPoolSize uint, image string) dto.ExecutionEnvironmentRequest {
|
||||||
|
path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString)
|
||||||
|
defaultNomadEnvironment := dto.ExecutionEnvironmentRequest{
|
||||||
|
PrewarmingPoolSize: prewarmingPoolSize,
|
||||||
|
CPULimit: 20,
|
||||||
|
MemoryLimit: 100,
|
||||||
|
Image: image,
|
||||||
|
NetworkAccess: false,
|
||||||
|
ExposedPorts: nil,
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := helpers.HTTPPutJSON(path, defaultNomadEnvironment)
|
||||||
|
if err != nil || resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent {
|
||||||
|
log.WithError(err).Fatal("Couldn't create default environment for e2e tests")
|
||||||
|
}
|
||||||
|
err = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Failed closing body")
|
||||||
|
}
|
||||||
|
return defaultNomadEnvironment
|
||||||
|
}
|
||||||
|
|
||||||
|
func WaitForDefaultEnvironment() {
|
||||||
|
path := helpers.BuildURL(api.BasePath, api.RunnersPath)
|
||||||
|
body := &dto.RunnerRequest{
|
||||||
|
ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger,
|
||||||
|
InactivityTimeout: 1,
|
||||||
|
}
|
||||||
|
var code int
|
||||||
|
const maxRetries = 60
|
||||||
|
for count := 0; count < maxRetries && code != http.StatusOK; count++ {
|
||||||
|
<-time.After(time.Second)
|
||||||
|
resp, err := helpers.HTTPPostJSON(path, body)
|
||||||
|
if err == nil {
|
||||||
|
code = resp.StatusCode
|
||||||
|
log.WithField("count", count).WithField("statusCode", code).Info("Waiting for idle runners")
|
||||||
|
} else {
|
||||||
|
log.WithField("count", count).WithError(err).Warn("Waiting for idle runners")
|
||||||
|
}
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
}
|
||||||
|
if code != http.StatusOK {
|
||||||
|
log.Fatal("Failed to provide a runner")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProvideRunner creates a runner with the given RunnerRequest via an external request.
|
||||||
|
// It needs a running Poseidon instance to work.
|
||||||
|
func ProvideRunner(request *dto.RunnerRequest) (string, error) {
|
||||||
|
runnerURL := helpers.BuildURL(api.BasePath, api.RunnersPath)
|
||||||
|
resp, err := helpers.HTTPPostJSON(runnerURL, request)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("cannot post provide runner: %w", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
//nolint:goerr113 // dynamic error is ok in here, as it is a test
|
||||||
|
return "", fmt.Errorf("expected response code 200 when getting runner, got %v", resp.StatusCode)
|
||||||
|
}
|
||||||
|
runnerResponse := new(dto.RunnerResponse)
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(runnerResponse)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("cannot decode runner response: %w", err)
|
||||||
|
}
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return runnerResponse.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request.
|
||||||
|
// It requires a running Poseidon instance.
|
||||||
|
func ProvideWebSocketURL(runnerID string, request *dto.ExecutionRequest) (string, error) {
|
||||||
|
url := helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.ExecutePath)
|
||||||
|
resp, err := helpers.HTTPPostJSON(url, request)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("cannot post provide websocket url: %w", err)
|
||||||
|
} else if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", dto.ErrMissingData
|
||||||
|
}
|
||||||
|
|
||||||
|
executionResponse := new(dto.ExecutionResponse)
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(executionResponse)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("cannot parse execution response: %w", err)
|
||||||
|
}
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return executionResponse.WebSocketURL, nil
|
||||||
|
}
|
@ -6,7 +6,6 @@ import (
|
|||||||
"github.com/openHPI/poseidon/internal/api"
|
"github.com/openHPI/poseidon/internal/api"
|
||||||
"github.com/openHPI/poseidon/internal/config"
|
"github.com/openHPI/poseidon/internal/config"
|
||||||
"github.com/openHPI/poseidon/pkg/dto"
|
"github.com/openHPI/poseidon/pkg/dto"
|
||||||
"github.com/openHPI/poseidon/pkg/logging"
|
|
||||||
"github.com/openHPI/poseidon/tests"
|
"github.com/openHPI/poseidon/tests"
|
||||||
"github.com/openHPI/poseidon/tests/helpers"
|
"github.com/openHPI/poseidon/tests/helpers"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
@ -23,7 +22,6 @@ import (
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
var (
|
var (
|
||||||
log = logging.GetLogger("e2e")
|
|
||||||
testDockerImage = flag.String("dockerImage", "", "Docker image to use in E2E tests")
|
testDockerImage = flag.String("dockerImage", "", "Docker image to use in E2E tests")
|
||||||
nomadClient *nomadApi.Client
|
nomadClient *nomadApi.Client
|
||||||
nomadNamespace string
|
nomadNamespace string
|
||||||
@ -94,56 +92,15 @@ func initNomad() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
createDefaultEnvironment()
|
createDefaultEnvironment()
|
||||||
waitForDefaultEnvironment()
|
WaitForDefaultEnvironment()
|
||||||
}
|
}
|
||||||
|
|
||||||
func createDefaultEnvironment() {
|
func createDefaultEnvironment() {
|
||||||
if *testDockerImage == "" {
|
if *testDockerImage == "" {
|
||||||
log.Fatal("You must specify the -dockerImage flag!")
|
log.Fatal("You must specify the -dockerImage flag!")
|
||||||
}
|
}
|
||||||
|
defaultNomadEnvironment = CreateDefaultEnvironment(10, *testDockerImage)
|
||||||
path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString)
|
|
||||||
|
|
||||||
defaultNomadEnvironment = dto.ExecutionEnvironmentRequest{
|
|
||||||
PrewarmingPoolSize: 10,
|
|
||||||
CPULimit: 20,
|
|
||||||
MemoryLimit: 100,
|
|
||||||
Image: *testDockerImage,
|
|
||||||
NetworkAccess: false,
|
|
||||||
ExposedPorts: nil,
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := helpers.HTTPPutJSON(path, defaultNomadEnvironment)
|
|
||||||
if err != nil || resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent {
|
|
||||||
log.WithError(err).Fatal("Couldn't create default environment for e2e tests")
|
|
||||||
}
|
|
||||||
environmentIDs = append(environmentIDs, tests.DefaultEnvironmentIDAsInteger)
|
environmentIDs = append(environmentIDs, tests.DefaultEnvironmentIDAsInteger)
|
||||||
err = resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal("Failed closing body")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForDefaultEnvironment() {
|
|
||||||
path := helpers.BuildURL(api.BasePath, api.RunnersPath)
|
|
||||||
body := &dto.RunnerRequest{
|
|
||||||
ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger,
|
|
||||||
InactivityTimeout: 1,
|
|
||||||
}
|
|
||||||
var code int
|
|
||||||
const maxRetries = 60
|
|
||||||
for count := 0; count < maxRetries && code != http.StatusOK; count++ {
|
|
||||||
<-time.After(time.Second)
|
|
||||||
if resp, err := helpers.HTTPPostJSON(path, body); err == nil {
|
|
||||||
code = resp.StatusCode
|
|
||||||
log.WithField("count", count).WithField("statusCode", code).Info("Waiting for idle runners")
|
|
||||||
} else {
|
|
||||||
log.WithField("count", count).WithError(err).Warn("Waiting for idle runners")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if code != http.StatusOK {
|
|
||||||
log.Fatal("Failed to provide a runner")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func deleteE2EEnvironments() {
|
func deleteE2EEnvironments() {
|
||||||
|
@ -56,31 +56,6 @@ func (s *E2ETestSuite) TestProvideRunnerRoute() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProvideRunner creates a runner with the given RunnerRequest via an external request.
|
|
||||||
// It needs a running Poseidon instance to work.
|
|
||||||
func ProvideRunner(request *dto.RunnerRequest) (string, error) {
|
|
||||||
runnerURL := helpers.BuildURL(api.BasePath, api.RunnersPath)
|
|
||||||
runnerRequestByteString, err := json.Marshal(request)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
reader := strings.NewReader(string(runnerRequestByteString))
|
|
||||||
resp, err := http.Post(runnerURL, "application/json", reader) //nolint:gosec // runnerURL is not influenced by a user
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
//nolint:goerr113 // dynamic error is ok in here, as it is a test
|
|
||||||
return "", fmt.Errorf("expected response code 200 when getting runner, got %v", resp.StatusCode)
|
|
||||||
}
|
|
||||||
runnerResponse := new(dto.RunnerResponse)
|
|
||||||
err = json.NewDecoder(resp.Body).Decode(runnerResponse)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return runnerResponse.ID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CopyFiles sends the dto.UpdateFileSystemRequest to Poseidon.
|
// CopyFiles sends the dto.UpdateFileSystemRequest to Poseidon.
|
||||||
// It needs a running Poseidon instance to work.
|
// It needs a running Poseidon instance to work.
|
||||||
func CopyFiles(runnerID string, request *dto.UpdateFileSystemRequest) (*http.Response, error) {
|
func CopyFiles(runnerID string, request *dto.UpdateFileSystemRequest) (*http.Response, error) {
|
||||||
@ -347,7 +322,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() {
|
|||||||
|
|
||||||
// User manipulates protected folder
|
// User manipulates protected folder
|
||||||
s.Run("User can create files", func() {
|
s.Run("User can create files", func() {
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{
|
||||||
Command: fmt.Sprintf("touch %s/userfile", protectedFolderPath),
|
Command: fmt.Sprintf("touch %s/userfile", protectedFolderPath),
|
||||||
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
||||||
PrivilegedExecution: false,
|
PrivilegedExecution: false,
|
||||||
@ -364,7 +339,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
s.Run("User can not delete protected folder", func() {
|
s.Run("User can not delete protected folder", func() {
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{
|
||||||
Command: fmt.Sprintf("rm -fr %s", protectedFolderPath),
|
Command: fmt.Sprintf("rm -fr %s", protectedFolderPath),
|
||||||
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
||||||
PrivilegedExecution: false,
|
PrivilegedExecution: false,
|
||||||
@ -381,7 +356,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
s.Run("User can not delete protected file", func() {
|
s.Run("User can not delete protected file", func() {
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{
|
||||||
Command: fmt.Sprintf("rm -f %s", protectedFolderPath+tests.DefaultFileName),
|
Command: fmt.Sprintf("rm -f %s", protectedFolderPath+tests.DefaultFileName),
|
||||||
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
||||||
PrivilegedExecution: false,
|
PrivilegedExecution: false,
|
||||||
@ -398,7 +373,7 @@ func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
s.Run("User can not write protected file", func() {
|
s.Run("User can not write protected file", func() {
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{
|
||||||
Command: fmt.Sprintf("echo Hi >> %s", protectedFolderPath+tests.DefaultFileName),
|
Command: fmt.Sprintf("echo Hi >> %s", protectedFolderPath+tests.DefaultFileName),
|
||||||
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
||||||
PrivilegedExecution: false,
|
PrivilegedExecution: false,
|
||||||
@ -469,7 +444,7 @@ func (s *E2ETestSuite) TestRunnerGetsDestroyedAfterInactivityTimeout() {
|
|||||||
executionTerminated := make(chan bool)
|
executionTerminated := make(chan bool)
|
||||||
var lastMessage *dto.WebSocketMessage
|
var lastMessage *dto.WebSocketMessage
|
||||||
go func() {
|
go func() {
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{Command: "sleep infinity"})
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "sleep infinity"})
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
connection, err := ConnectToWebSocket(webSocketURL)
|
connection, err := ConnectToWebSocket(webSocketURL)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
@ -497,7 +472,7 @@ func (s *E2ETestSuite) assertFileContent(runnerID, fileName, expectedContent str
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *E2ETestSuite) PrintContentOfFileOnRunner(runnerID, filename string) (stdout, stderr string) {
|
func (s *E2ETestSuite) PrintContentOfFileOnRunner(runnerID, filename string) (stdout, stderr string) {
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{
|
||||||
Command: fmt.Sprintf("cat %s", filename),
|
Command: fmt.Sprintf("cat %s", filename),
|
||||||
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
TimeLimit: int(tests.DefaultTestTimeout.Seconds()),
|
||||||
})
|
})
|
||||||
|
@ -3,10 +3,8 @@ package e2e
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/gorilla/websocket"
|
"github.com/gorilla/websocket"
|
||||||
"github.com/openHPI/poseidon/internal/api"
|
|
||||||
"github.com/openHPI/poseidon/internal/nomad"
|
"github.com/openHPI/poseidon/internal/nomad"
|
||||||
"github.com/openHPI/poseidon/pkg/dto"
|
"github.com/openHPI/poseidon/pkg/dto"
|
||||||
"github.com/openHPI/poseidon/tests"
|
"github.com/openHPI/poseidon/tests"
|
||||||
@ -25,7 +23,7 @@ func (s *E2ETestSuite) TestExecuteCommandRoute() {
|
|||||||
runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)})
|
runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)})
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{Command: "true"})
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "true"})
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
s.NotEqual("", webSocketURL)
|
s.NotEqual("", webSocketURL)
|
||||||
|
|
||||||
@ -222,7 +220,7 @@ func (s *E2ETestSuite) TestNomadStderrFifoIsRemoved() {
|
|||||||
})
|
})
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
webSocketURL, err := ProvideWebSocketURL(&s.Suite, runnerID, &dto.ExecutionRequest{Command: "ls -a /tmp/"})
|
webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "ls -a /tmp/"})
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
connection, err := ConnectToWebSocket(webSocketURL)
|
connection, err := ConnectToWebSocket(webSocketURL)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
@ -290,10 +288,8 @@ func ProvideWebSocketConnection(s *suite.Suite, environmentID dto.EnvironmentID,
|
|||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
s.Require().Equal(http.StatusNoContent, resp.StatusCode)
|
s.Require().Equal(http.StatusNoContent, resp.StatusCode)
|
||||||
}
|
}
|
||||||
webSocketURL, err := ProvideWebSocketURL(s, runnerID, executionRequest)
|
webSocketURL, err := ProvideWebSocketURL(runnerID, executionRequest)
|
||||||
if err != nil {
|
s.Require().NoError(err)
|
||||||
return nil, fmt.Errorf("error providing WebSocket URL: %w", err)
|
|
||||||
}
|
|
||||||
connection, err := ConnectToWebSocket(webSocketURL)
|
connection, err := ConnectToWebSocket(webSocketURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("error connecting to WebSocket: %w", err)
|
return nil, fmt.Errorf("error connecting to WebSocket: %w", err)
|
||||||
@ -301,23 +297,6 @@ func ProvideWebSocketConnection(s *suite.Suite, environmentID dto.EnvironmentID,
|
|||||||
return connection, nil
|
return connection, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request.
|
|
||||||
// It requires a running Poseidon instance.
|
|
||||||
func ProvideWebSocketURL(s *suite.Suite, runnerID string, request *dto.ExecutionRequest) (string, error) {
|
|
||||||
url := helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.ExecutePath)
|
|
||||||
executionRequestByteString, err := json.Marshal(request)
|
|
||||||
s.Require().NoError(err)
|
|
||||||
reader := strings.NewReader(string(executionRequestByteString))
|
|
||||||
resp, err := http.Post(url, "application/json", reader) //nolint:gosec // url is not influenced by a user
|
|
||||||
s.Require().NoError(err)
|
|
||||||
s.Require().Equal(http.StatusOK, resp.StatusCode)
|
|
||||||
|
|
||||||
executionResponse := new(dto.ExecutionResponse)
|
|
||||||
err = json.NewDecoder(resp.Body).Decode(executionResponse)
|
|
||||||
s.Require().NoError(err)
|
|
||||||
return executionResponse.WebSocketURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ConnectToWebSocket establish an external WebSocket connection to the provided url.
|
// ConnectToWebSocket establish an external WebSocket connection to the provided url.
|
||||||
// It requires a running Poseidon instance.
|
// It requires a running Poseidon instance.
|
||||||
func ConnectToWebSocket(url string) (conn *websocket.Conn, err error) {
|
func ConnectToWebSocket(url string) (conn *websocket.Conn, err error) {
|
||||||
|
136
tests/recovery/e2e_recovery_test.go
Normal file
136
tests/recovery/e2e_recovery_test.go
Normal file
@ -0,0 +1,136 @@
|
|||||||
|
package recovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
nomadApi "github.com/hashicorp/nomad/api"
|
||||||
|
"github.com/openHPI/poseidon/internal/api"
|
||||||
|
"github.com/openHPI/poseidon/internal/config"
|
||||||
|
"github.com/openHPI/poseidon/pkg/dto"
|
||||||
|
"github.com/openHPI/poseidon/pkg/logging"
|
||||||
|
"github.com/openHPI/poseidon/tests"
|
||||||
|
"github.com/openHPI/poseidon/tests/e2e"
|
||||||
|
"github.com/openHPI/poseidon/tests/helpers"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* # E2E Recovery Tests
|
||||||
|
*
|
||||||
|
* For the e2e tests a nomad cluster must be connected and poseidon must be running.
|
||||||
|
* These cases test the behavior of Poseidon when restarting / recovering.
|
||||||
|
*/
|
||||||
|
|
||||||
|
var (
|
||||||
|
log = logging.GetLogger("e2e-recovery")
|
||||||
|
testDockerImage = flag.String("dockerImage", "", "Docker image to use in E2E tests")
|
||||||
|
poseidonBinary = flag.String("poseidonPath", "", "The path to the Poseidon binary")
|
||||||
|
nomadClient *nomadApi.Client
|
||||||
|
nomadNamespace string
|
||||||
|
)
|
||||||
|
|
||||||
|
// InactivityTimeout of the created runner in seconds.
|
||||||
|
const (
|
||||||
|
InactivityTimeout = 1
|
||||||
|
PrewarmingPoolSize = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
type E2ERecoveryTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
runnerID string
|
||||||
|
poseidonCancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
// Overwrite TestMain for custom setup.
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
if err := config.InitConfig(); err != nil {
|
||||||
|
log.WithError(err).Fatal("Could not initialize configuration")
|
||||||
|
}
|
||||||
|
|
||||||
|
if *poseidonBinary == "" {
|
||||||
|
log.Fatal("You must specify the -path to the Poseidon binary!")
|
||||||
|
}
|
||||||
|
if *testDockerImage == "" {
|
||||||
|
log.Fatal("You must specify the -dockerImage flag!")
|
||||||
|
}
|
||||||
|
|
||||||
|
nomadNamespace = config.Config.Nomad.Namespace
|
||||||
|
var err error
|
||||||
|
nomadClient, err = nomadApi.NewClient(&nomadApi.Config{
|
||||||
|
Address: config.Config.Nomad.URL().String(),
|
||||||
|
TLSConfig: &nomadApi.TLSConfig{},
|
||||||
|
Namespace: nomadNamespace,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Fatal("Could not create Nomad client")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestE2ERecoveryTests(t *testing.T) {
|
||||||
|
testSuite := new(E2ERecoveryTestSuite)
|
||||||
|
|
||||||
|
ctx, cancelPoseidon := context.WithCancel(context.Background())
|
||||||
|
testSuite.poseidonCancel = cancelPoseidon
|
||||||
|
|
||||||
|
startPoseidon(ctx, cancelPoseidon)
|
||||||
|
waitForPoseidon()
|
||||||
|
|
||||||
|
e2e.CreateDefaultEnvironment(PrewarmingPoolSize, *testDockerImage)
|
||||||
|
e2e.WaitForDefaultEnvironment()
|
||||||
|
|
||||||
|
suite.Run(t, testSuite)
|
||||||
|
|
||||||
|
TearDown()
|
||||||
|
testSuite.poseidonCancel()
|
||||||
|
<-time.After(tests.ShortTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *E2ERecoveryTestSuite) TestInactivityTimer_Valid() {
|
||||||
|
_, err := e2e.ProvideWebSocketURL(s.runnerID, &dto.ExecutionRequest{Command: "true"})
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *E2ERecoveryTestSuite) TestInactivityTimer_Expired() {
|
||||||
|
<-time.After(InactivityTimeout * time.Second)
|
||||||
|
_, err := e2e.ProvideWebSocketURL(s.runnerID, &dto.ExecutionRequest{Command: "true"})
|
||||||
|
s.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// We expect the runner count to be equal to the prewarming pool size plus the one provided runner.
|
||||||
|
// If the count does not include the provided runner, the evaluation of the runner status may be wrong.
|
||||||
|
func (s *E2ERecoveryTestSuite) TestRunnerCount() {
|
||||||
|
jobListStubs, _, err := nomadClient.Jobs().List(&nomadApi.QueryOptions{
|
||||||
|
Prefix: tests.DefaultEnvironmentIDAsString,
|
||||||
|
Namespace: nomadNamespace,
|
||||||
|
})
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(PrewarmingPoolSize+1, len(jobListStubs))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *E2ERecoveryTestSuite) TestEnvironmentStatistics() {
|
||||||
|
url := helpers.BuildURL(api.BasePath, api.StatisticsPath, api.EnvironmentsPath)
|
||||||
|
response, err := http.Get(url) //nolint:gosec // The variability of this url is limited by our configurations.
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Require().Equal(http.StatusOK, response.StatusCode)
|
||||||
|
|
||||||
|
statistics := make(map[string]*dto.StatisticalExecutionEnvironmentData)
|
||||||
|
err = json.NewDecoder(response.Body).Decode(&statistics)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
err = response.Body.Close()
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
environmentStatistics, ok := statistics[tests.DefaultEnvironmentIDAsString]
|
||||||
|
s.Require().True(ok)
|
||||||
|
s.Equal(tests.DefaultEnvironmentIDAsInteger, environmentStatistics.ID)
|
||||||
|
s.Equal(uint(PrewarmingPoolSize), environmentStatistics.PrewarmingPoolSize)
|
||||||
|
s.Equal(uint(PrewarmingPoolSize), environmentStatistics.IdleRunners)
|
||||||
|
s.Equal(uint(1), environmentStatistics.UsedRunners)
|
||||||
|
}
|
63
tests/recovery/setup_test.go
Normal file
63
tests/recovery/setup_test.go
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
package recovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"github.com/openHPI/poseidon/internal/api"
|
||||||
|
"github.com/openHPI/poseidon/pkg/dto"
|
||||||
|
"github.com/openHPI/poseidon/tests"
|
||||||
|
"github.com/openHPI/poseidon/tests/e2e"
|
||||||
|
"github.com/openHPI/poseidon/tests/helpers"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *E2ERecoveryTestSuite) SetupTest() {
|
||||||
|
<-time.After(InactivityTimeout * time.Second)
|
||||||
|
// We do not want runner from the previous tests
|
||||||
|
|
||||||
|
var err error
|
||||||
|
s.runnerID, err = e2e.ProvideRunner(&dto.RunnerRequest{
|
||||||
|
ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger,
|
||||||
|
InactivityTimeout: InactivityTimeout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Fatal("Could not provide runner")
|
||||||
|
}
|
||||||
|
|
||||||
|
<-time.After(tests.ShortTimeout)
|
||||||
|
s.poseidonCancel()
|
||||||
|
<-time.After(tests.ShortTimeout)
|
||||||
|
ctx, cancelPoseidon := context.WithCancel(context.Background())
|
||||||
|
s.poseidonCancel = cancelPoseidon
|
||||||
|
startPoseidon(ctx, cancelPoseidon)
|
||||||
|
waitForPoseidon()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TearDown() {
|
||||||
|
path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString)
|
||||||
|
_, err := helpers.HTTPDelete(path, nil)
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Fatal("Could not remove default environment")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func startPoseidon(ctx context.Context, cancelPoseidon context.CancelFunc) {
|
||||||
|
poseidon := exec.CommandContext(ctx, *poseidonBinary) //nolint:gosec // We accept that another binary can be executed.
|
||||||
|
poseidon.Stdout = os.Stdout
|
||||||
|
poseidon.Stderr = os.Stderr
|
||||||
|
if err := poseidon.Start(); err != nil {
|
||||||
|
cancelPoseidon()
|
||||||
|
log.WithError(err).Fatal("Failed to start Poseidon")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForPoseidon() {
|
||||||
|
done := false
|
||||||
|
for !done {
|
||||||
|
<-time.After(time.Second)
|
||||||
|
resp, err := http.Get(helpers.BuildURL(api.BasePath, api.HealthPath))
|
||||||
|
done = err == nil && resp.StatusCode == http.StatusNoContent
|
||||||
|
}
|
||||||
|
}
|
Reference in New Issue
Block a user