From dba7160a41a9ca5e01c43fb8d2dd67a6fbd283a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Pa=C3=9F?= Date: Thu, 6 May 2021 13:26:55 +0200 Subject: [PATCH] Implement merge request review comments --- .gitlab-ci.yml | 52 +++++++++++++++++++----------- ci/Dockerfile | 23 ++++++------- ci/python.nomad | 50 ++++++++++++++++++++++++++++ e2e/e2e_tests.go | 18 ----------- e2e_tests/e2e_test.go | 36 +++++++++++++++++++++ {e2e => e2e_tests}/health_test.go | 8 ++--- {e2e => e2e_tests}/runners_test.go | 2 +- main_test.go | 32 ------------------ mocks/ExecutorApi.go | 4 +-- nomad/nomad.go | 23 ------------- 10 files changed, 136 insertions(+), 112 deletions(-) create mode 100644 ci/python.nomad delete mode 100644 e2e/e2e_tests.go create mode 100644 e2e_tests/e2e_test.go rename {e2e => e2e_tests}/health_test.go (55%) rename {e2e => e2e_tests}/runners_test.go (97%) delete mode 100644 main_test.go diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2bd7301..9c8e92f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ stages: - lint - test - docker + - e2e - deploy - cleanup @@ -48,7 +49,7 @@ test: stage: test needs: [] script: - - go test $(go list ./... | grep -v /e2e | grep -v /poseidon$) -v -coverprofile coverage.cov + - go test $(go list ./... | grep -v /e2e_tests) -v -coverprofile coverage.cov - go tool cover -func=coverage.cov - go tool cover -html=coverage.cov -o coverage_unit.html artifacts: @@ -57,24 +58,6 @@ test: expire_in: 1 week expose_as: coverageReport -e2e_test: - image: drp.codemoon.xopic.de/nomad-ci:latest - stage: test - services: - - name: docker:dind - alias: docker - script: - # Avoid docker pull limit - - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY - - docker pull $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 - - docker tag $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 openhpi/co_execenv_python:3.8 - # Setup own nomad cluster - - export NOMAD_ADDR=http://localhost:4646 - - nomad agent -dev -log-level=WARN & - - sleep 15 - # Start tests - - go test ./ ./e2e -v - dockerimage: stage: docker image: docker:latest @@ -111,6 +94,37 @@ nomadimage: - docker push $NOMAD_CI_IMAGE_NAME_ENV - docker push $NOMAD_CI_IMAGE_NAME_GENERAL + +test_e2e: + image: $NOMAD_CI_IMAGE_NAME_ENV + stage: e2e + needs: + - compile + - job: nomadimage + services: + - name: docker:dind + alias: docker + variables: + DOCKER_HOST: "tcp://docker:2375" + script: + # Avoid docker pull limit + - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY + - docker pull $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 + - docker tag $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 openhpi/co_execenv_python:3.8 + # Setup own Nomad cluster and wait for startup + - export NOMAD_ADDR=http://localhost:4646 + - nomad agent -dev -log-level=WARN & + - sleep 5 + # Setup Nomad job and wait for it + - nomad run ci/python.nomad + - sleep 5 + # Start Poseidon and wait for it + - ./poseidon & + - sleep 20 + # Start tests + - go test ./e2e_tests -v + + .start_deployment: &start_deployment image: $NOMAD_CI_IMAGE_NAME_ENV stage: deploy diff --git a/ci/Dockerfile b/ci/Dockerfile index 33ffd28..75fbc01 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -4,22 +4,25 @@ FROM golang:latest # Install prerequisites, gettext contains envsubst used in the CI RUN apt-get update && \ - apt install -y unzip wget gettext && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists && \ - apt-get install -y \ + apt install -y \ + unzip \ + wget \ + gettext \ apt-transport-https \ ca-certificates \ curl \ - gnupg + gnupg \ + lsb-release && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists -RUN apt-get update && apt-get install -y lsb-release && apt-get clean all RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg RUN echo \ "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null RUN apt-get update && \ - apt-get install -y docker-ce docker-ce-cli containerd.io + apt-get install -y docker-ce docker-ce-cli containerd.io && \ + rm -rf /var/lib/apt/lists # Download Nomad RUN wget "https://releases.hashicorp.com/nomad/1.0.4/nomad_1.0.4_linux_amd64.zip" && \ @@ -32,9 +35,3 @@ RUN mv nomad /usr/sbin/ && nomad -version COPY nomad-run-and-wait /usr/sbin/ RUN chmod +x /usr/sbin/nomad-run-and-wait - -RUN wget "https://raw.githubusercontent.com/docker-library/docker/f6a0c427f0354dcf5870c430c72c8f1d6b4e6d5e/20.10/docker-entrypoint.sh" -RUN mv ./docker-entrypoint.sh /usr/local/bin/ -RUN chmod +x /usr/local/bin/docker-entrypoint.sh -ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["sh"] diff --git a/ci/python.nomad b/ci/python.nomad new file mode 100644 index 0000000..e169f7b --- /dev/null +++ b/ci/python.nomad @@ -0,0 +1,50 @@ +// ToDo: Delete when creating job on test startup #26. +job "python" { + datacenters = ["dc1"] + type = "batch" + + group "python" { + ephemeral_disk { + migrate = false + size = 10 + sticky = false + } + count = 1 + scaling { + enabled = true + max = 300 + } + spread { + // see https://www.nomadproject.io/docs/job-specification/spread#even-spread-across-data-center + // This spreads the load evenly amongst our nodes + attribute = "${node.unique.name}" + weight = 100 + } + + task "python" { + driver = "docker" + kill_timeout = "0s" + kill_signal = "SIGKILL" + + config { + image = "openhpi/co_execenv_python:3.8" + command = "sleep" + args = ["infinity"] + } + + logs { + max_files = 1 + max_file_size = 1 + } + + resources { + cpu = 40 + memory = 40 + } + + restart { + delay = "0s" + } + } + } +} diff --git a/e2e/e2e_tests.go b/e2e/e2e_tests.go deleted file mode 100644 index de815d8..0000000 --- a/e2e/e2e_tests.go +++ /dev/null @@ -1,18 +0,0 @@ -package e2e - -import ( - "gitlab.hpi.de/codeocean/codemoon/poseidon/api" - "gitlab.hpi.de/codeocean/codemoon/poseidon/config" - "strings" -) - -/* -* # E2E Tests -* -* For the e2e tests a nomad cluster must be connected and poseidon must be running. - */ - -func buildURL(parts ...string) (url string) { - parts = append([]string{config.Config.PoseidonAPIURL().String(), api.RouteBase}, parts...) - return strings.Join(parts, "") -} diff --git a/e2e_tests/e2e_test.go b/e2e_tests/e2e_test.go new file mode 100644 index 0000000..5b2c6f9 --- /dev/null +++ b/e2e_tests/e2e_test.go @@ -0,0 +1,36 @@ +package e2e_tests + +import ( + "gitlab.hpi.de/codeocean/codemoon/poseidon/api" + "gitlab.hpi.de/codeocean/codemoon/poseidon/config" + "gitlab.hpi.de/codeocean/codemoon/poseidon/logging" + "os" + "strings" + "testing" +) + +/* +* # E2E Tests +* +* For the e2e tests a nomad cluster must be connected and poseidon must be running. + */ + +var log = logging.GetLogger("e2e_tests") + +// Overwrite TestMain for custom setup. +func TestMain(m *testing.M) { + log.Info("Test Setup") + err := config.InitConfig() + if err != nil { + log.Warn("Could not initialize configuration") + } + // ToDo: Add Nomad job here when it is possible to create execution environments. See #26. + log.Info("Test Run") + code := m.Run() + os.Exit(code) +} + +func buildURL(parts ...string) (url string) { + parts = append([]string{config.Config.PoseidonAPIURL().String(), api.RouteBase}, parts...) + return strings.Join(parts, "") +} diff --git a/e2e/health_test.go b/e2e_tests/health_test.go similarity index 55% rename from e2e/health_test.go rename to e2e_tests/health_test.go index 827281c..c720744 100644 --- a/e2e/health_test.go +++ b/e2e_tests/health_test.go @@ -1,8 +1,7 @@ -package e2e +package e2e_tests import ( "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "gitlab.hpi.de/codeocean/codemoon/poseidon/api" "net/http" "testing" @@ -10,6 +9,7 @@ import ( func TestHealthRoute(t *testing.T) { resp, err := http.Get(buildURL(api.RouteHealth)) - require.NoError(t, err) - assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent") + if assert.NoError(t, err) { + assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent") + } } diff --git a/e2e/runners_test.go b/e2e_tests/runners_test.go similarity index 97% rename from e2e/runners_test.go rename to e2e_tests/runners_test.go index 5d6b570..507aeeb 100644 --- a/e2e/runners_test.go +++ b/e2e_tests/runners_test.go @@ -1,4 +1,4 @@ -package e2e +package e2e_tests import ( "encoding/json" diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 07455db..0000000 --- a/main_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "gitlab.hpi.de/codeocean/codemoon/poseidon/config" - "gitlab.hpi.de/codeocean/codemoon/poseidon/nomad" - "os" - "testing" - "time" -) - -// overwrite TestMain for custom setup -func TestMain(m *testing.M) { - log.Print("Test Setup") - setup() - log.Print("Test Run") - code := m.Run() - os.Exit(code) -} - -func setup() { - createNomadJob() - go main() - time.Sleep(15 * time.Second) -} - -func createNomadJob() { - nomadAPIClient, err := nomad.New(config.Config.NomadAPIURL()) - if err != nil { - log.WithError(err).Fatal("[Test] Can not parse nomad config") - } - nomadAPIClient.CreateDebugJob() -} diff --git a/mocks/ExecutorApi.go b/mocks/ExecutorApi.go index 8282854..1e2283a 100644 --- a/mocks/ExecutorApi.go +++ b/mocks/ExecutorApi.go @@ -12,8 +12,8 @@ type ExecutorApi struct { mock.Mock } -// CreateDebugJob provides a mock function with given fields: -func (_m *ExecutorApi) CreateDebugJob() { +// CreateJob provides a mock function with given fields: +func (_m *ExecutorApi) CreateJob() { _m.Called() } diff --git a/nomad/nomad.go b/nomad/nomad.go index b8ba8da..9297e91 100644 --- a/nomad/nomad.go +++ b/nomad/nomad.go @@ -2,19 +2,15 @@ package nomad import ( nomadApi "github.com/hashicorp/nomad/api" - "gitlab.hpi.de/codeocean/codemoon/poseidon/logging" "net/url" ) -var log = logging.GetLogger("nomad") - // ExecutorApi provides access to an container orchestration solution type ExecutorApi interface { LoadJobList() (list []*nomadApi.JobListStub, err error) GetJobScale(jobId string) (jobScale int, err error) SetJobScaling(jobId string, count int, reason string) (err error) LoadRunners(jobId string) (runnerIds []string, err error) - CreateDebugJob() } // ApiClient provides access to the Nomad functionality @@ -45,25 +41,6 @@ func (apiClient *ApiClient) LoadJobList() (list []*nomadApi.JobListStub, err err return } -// CreateDebugJob creates a simple python job in the nomad cluster -func (apiClient *ApiClient) CreateDebugJob() { - job := nomadApi.NewBatchJob("python", "python", "global", 50) - job.AddDatacenter("dc1") - group := nomadApi.NewTaskGroup("python", 5) - task := nomadApi.NewTask("python", "docker") - task.SetConfig("image", "openhpi/co_execenv_python:3.8") - task.SetConfig("command", "sleep") - task.SetConfig("args", []string{"infinity"}) - group.AddTask(task) - job.AddTaskGroup(group) - register, w, err := apiClient.client.Jobs().Register(job, nil) - log.Printf("response: %+v", register) - log.Printf("meta: %+v", w) - if err != nil { - log.WithError(err).Fatal("Error creating nomad job") - } -} - // GetJobScale returns the scale of the passed job. func (apiClient *ApiClient) GetJobScale(jobId string) (jobScale int, err error) { status, _, err := apiClient.client.Jobs().ScaleStatus(jobId, nil)