Add e2e tests in ci stage

2021-04-28 13:42:33 +02:00
parent 77f84ccf56
commit 0dc4d28c1b
12 changed files with 118 additions and 23 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -48,15 +48,33 @@ test:
  stage: test
  needs: []
  script:
-    - go test $(go list ./... | grep -v /e2e) -v -coverprofile coverage.cov
+    - go test $(go list ./... | grep -v /e2e | grep -v /poseidon$) -v -coverprofile coverage.cov
    - go tool cover -func=coverage.cov
-    - go tool cover -html=coverage.cov -o coverage.html
+    - go tool cover -html=coverage.cov -o coverage_unit.html
  artifacts:
    paths:
-      - coverage.html
+      - coverage_unit.html
    expire_in: 1 week
    expose_as: coverageReport

+e2e_test:
+  image: drp.codemoon.xopic.de/nomad-ci:latest
+  stage: test
+  services:
+    - name: docker:dind
+      alias: docker
+  script:
+    # Avoid docker pull limit
+    - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY
+    - docker pull $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8
+    - docker tag $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 openhpi/co_execenv_python:3.8
+    # Setup own nomad cluster
+    - export NOMAD_ADDR=http://localhost:4646
+    - nomad agent -dev -log-level=WARN &
+    - sleep 15
+    # Start tests
+    - go test ./ ./e2e -v
+
 dockerimage:
  stage: docker
  image: docker:latest
@ -85,8 +103,8 @@ nomadimage:
  script:
    - cd ci
    - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY
-    - docker pull $DOCKER_REGISTRY/library/debian:buster-slim
-    - docker tag $DOCKER_REGISTRY/library/debian:buster-slim debian:buster-slim
+    - docker pull $DOCKER_REGISTRY/library/golang:latest
+    - docker tag $DOCKER_REGISTRY/library/golang:latest golang:latest
    # Pull base image to avoid rebuilding every pipeline if nothing changed, prioritize image from branch
    - docker pull $NOMAD_CI_IMAGE_NAME_GENERAL || docker pull $NOMAD_CI_BASE_IMAGE || true
    - docker build --cache-from $NOMAD_CI_BASE_IMAGE --cache-from $NOMAD_CI_IMAGE_NAME_GENERAL -t $NOMAD_CI_IMAGE_NAME_ENV -t $NOMAD_CI_IMAGE_NAME_GENERAL .
--- a/api/runners.go
+++ b/api/runners.go
@ -27,12 +27,12 @@ func provideRunner(writer http.ResponseWriter, request *http.Request) {
 	if err := parseJSONRequestBody(writer, request, runnerRequest); err != nil {
 		return
 	}
-	environment, err := environment.GetExecutionEnvironment(runnerRequest.ExecutionEnvironmentId)
+	env, err := environment.GetExecutionEnvironment(runnerRequest.ExecutionEnvironmentId)
 	if err != nil {
 		writeNotFound(writer, err)
 		return
 	}
-	runner, err := environment.NextRunner()
+	runner, err := env.NextRunner()
 	if err != nil {
 		writeInternalServerError(writer, err, dto.ErrorNomadOverload)
 		return
--- a/ci/Dockerfile
+++ b/ci/Dockerfile
@ -1,12 +1,25 @@
 # Simple image containing the Nomad binary to deploy Nomad jobs

-FROM debian:buster-slim
+FROM golang:latest

 # Install prerequisites, gettext contains envsubst used in the CI
 RUN apt-get update && \
    apt install -y unzip wget gettext && \
    apt-get clean && \
-    rm -rf /var/lib/apt/lists
+    rm -rf /var/lib/apt/lists && \
+    apt-get install -y \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        gnupg
+
+RUN apt-get update && apt-get install -y lsb-release && apt-get clean all
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+RUN echo \
+      "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
+      $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+RUN apt-get update && \
+    apt-get install -y docker-ce docker-ce-cli containerd.io

 # Download Nomad
 RUN wget "https://releases.hashicorp.com/nomad/1.0.4/nomad_1.0.4_linux_amd64.zip" && \
@ -19,3 +32,9 @@ RUN mv nomad /usr/sbin/ && nomad -version

 COPY nomad-run-and-wait /usr/sbin/
 RUN chmod +x /usr/sbin/nomad-run-and-wait
+
+RUN wget "https://raw.githubusercontent.com/docker-library/docker/f6a0c427f0354dcf5870c430c72c8f1d6b4e6d5e/20.10/docker-entrypoint.sh"
+RUN mv ./docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["sh"]
--- a/config/config.go
+++ b/config/config.go
@ -26,7 +26,7 @@ var (
 			KeyFile:  "",
 		},
 		Nomad: nomad{
-			Address: "",
+			Address: "127.0.0.1",
 			Port:    4646,
 			Token:   "",
 			TLS:     false,
--- a/e2e/e2e_tests.go
+++ b/e2e/e2e_tests.go
@ -12,9 +12,7 @@ import (
 * For the e2e tests a nomad cluster must be connected and poseidon must be running.
 */

-var baseURL = config.Config.PoseidonAPIURL().String()
-
 func buildURL(parts ...string) (url string) {
-	parts = append([]string{baseURL, api.RouteBase}, parts...)
+	parts = append([]string{config.Config.PoseidonAPIURL().String(), api.RouteBase}, parts...)
 	return strings.Join(parts, "")
 }
--- a/e2e/health_test.go
+++ b/e2e/health_test.go
@ -2,6 +2,7 @@ package e2e

 import (
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"gitlab.hpi.de/codeocean/codemoon/poseidon/api"
 	"net/http"
 	"testing"
@ -9,6 +10,6 @@ import (

 func TestHealthRoute(t *testing.T) {
 	resp, err := http.Get(buildURL(api.RouteHealth))
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent")
 }
--- a/go.mod
+++ b/go.mod
@ -8,12 +8,12 @@ require (
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/hashicorp/cronexpr v1.1.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f
+	github.com/hashicorp/nomad/api v0.0.0-20210505182403-7d5a9ecde95c
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/objx v0.1.1 // indirect
 	github.com/stretchr/testify v1.7.0
-	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
+	golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
--- a/go.sum
+++ b/go.sum
@ -34,10 +34,8 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
-github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f h1:XaaK4F+pOV2ZYk3aCIDAZThm3voKD6nyOUNeUY3yHgY=
-github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f h1:XaaK4F+pOV2ZYk3aCIDAZThm3voKD6nyOUNeUY3yHgY=
-github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f/go.mod h1:vYHP9jMXk4/T2qNUbWlQ1OHCA1hHLil3nvqSmz8mtgc=
-github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f/go.mod h1:vYHP9jMXk4/T2qNUbWlQ1OHCA1hHLil3nvqSmz8mtgc=
+github.com/hashicorp/nomad/api v0.0.0-20210505182403-7d5a9ecde95c h1:b0V392CjnzubekagqWqgxG4YCQQi3lf74yEijJLmyE8=
+github.com/hashicorp/nomad/api v0.0.0-20210505182403-7d5a9ecde95c/go.mod h1:vYHP9jMXk4/T2qNUbWlQ1OHCA1hHLil3nvqSmz8mtgc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -66,8 +64,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c=
+golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
--- a/main.go
+++ b/main.go
@ -11,6 +11,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"syscall"
 	"time"
 )

@ -53,7 +54,7 @@ func initServer(runnerPool pool.RunnerPool) *http.Server {
 func shutdownOnOSSignal(server *http.Server) {
 	// wait for SIGINT
 	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt)
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 	<-signals
 	log.Info("Received SIGINT, shutting down ...")

--- a/main_test.go
+++ b/main_test.go
@ -0,0 +1,32 @@
+package main
+
+import (
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/config"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/nomad"
+	"os"
+	"testing"
+	"time"
+)
+
+// overwrite TestMain for custom setup
+func TestMain(m *testing.M) {
+	log.Print("Test Setup")
+	setup()
+	log.Print("Test Run")
+	code := m.Run()
+	os.Exit(code)
+}
+
+func setup() {
+	createNomadJob()
+	go main()
+	time.Sleep(15 * time.Second)
+}
+
+func createNomadJob() {
+	nomadAPIClient, err := nomad.New(config.Config.NomadAPIURL())
+	if err != nil {
+		log.WithError(err).Fatal("[Test] Can not parse nomad config")
+	}
+	nomadAPIClient.CreateDebugJob()
+}
--- a/mocks/ExecutorApi.go
+++ b/mocks/ExecutorApi.go
@ -12,6 +12,11 @@ type ExecutorApi struct {
 	mock.Mock
 }

+// CreateDebugJob provides a mock function with given fields:
+func (_m *ExecutorApi) CreateDebugJob() {
+	_m.Called()
+}
+
 // GetJobScale provides a mock function with given fields: jobId
 func (_m *ExecutorApi) GetJobScale(jobId string) (int, error) {
 	ret := _m.Called(jobId)
--- a/nomad/nomad.go
+++ b/nomad/nomad.go
@ -2,15 +2,19 @@ package nomad

 import (
 	nomadApi "github.com/hashicorp/nomad/api"
+	"gitlab.hpi.de/codeocean/codemoon/poseidon/logging"
 	"net/url"
 )

+var log = logging.GetLogger("nomad")
+
 // ExecutorApi provides access to an container orchestration solution
 type ExecutorApi interface {
 	LoadJobList() (list []*nomadApi.JobListStub, err error)
 	GetJobScale(jobId string) (jobScale int, err error)
 	SetJobScaling(jobId string, count int, reason string) (err error)
 	LoadRunners(jobId string) (runnerIds []string, err error)
+	CreateDebugJob()
 }

 // ApiClient provides access to the Nomad functionality
@ -41,6 +45,25 @@ func (apiClient *ApiClient) LoadJobList() (list []*nomadApi.JobListStub, err err
 	return
 }

+// CreateDebugJob creates a simple python job in the nomad cluster
+func (apiClient *ApiClient) CreateDebugJob() {
+	job := nomadApi.NewBatchJob("python", "python", "global", 50)
+	job.AddDatacenter("dc1")
+	group := nomadApi.NewTaskGroup("python", 5)
+	task := nomadApi.NewTask("python", "docker")
+	task.SetConfig("image", "openhpi/co_execenv_python:3.8")
+	task.SetConfig("command", "sleep")
+	task.SetConfig("args", []string{"infinity"})
+	group.AddTask(task)
+	job.AddTaskGroup(group)
+	register, w, err := apiClient.client.Jobs().Register(job, nil)
+	log.Printf("response: %+v", register)
+	log.Printf("meta: %+v", w)
+	if err != nil {
+		log.WithError(err).Fatal("Error creating nomad job")
+	}
+}
+
 // GetJobScale returns the scale of the passed job.
 func (apiClient *ApiClient) GetJobScale(jobId string) (jobScale int, err error) {
 	status, _, err := apiClient.client.Jobs().ScaleStatus(jobId, nil)