diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba4cf0a..2bd7301 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,15 +48,33 @@ test: stage: test needs: [] script: - - go test $(go list ./... | grep -v /e2e) -v -coverprofile coverage.cov + - go test $(go list ./... | grep -v /e2e | grep -v /poseidon$) -v -coverprofile coverage.cov - go tool cover -func=coverage.cov - - go tool cover -html=coverage.cov -o coverage.html + - go tool cover -html=coverage.cov -o coverage_unit.html artifacts: paths: - - coverage.html + - coverage_unit.html expire_in: 1 week expose_as: coverageReport +e2e_test: + image: drp.codemoon.xopic.de/nomad-ci:latest + stage: test + services: + - name: docker:dind + alias: docker + script: + # Avoid docker pull limit + - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY + - docker pull $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 + - docker tag $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 openhpi/co_execenv_python:3.8 + # Setup own nomad cluster + - export NOMAD_ADDR=http://localhost:4646 + - nomad agent -dev -log-level=WARN & + - sleep 15 + # Start tests + - go test ./ ./e2e -v + dockerimage: stage: docker image: docker:latest @@ -85,8 +103,8 @@ nomadimage: script: - cd ci - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY - - docker pull $DOCKER_REGISTRY/library/debian:buster-slim - - docker tag $DOCKER_REGISTRY/library/debian:buster-slim debian:buster-slim + - docker pull $DOCKER_REGISTRY/library/golang:latest + - docker tag $DOCKER_REGISTRY/library/golang:latest golang:latest # Pull base image to avoid rebuilding every pipeline if nothing changed, prioritize image from branch - docker pull $NOMAD_CI_IMAGE_NAME_GENERAL || docker pull $NOMAD_CI_BASE_IMAGE || true - docker build --cache-from $NOMAD_CI_BASE_IMAGE --cache-from $NOMAD_CI_IMAGE_NAME_GENERAL -t $NOMAD_CI_IMAGE_NAME_ENV -t $NOMAD_CI_IMAGE_NAME_GENERAL . diff --git a/api/runners.go b/api/runners.go index cbcfcb6..c50ac5c 100644 --- a/api/runners.go +++ b/api/runners.go @@ -27,12 +27,12 @@ func provideRunner(writer http.ResponseWriter, request *http.Request) { if err := parseJSONRequestBody(writer, request, runnerRequest); err != nil { return } - environment, err := environment.GetExecutionEnvironment(runnerRequest.ExecutionEnvironmentId) + env, err := environment.GetExecutionEnvironment(runnerRequest.ExecutionEnvironmentId) if err != nil { writeNotFound(writer, err) return } - runner, err := environment.NextRunner() + runner, err := env.NextRunner() if err != nil { writeInternalServerError(writer, err, dto.ErrorNomadOverload) return diff --git a/ci/Dockerfile b/ci/Dockerfile index 2ec27e5..33ffd28 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,12 +1,25 @@ # Simple image containing the Nomad binary to deploy Nomad jobs -FROM debian:buster-slim +FROM golang:latest # Install prerequisites, gettext contains envsubst used in the CI RUN apt-get update && \ apt install -y unzip wget gettext && \ apt-get clean && \ - rm -rf /var/lib/apt/lists + rm -rf /var/lib/apt/lists && \ + apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg + +RUN apt-get update && apt-get install -y lsb-release && apt-get clean all +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +RUN echo \ + "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null +RUN apt-get update && \ + apt-get install -y docker-ce docker-ce-cli containerd.io # Download Nomad RUN wget "https://releases.hashicorp.com/nomad/1.0.4/nomad_1.0.4_linux_amd64.zip" && \ @@ -19,3 +32,9 @@ RUN mv nomad /usr/sbin/ && nomad -version COPY nomad-run-and-wait /usr/sbin/ RUN chmod +x /usr/sbin/nomad-run-and-wait + +RUN wget "https://raw.githubusercontent.com/docker-library/docker/f6a0c427f0354dcf5870c430c72c8f1d6b4e6d5e/20.10/docker-entrypoint.sh" +RUN mv ./docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["sh"] diff --git a/config/config.go b/config/config.go index 516bf5a..1438389 100644 --- a/config/config.go +++ b/config/config.go @@ -26,7 +26,7 @@ var ( KeyFile: "", }, Nomad: nomad{ - Address: "", + Address: "127.0.0.1", Port: 4646, Token: "", TLS: false, diff --git a/e2e/e2e_tests.go b/e2e/e2e_tests.go index e38f023..de815d8 100644 --- a/e2e/e2e_tests.go +++ b/e2e/e2e_tests.go @@ -12,9 +12,7 @@ import ( * For the e2e tests a nomad cluster must be connected and poseidon must be running. */ -var baseURL = config.Config.PoseidonAPIURL().String() - func buildURL(parts ...string) (url string) { - parts = append([]string{baseURL, api.RouteBase}, parts...) + parts = append([]string{config.Config.PoseidonAPIURL().String(), api.RouteBase}, parts...) return strings.Join(parts, "") } diff --git a/e2e/health_test.go b/e2e/health_test.go index 5ce45af..827281c 100644 --- a/e2e/health_test.go +++ b/e2e/health_test.go @@ -2,6 +2,7 @@ package e2e import ( "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gitlab.hpi.de/codeocean/codemoon/poseidon/api" "net/http" "testing" @@ -9,6 +10,6 @@ import ( func TestHealthRoute(t *testing.T) { resp, err := http.Get(buildURL(api.RouteHealth)) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent") } diff --git a/go.mod b/go.mod index d58fe09..cfa7830 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,12 @@ require ( github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/cronexpr v1.1.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f + github.com/hashicorp/nomad/api v0.0.0-20210505182403-7d5a9ecde95c github.com/kr/text v0.2.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/sirupsen/logrus v1.8.1 github.com/stretchr/objx v0.1.1 // indirect github.com/stretchr/testify v1.7.0 - golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect + golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/go.sum b/go.sum index 1509de4..6d9b3b4 100644 --- a/go.sum +++ b/go.sum @@ -34,10 +34,8 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f h1:XaaK4F+pOV2ZYk3aCIDAZThm3voKD6nyOUNeUY3yHgY= -github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f h1:XaaK4F+pOV2ZYk3aCIDAZThm3voKD6nyOUNeUY3yHgY= -github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f/go.mod h1:vYHP9jMXk4/T2qNUbWlQ1OHCA1hHLil3nvqSmz8mtgc= -github.com/hashicorp/nomad/api v0.0.0-20210430020956-28b8767b278f/go.mod h1:vYHP9jMXk4/T2qNUbWlQ1OHCA1hHLil3nvqSmz8mtgc= +github.com/hashicorp/nomad/api v0.0.0-20210505182403-7d5a9ecde95c h1:b0V392CjnzubekagqWqgxG4YCQQi3lf74yEijJLmyE8= +github.com/hashicorp/nomad/api v0.0.0-20210505182403-7d5a9ecde95c/go.mod h1:vYHP9jMXk4/T2qNUbWlQ1OHCA1hHLil3nvqSmz8mtgc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -66,8 +64,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c= +golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= diff --git a/main.go b/main.go index a5ffdcf..46d31c4 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "os/signal" + "syscall" "time" ) @@ -53,7 +54,7 @@ func initServer(runnerPool pool.RunnerPool) *http.Server { func shutdownOnOSSignal(server *http.Server) { // wait for SIGINT signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) <-signals log.Info("Received SIGINT, shutting down ...") diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..07455db --- /dev/null +++ b/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "gitlab.hpi.de/codeocean/codemoon/poseidon/config" + "gitlab.hpi.de/codeocean/codemoon/poseidon/nomad" + "os" + "testing" + "time" +) + +// overwrite TestMain for custom setup +func TestMain(m *testing.M) { + log.Print("Test Setup") + setup() + log.Print("Test Run") + code := m.Run() + os.Exit(code) +} + +func setup() { + createNomadJob() + go main() + time.Sleep(15 * time.Second) +} + +func createNomadJob() { + nomadAPIClient, err := nomad.New(config.Config.NomadAPIURL()) + if err != nil { + log.WithError(err).Fatal("[Test] Can not parse nomad config") + } + nomadAPIClient.CreateDebugJob() +} diff --git a/mocks/ExecutorApi.go b/mocks/ExecutorApi.go index 8bc5b56..8282854 100644 --- a/mocks/ExecutorApi.go +++ b/mocks/ExecutorApi.go @@ -12,6 +12,11 @@ type ExecutorApi struct { mock.Mock } +// CreateDebugJob provides a mock function with given fields: +func (_m *ExecutorApi) CreateDebugJob() { + _m.Called() +} + // GetJobScale provides a mock function with given fields: jobId func (_m *ExecutorApi) GetJobScale(jobId string) (int, error) { ret := _m.Called(jobId) diff --git a/nomad/nomad.go b/nomad/nomad.go index 9297e91..b8ba8da 100644 --- a/nomad/nomad.go +++ b/nomad/nomad.go @@ -2,15 +2,19 @@ package nomad import ( nomadApi "github.com/hashicorp/nomad/api" + "gitlab.hpi.de/codeocean/codemoon/poseidon/logging" "net/url" ) +var log = logging.GetLogger("nomad") + // ExecutorApi provides access to an container orchestration solution type ExecutorApi interface { LoadJobList() (list []*nomadApi.JobListStub, err error) GetJobScale(jobId string) (jobScale int, err error) SetJobScaling(jobId string, count int, reason string) (err error) LoadRunners(jobId string) (runnerIds []string, err error) + CreateDebugJob() } // ApiClient provides access to the Nomad functionality @@ -41,6 +45,25 @@ func (apiClient *ApiClient) LoadJobList() (list []*nomadApi.JobListStub, err err return } +// CreateDebugJob creates a simple python job in the nomad cluster +func (apiClient *ApiClient) CreateDebugJob() { + job := nomadApi.NewBatchJob("python", "python", "global", 50) + job.AddDatacenter("dc1") + group := nomadApi.NewTaskGroup("python", 5) + task := nomadApi.NewTask("python", "docker") + task.SetConfig("image", "openhpi/co_execenv_python:3.8") + task.SetConfig("command", "sleep") + task.SetConfig("args", []string{"infinity"}) + group.AddTask(task) + job.AddTaskGroup(group) + register, w, err := apiClient.client.Jobs().Register(job, nil) + log.Printf("response: %+v", register) + log.Printf("meta: %+v", w) + if err != nil { + log.WithError(err).Fatal("Error creating nomad job") + } +} + // GetJobScale returns the scale of the passed job. func (apiClient *ApiClient) GetJobScale(jobId string) (jobScale int, err error) { status, _, err := apiClient.client.Jobs().ScaleStatus(jobId, nil)