diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d2c5102 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,135 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + NOMAD_VERSION: 1.1.2 + +jobs: + compile: + runs-on: ubuntu-latest + env: + CGO_ENABLED: 0 + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Build + run: make build + - name: Upload Poseidon binary + uses: actions/upload-artifact@v2 + with: + name: poseidon + path: poseidon + + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: golangci-lint + uses: golangci/golangci-lint-action@v2.5.2 + with: + version: latest + + test: + runs-on: ubuntu-latest + needs: [ compile ] + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Run tests + run: make coverhtml + - name: Upload coverage report + uses: actions/upload-artifact@v2 + with: + name: coverage + path: coverage_unit.html + + dep-scan: + runs-on: ubuntu-latest + needs: [ compile ] + steps: + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + severity: 'HIGH,CRITICAL' + exit-code: '1' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + + e2e-test: + runs-on: ubuntu-latest + needs: [ compile, dep-scan, test ] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - name: Cache Go modules + uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Cache Nomad binary + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/nomad + key: ${{ runner.os }}-nomad-${{ env.NOMAD_VERSION }} + restore-keys: | + ${{ runner.os }}-nomad-${{ env.NOMAD_VERSION }} + - name: Download Nomad binary + run: | + if [[ -f ./nomad ]]; then exit 0; fi + wget -q "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip" + wget -q "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_SHA256SUMS" + grep "nomad_${NOMAD_VERSION}_linux_amd64.zip" nomad_${NOMAD_VERSION}_SHA256SUMS | sha256sum -c - + unzip nomad_${NOMAD_VERSION}_linux_amd64.zip + - name: Download Poseidon binary + uses: actions/download-artifact@v2 + with: + name: poseidon + - name: Run e2e tests + run: | + sudo ./nomad agent -dev -log-level=WARN & + until curl -s --fail http://localhost:4646/v1/agent/health ; do sleep 1; done + chmod +x ./poseidon + ./poseidon & + make e2e-test diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..576bc6f --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,53 @@ +name: Create and publish Poseidon image + +on: + push: + branches: [ main ] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Build + run: make build + - name: Log in to the Container registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: deploy/poseidon/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 8f73d7a..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,227 +0,0 @@ -default: - image: golang:latest - -stages: - - build - - lint - - test - - docker - - e2e - - deploy - - cleanup - -variables: - DOCKER_TLS_CERTDIR: "" - NOMAD_SLUG: $NOMAD_PREFIX-$CI_ENVIRONMENT_SLUG - IMAGE_NAME_ENV: $DOCKER_REGISTRY/$DOCKER_IMAGE_NAME/$CI_COMMIT_REF_SLUG:$CI_PIPELINE_IID - IMAGE_NAME_GENERAL: $DOCKER_REGISTRY/$DOCKER_IMAGE_NAME/$CI_COMMIT_REF_SLUG:latest - NOMAD_CI_IMAGE_NAME_ENV: $DOCKER_REGISTRY/nomad-ci/$CI_COMMIT_REF_SLUG:$CI_PIPELINE_IID - NOMAD_CI_IMAGE_NAME_GENERAL: $DOCKER_REGISTRY/nomad-ci/$CI_COMMIT_REF_SLUG:latest - NOMAD_CI_BASE_IMAGE: $DOCKER_REGISTRY/nomad-ci/main:latest - -compile: - stage: build - needs: [] - variables: - CGO_ENABLED: 0 - script: - - make build - artifacts: - paths: - - poseidon - expire_in: 1 week - -golangci-lint: - stage: lint - needs: [] - image: golangci/golangci-lint:latest - script: - - make golangci-lint - -test: - stage: test - needs: [] - script: - - make coverhtml - artifacts: - paths: - - coverage_unit.html - expire_in: 1 week - expose_as: coverageReport - -dep-scan: - stage: test - needs: - - compile - script: - - make trivy-scan-deps - artifacts: - reports: - dependency_scanning: .trivy/gl-scanning-report.json - cache: - paths: - - .trivy/.trivycache/ - -dockerimage: - stage: docker - image: $DOCKER_REGISTRY/docker-make:latest - services: - - name: docker:dind - alias: docker - needs: - - compile - - dep-scan - - test - script: - - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY - # Prevent pull rate limit but still have normal alpine image in Dockerfile - - docker pull $DOCKER_REGISTRY/library/alpine:latest - - docker tag $DOCKER_REGISTRY/library/alpine:latest alpine:latest - - docker build -t $IMAGE_NAME_ENV -f deploy/poseidon/Dockerfile . - # Run vulnerability scan before pushing the image - - make trivy-scan-docker DOCKER_TAG=$IMAGE_NAME_ENV - - docker push $IMAGE_NAME_ENV - - docker tag $IMAGE_NAME_ENV $IMAGE_NAME_GENERAL - - docker push $IMAGE_NAME_GENERAL - cache: - paths: - - .trivy/.trivycache/ - artifacts: - reports: - container_scanning: .trivy/gl-scanning-report.json - -nomadimage: - stage: docker - image: docker:latest - services: - - name: docker:dind - alias: docker - needs: [] - script: - - cd deploy/nomad-ci - - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY - - docker pull $DOCKER_REGISTRY/library/golang:latest - - docker tag $DOCKER_REGISTRY/library/golang:latest golang:latest - # Pull base image to avoid rebuilding every pipeline if nothing changed, prioritize image from branch - - docker pull $NOMAD_CI_IMAGE_NAME_GENERAL || docker pull $NOMAD_CI_BASE_IMAGE || true - - docker build --cache-from $NOMAD_CI_BASE_IMAGE --cache-from $NOMAD_CI_IMAGE_NAME_GENERAL -t $NOMAD_CI_IMAGE_NAME_ENV -t $NOMAD_CI_IMAGE_NAME_GENERAL . - - docker push $NOMAD_CI_IMAGE_NAME_ENV - - docker push $NOMAD_CI_IMAGE_NAME_GENERAL - - -test_e2e: - image: $NOMAD_CI_IMAGE_NAME_ENV - stage: e2e - needs: - - compile - - dep-scan - - nomadimage - services: - - name: docker:dind - alias: docker - variables: - DOCKER_HOST: "tcp://docker:2375" - script: - # Avoid docker pull limit - - docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY - - docker pull $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 - - docker tag $DOCKER_REGISTRY/openhpi/co_execenv_python:3.8 openhpi/co_execenv_python:3.8 - # Setup own Nomad cluster and wait for startup - - export NOMAD_ADDR=http://localhost:4646 - - nomad agent -dev -log-level=WARN & - - sleep 5 - # Start Poseidon and wait for it - - ./poseidon & - - sleep 20 - - make e2e-test - - -.start_deployment: &start_deployment - image: $NOMAD_CI_IMAGE_NAME_ENV - stage: deploy - needs: - - job: dockerimage - artifacts: false - - test_e2e - before_script: - - export NOMAD_NAMESPACE="$NOMAD_SLUG" - - nomad namespace apply $NOMAD_NAMESPACE - script: - - export NOMAD_CACERT_DATA=$(cat $NOMAD_CACERT) - # Only replace set env vars - - envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < deploy/api.tpl.nomad > deploy/api.nomad - # Make sure to set NOMAD_ADDR, NOMAD_SKIP_VERIFY and NOMAD_TOKEN env vars in CI settings appropriately - - nomad validate deploy/api.nomad - # nomad plan returns 1 if allocation is created or destroyed which is what we want here - - nomad plan deploy/api.nomad || [ $? == 1 ] - - nomad run deploy/api.nomad - artifacts: - paths: - - deploy/api.nomad - expire_in: 1 month - expose_as: api-nomad - -deploy_review: - <<: *start_deployment - variables: - HOSTNAME: $CI_ENVIRONMENT_SLUG.$BASE_DOMAIN - environment: - name: $CI_COMMIT_REF_SLUG - url: http://$CI_ENVIRONMENT_SLUG.$BASE_DOMAIN - on_stop: stop_review - before_script: - - export NOMAD_NAMESPACE="$NOMAD_SLUG" - - nomad namespace apply $NOMAD_NAMESPACE - only: - - branches - - tags - except: - - main - when: manual - -stop_review: - # See: - # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - stage: cleanup - image: $NOMAD_CI_IMAGE_NAME_ENV - variables: - GIT_STRATEGY: none - script: - - export NOMAD_NAMESPACE="$NOMAD_SLUG" - # Stop all jobs before deleting the namespace - - nomad job status | cut -d" " -f1 | tail -n +2 | xargs -n1 nomad stop - - nomad namespace delete $NOMAD_NAMESPACE - environment: - name: $CI_COMMIT_REF_SLUG - action: stop - needs: [] - allow_failure: true - only: - - branches - - tags - except: - - main - when: manual - -deploy_staging: - <<: *start_deployment - variables: - NOMAD_SLUG: $NOMAD_PREFIX-staging - HOSTNAME: staging.$BASE_DOMAIN - environment: - name: staging - url: http://staging.$BASE_DOMAIN - only: - - main - -deploy_production: - <<: *start_deployment - variables: - NOMAD_SLUG: $NOMAD_PREFIX-production - HOSTNAME: $PRODUCTION_URL - environment: - name: production - url: https://$PRODUCTION_URL - only: - - main - when: manual diff --git a/Makefile b/Makefile index 923f54f..f08816b 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,8 @@ UNIT_TESTS = $(shell go list ./... | grep -v /e2e) DOCKER_E2E_CONTAINER_NAME := "$(PROJECT_NAME)-e2e-tests" DOCKER_TAG := "poseidon:latest" DOCKER_OPTS := -v $(shell pwd)/configuration.yaml:/configuration.yaml +# don't use :latest to prevent Nomad from trying to pull the image automatically +E2E_TEST_DOCKER_IMAGE = "poseidon/e2e-docker-image:ci" default: help @@ -79,9 +81,13 @@ coverage: deps ## Generate code coverage report coverhtml: coverage ## Generate HTML coverage report @go tool cover -html=coverage_cleaned.cov -o coverage_unit.html +.PHONY: e2e-test-docker-image ## Build Docker image used in e2e tests +e2e-test-docker-image: deploy/e2e-test-image/Dockerfile + @docker build -t $(E2E_TEST_DOCKER_IMAGE) deploy/e2e-test-image + .PHONY: e2e-test -e2e-test: deps ## Run e2e tests - @go test -count=1 ./tests/e2e -v -args -dockerImage="drp.codemoon.xopic.de/openhpi/co_execenv_python:3.8" +e2e-test: deps e2e-test-docker-image ## Run e2e tests + @go test -count=1 ./tests/e2e -v -args -dockerImage="$(E2E_TEST_DOCKER_IMAGE)" .PHONY: e2e-docker e2e-docker: docker ## Run e2e tests against the Docker container diff --git a/README.md b/README.md index f69ba07..a5d092c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Poseidon -[![pipeline status](https://gitlab.hpi.de/codeocean/codemoon/poseidon/badges/main/pipeline.svg)](https://gitlab.hpi.de/codeocean/codemoon/poseidon/-/commits/main) -[![coverage report](https://gitlab.hpi.de/codeocean/codemoon/poseidon/badges/main/coverage.svg)](https://gitlab.hpi.de/codeocean/codemoon/poseidon/-/commits/main) +[![CI](https://github.com/openHPI/poseidon/actions/workflows/ci.yml/badge.svg)](https://github.com/openHPI/poseidon/actions/workflows/ci.yml) Poseidon logo diff --git a/deploy/e2e-test-image/Dockerfile b/deploy/e2e-test-image/Dockerfile new file mode 100644 index 0000000..03a9d2e --- /dev/null +++ b/deploy/e2e-test-image/Dockerfile @@ -0,0 +1,9 @@ +# Minimal working Docker image used in our e2e tests +FROM python:latest + +RUN useradd --home-dir /workspace --no-create-home --user-group user && \ + mkdir /workspace && chown user:user /workspace + +WORKDIR /workspace + +USER user diff --git a/deploy/nomad-ci/README.md b/deploy/nomad-ci/README.md new file mode 100644 index 0000000..8bcced1 --- /dev/null +++ b/deploy/nomad-ci/README.md @@ -0,0 +1,21 @@ +# Nomad-in-Docker Image + +The [`Dockerfile`](Dockerfile) in this folder creates a Docker image that contains Docker and Nomad. + +Running the image requires the following Docker options: + +- Allow Nomad to use mount: `--cap-add=SYS_ADMIN` +- Allow Nomad to use bind mounts: `--security-opt apparmor=unconfined` +- Add access to Docker daemon: `-v /var/run/docker.sock:/var/run/docker.sock` +- Map port to host: `-p 4646:4646` + +A complete command to run the container is as follows. + +```shell +docker run --rm --name nomad \ + --cap-add=SYS_ADMIN \ + --security-opt apparmor=unconfined \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -p 4646:4646 \ + nomad-ci +```