From 130dfb96981f74342ac7f7f0a44709e269f9c564 Mon Sep 17 00:00:00 2001 From: jkreller Date: Tue, 6 Feb 2024 20:16:27 +0000 Subject: [PATCH] Add deployment and improve Dockerfiles and CICD --- .gitlab-ci.yml | 75 ++++++++++++------- backend/Dockerfile | 37 +++++++--- backend/go.mod | 2 +- backend/service/date/dateFormat.go | 10 ++- backend/service/date/dateFormat_test.go | 1 + backend/service/ical/icalFileGeneration.go | 1 + docker-compose.prod.yml | 27 ++++--- docker-compose.yml | 22 +++--- frontend/Dockerfile | 23 +++++- frontend/Dockerfile_prod | 14 ---- frontend/nginx.conf | 85 +++++++++++----------- reverseproxy.conf | 16 ++-- 12 files changed, 188 insertions(+), 125 deletions(-) delete mode 100644 frontend/Dockerfile_prod diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a79c77..74ba521 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,8 @@ stages: - lint - build - test - - docker + - oci-build + - deploy lint-frontend: image: node:lts @@ -29,7 +30,7 @@ lint-backend: build-backend: - image: golang:1.21-alpine + image: golang:alpine stage: build rules: - changes: @@ -58,7 +59,7 @@ build-frontend: - frontend/build test-backend: - image: golang:1.21-alpine + image: golang:alpine stage: test rules: - changes: @@ -83,11 +84,10 @@ test-frontend: - lint-frontend build-backend-image: - stage: docker - image: docker:20.10.16 + stage: oci-build + image: docker:latest services: - - name: docker:20.10.16-dind - alias: docker + - docker:dind tags: - image variables: @@ -96,22 +96,19 @@ build-backend-image: DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "/certs/client" - script: - - cd backend + before_script: - docker login -u $CI_DOCKER_REGISTRY_USER -p $CI_DOCKER_REGISTRY_PASSWORD $CI_DOCKER_REGISTRY - - docker build -t htwkalender-backend$IMAGE_TAG . - - docker tag htwkalender-backend$IMAGE_TAG $CI_DOCKER_REGISTRY_USER/htwkalender:backend - - docker push $CI_DOCKER_REGISTRY_USER/htwkalender:backend - only: - - main - - development + script: + - docker build --pull -t $CI_DOCKER_REGISTRY_REPO:backend -f ./backend/Dockerfile --target prod ./backend + - docker push $CI_DOCKER_REGISTRY_REPO:backend + rules: + - if: $CI_COMMIT_BRANCH == "main" build-frontend-image: - stage: docker - image: docker:20.10.16 + stage: oci-build + image: docker:latest services: - - name: docker:20.10.16-dind - alias: docker + - docker:dind tags: - image variables: @@ -120,12 +117,36 @@ build-frontend-image: DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "/certs/client" - script: - - cd frontend + before_script: - docker login -u $CI_DOCKER_REGISTRY_USER -p $CI_DOCKER_REGISTRY_PASSWORD $CI_DOCKER_REGISTRY - - docker build -f Dockerfile_prod -t htwkalender-frontend$IMAGE_TAG . - - docker tag htwkalender-frontend$IMAGE_TAG $CI_DOCKER_REGISTRY_USER/htwkalender:frontend - - docker push $CI_DOCKER_REGISTRY_USER/htwkalender:frontend - only: - - main - - development \ No newline at end of file + script: + - docker build --pull -t $CI_DOCKER_REGISTRY_REPO:frontend -f ./frontend/Dockerfile --target prod . + - docker push $CI_DOCKER_REGISTRY_REPO:frontend + rules: + - if: $CI_COMMIT_BRANCH == "main" + +deploy-all: + stage: deploy + image: alpine:latest + before_script: + - apk add --no-cache openssh-client sed # install dependencies + - eval $(ssh-agent -s) # set some ssh variables + - ssh-add <(echo "$CI_SSH_KEY" | tr -d '\r') + script: + # replace some placeholders + - sed -i -e "s|DOCKER_REGISTRY_REPO|$CI_DOCKER_REGISTRY_REPO|" docker-compose.prod.yml + # upload necessary files to the server + - > + scp -P $CI_SSH_PORT -o StrictHostKeyChecking=no -o LogLevel=ERROR ./docker-compose.prod.yml ./reverseproxy.conf + $CI_SSH_USER@$CI_SSH_HOST:/home/$CI_SSH_USER/docker/htwkalender/ + # ssh to the server and start the service + - > + ssh -p $CI_SSH_PORT -o StrictHostKeyChecking=no -o LogLevel=ERROR $CI_SSH_USER@$CI_SSH_HOST + "cd /home/$CI_SSH_USER/docker/htwkalender/ && + docker login -u $CI_DOCKER_REGISTRY_USER -p $CI_DOCKER_REGISTRY_PASSWORD $CI_DOCKER_REGISTRY && + docker compose -f ./docker-compose.prod.yml down && docker compose -f ./docker-compose.prod.yml up -d --remove-orphans && docker logout" + needs: + - build-frontend-image + - build-backend-image + rules: + - if: $CI_COMMIT_BRANCH == "main" diff --git a/backend/Dockerfile b/backend/Dockerfile index b0e763e..415fce4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,18 +1,31 @@ -FROM golang:1.21.6 +# build stage +FROM golang:alpine AS build -# Set the Current Working Directory inside the container WORKDIR /app -# Copy go mod and sum files -COPY go.mod go.sum ./ -RUN go mod download - # Copy the source from the current directory to the Working Directory inside the container -COPY *.go ./ -COPY . . +COPY . ./ +# download needed modules +RUN apk add --no-cache --update go gcc g++ && \ + go mod download && \ + CGO_ENABLED=1 GOOS=linux go build -o /htwkalender -# Build the Go app -RUN CGO_ENABLED=1 GOOS=linux go build -o /htwkalender +# production stage +FROM alpine:latest AS prod -# Expose port 8080 to the outside world -EXPOSE 8080 +WORKDIR /htwkalender + +ARG USER=ical +RUN adduser -Ds /bin/sh $USER && \ + chown $USER:$USER ./ + +USER $USER +RUN mkdir -p data + +# copies executable from build container +COPY --chown=$USER:$USER --from=build /htwkalender ./ + +# Expose port 8090 to the outside world +EXPOSE 8090 + +ENTRYPOINT ["./htwkalender", "serve"] diff --git a/backend/go.mod b/backend/go.mod index 7a92cc2..e9253c8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,6 +1,6 @@ module htwkalender -go 1.21 +go 1.21.6 require ( github.com/PuerkitoBio/goquery v1.8.1 diff --git a/backend/service/date/dateFormat.go b/backend/service/date/dateFormat.go index 989c99b..ff07ebd 100644 --- a/backend/service/date/dateFormat.go +++ b/backend/service/date/dateFormat.go @@ -1,14 +1,22 @@ package date import ( + "log/slog" "strconv" "strings" "time" + _ "time/tzdata" ) func GetDateFromWeekNumber(year int, weekNumber int, dayName string) (time.Time, error) { // Create a time.Date for the first day of the year - europeTime, _ := time.LoadLocation("Europe/Berlin") + europeTime, err := time.LoadLocation("Europe/Berlin") + + if err != nil { + slog.Error("Failed to load location: ", err) + return time.Time{}, err + } + firstDayOfYear := time.Date(year, time.January, 1, 0, 0, 0, 0, europeTime) // Calculate the number of days to add to reach the desired week diff --git a/backend/service/date/dateFormat_test.go b/backend/service/date/dateFormat_test.go index 7566353..d42ea0b 100644 --- a/backend/service/date/dateFormat_test.go +++ b/backend/service/date/dateFormat_test.go @@ -4,6 +4,7 @@ import ( "reflect" "testing" "time" + _ "time/tzdata" ) func Test_getDateFromWeekNumber(t *testing.T) { diff --git a/backend/service/ical/icalFileGeneration.go b/backend/service/ical/icalFileGeneration.go index 09019ba..e4b3134 100644 --- a/backend/service/ical/icalFileGeneration.go +++ b/backend/service/ical/icalFileGeneration.go @@ -7,6 +7,7 @@ import ( "time" "github.com/jordic/goics" + _ "time/tzdata" ) // IcalModel local type for EmitICal function diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 13b54dd..fec8a95 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,25 +2,32 @@ version: "3.9" services: htwkalender-backend: - build: - dockerfile: Dockerfile - context: ./backend + image: DOCKER_REGISTRY_REPO:backend # DOCKER_REGISTRY_REPO will be replaced by CI + command: "--http=0.0.0.0:8090 --dir=/htwkalender/data/pb_data" + pull_policy: always # open port 8090 ports: - "8090:8090" - command: "/htwkalender serve --http=0.0.0.0:8090 --dir=/pb_data" volumes: - - ./backend/pb_data:/pb_data + - pb_data:/htwkalender/data + htwkalender-frontend: - build: - context: ./frontend - dockerfile: Dockerfile_prod + image: DOCKER_REGISTRY_REPO:frontend # DOCKER_REGISTRY_REPO will be replaced by CI + pull_policy: always + ports: + - "8000:8000" + depends_on: + - htwkalender-backend + rproxy: - image: nginx:stable + image: bitnami/nginx:1.25 volumes: - - ./reverseproxy.conf:/etc/nginx/nginx.conf + - ./reverseproxy.conf:/opt/bitnami/nginx/conf/nginx.conf depends_on: - htwkalender-backend - htwkalender-frontend ports: - "80:80" + +volumes: + pb_data: diff --git a/docker-compose.yml b/docker-compose.yml index 8e105d5..cffeaa5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,29 +5,33 @@ services: build: dockerfile: Dockerfile context: ./backend + target: prod + command: "--http=0.0.0.0:8090 --dir=/htwkalender/data/pb_data" # open port 8090 ports: - - "8090:8090" - command: "/htwkalender serve --http=0.0.0.0:8090 --dir=/pb_data" + - "8090:8090" volumes: - - ./backend/pb_data:/pb_data + - pb_data:/htwkalender/data + htwkalender-frontend: - volumes: - - ./frontend/src:/app/src build: dockerfile: Dockerfile context: ./frontend + target: dev + command: "npm run dev" # open port 8000 ports: - - "8000:8000" - command: "npm run dev" + - "8000:8000" rproxy: - image: nginx:stable + image: bitnami/nginx:1.25 volumes: - - ./reverseproxy.conf:/etc/nginx/nginx.conf + - ./reverseproxy.conf:/opt/bitnami/nginx/conf/nginx.conf depends_on: - htwkalender-backend - htwkalender-frontend ports: - "80:80" + +volumes: + pb_data: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 4eb781c..9e92671 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,7 +1,26 @@ -FROM node:lts-alpine3.18 +# build stage +FROM node:lts-alpine AS build + +WORKDIR /app +COPY frontend/package*.json ./ +RUN npm ci +COPY frontend/ ./ +RUN npm run build + +# development stage +FROM node:lts-alpine AS dev WORKDIR /app COPY package*.json ./ RUN npm install -COPY ./ ./ +COPY . ./ +# production stage +# https://hub.docker.com/r/bitnami/nginx -> always run as non-root user +FROM bitnami/nginx:1.25 AS prod + +# copy build files from build container +COPY --from=build /app/dist /app +COPY ./frontend/nginx.conf /opt/bitnami/nginx/conf/nginx.conf + +EXPOSE 8000 diff --git a/frontend/Dockerfile_prod b/frontend/Dockerfile_prod deleted file mode 100644 index 0d4cb85..0000000 --- a/frontend/Dockerfile_prod +++ /dev/null @@ -1,14 +0,0 @@ -# build stage -FROM node:lts-alpine3.18 as build-stage -WORKDIR /app -COPY package*.json ./ -RUN npm ci -COPY . . -RUN npm run build - -# production stage -FROM nginx:stable as production-stage -COPY --from=build-stage /app/dist /usr/share/nginx/html -COPY nginx.conf /etc/nginx/conf.d/default.conf -EXPOSE 8000 -CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 19f545c..0820513 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,47 +1,44 @@ -server { - listen 8000; - listen [::]:8000; - server_name localhost; +worker_processes 4; - #access_log /var/log/nginx/host.access.log main; +error_log /opt/bitnami/nginx/logs/error.log; +pid /opt/bitnami/nginx/tmp/nginx.pid; - location / { - root /usr/share/nginx/html; - index index.html index.htm; - - #necessary to display vue subpage - try_files $uri $uri/ /index.html; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - - # proxy the PHP scripts to Apache listening on 127.0.0.1:80 - # - #location ~ \.php$ { - # proxy_pass http://127.0.0.1; - #} - - # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 - # - #location ~ \.php$ { - # root html; - # fastcgi_pass 127.0.0.1:9000; - # fastcgi_index index.php; - # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; - # include fastcgi_params; - #} - - # deny access to .htaccess files, if Apache's document root - # concurs with nginx's one - # - #location ~ /\.ht { - # deny all; - #} +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + + access_log /opt/bitnami/nginx/logs/proxy_access.log; + error_log /opt/bitnami/nginx/logs/proxy_error.log; + + sendfile on; + keepalive_timeout 180s; + send_timeout 180s; + + client_body_temp_path /opt/bitnami/nginx/tmp/client_temp; + proxy_temp_path /opt/bitnami/nginx/tmp/proxy_temp_path; + fastcgi_temp_path /opt/bitnami/nginx/tmp/fastcgi_temp; + uwsgi_temp_path /opt/bitnami/nginx/tmp/uwsgi_temp; + scgi_temp_path /opt/bitnami/nginx/tmp/scgi_temp; + + server { + listen 8000; + listen [::]:8000; + server_name localhost; + + location / { + root /opt/bitnami/nginx/html; + index index.html index.htm; + + #necessary to display vue subpage + try_files $uri $uri/ /index.html; + } + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /opt/bitnami/nginx/html; + } + } } diff --git a/reverseproxy.conf b/reverseproxy.conf index 2b78b9b..383aeeb 100644 --- a/reverseproxy.conf +++ b/reverseproxy.conf @@ -1,7 +1,7 @@ -worker_processes 1; +worker_processes 4; -error_log /var/log/nginx/error.log; -pid /var/run/nginx.pid; +error_log /opt/bitnami/nginx/logs/error.log; +pid /opt/bitnami/nginx/tmp/nginx.pid; events { worker_connections 1024; @@ -11,8 +11,8 @@ http { include mime.types; default_type application/octet-stream; - access_log /var/log/nginx/proxy_access.log; - error_log /var/log/nginx/proxy_error.log; + access_log /opt/bitnami/nginx/logs/proxy_access.log; + error_log /opt/bitnami/nginx/logs/proxy_error.log; sendfile on; keepalive_timeout 180s; @@ -23,6 +23,12 @@ http { POST 1; } + client_body_temp_path /opt/bitnami/nginx/tmp/client_temp; + proxy_temp_path /opt/bitnami/nginx/tmp/proxy_temp_path; + fastcgi_temp_path /opt/bitnami/nginx/tmp/fastcgi_temp; + uwsgi_temp_path /opt/bitnami/nginx/tmp/uwsgi_temp; + scgi_temp_path /opt/bitnami/nginx/tmp/scgi_temp; + proxy_cache_path /dev/shm levels=1:2 keys_zone=mcache:16m inactive=600s max_size=512m; proxy_cache_methods GET HEAD; proxy_cache_min_uses 1;