From 657aca005b1a047c224ea97c5bd6f551151f7ec2 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Sun, 21 Apr 2024 01:29:18 +0200 Subject: [PATCH] feat:#59 add dev deployment and use gitlab registry --- .gitlab-ci.yml | 46 ++++++-- docker-compose.dev.yml | 56 ++++++++++ docker-compose.yml | 10 +- frontend/Dockerfile | 6 +- reverseproxy.dev.conf | 235 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 18 deletions(-) create mode 100644 docker-compose.dev.yml create mode 100644 reverseproxy.dev.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49d6a97..3426a74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ stages: - test - oci-build - deploy + - deploy-dev # New stage for development deployment lint-frontend: image: node:lts @@ -113,12 +114,12 @@ build-backend-image: DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "/certs/client" before_script: - - docker login -u $CI_DOCKER_REGISTRY_USER -p $CI_DOCKER_REGISTRY_PASSWORD $CI_DOCKER_REGISTRY + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - - docker build --pull -t $CI_DOCKER_REGISTRY_REPO:backend -f ./backend/Dockerfile --target prod ./backend - - docker push $CI_DOCKER_REGISTRY_REPO:backend + - docker build --pull -t $IMAGE_TAG -f ./backend/Dockerfile --target prod ./backend + - docker push $IMAGE_TAG rules: - - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "development" changes: - backend/**/* @@ -136,15 +137,40 @@ build-frontend-image: DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "/certs/client" before_script: - - docker login -u $CI_DOCKER_REGISTRY_USER -p $CI_DOCKER_REGISTRY_PASSWORD $CI_DOCKER_REGISTRY + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - - docker build --pull -t $CI_DOCKER_REGISTRY_REPO:frontend -f ./frontend/Dockerfile --target prod . - - docker push $CI_DOCKER_REGISTRY_REPO:frontend + - docker build --pull -t $IMAGE_TAG -f ./frontend/Dockerfile --target prod . + - docker push $IMAGE_TAG rules: - - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "development" changes: - frontend/**/* +# Development deployment job +deploy-dev: + stage: deploy-dev # New stage for development deployment + image: alpine:latest + before_script: + - apk add --no-cache openssh-client sed # install dependencies + - eval $(ssh-agent -s) # set some ssh variables + - ssh-add <(echo "$CI_SSH_KEY" | tr -d '\r') + script: + # replace some placeholders + - sed -i -e "s|REGISTRY_REPO|$CI_REGISTRY_REPO|" docker-compose.dev.yml # Assuming you have a separate docker-compose file for development + # upload necessary files to the dev server + - > + scp -P $CI_SSH_PORT -o StrictHostKeyChecking=no -o LogLevel=ERROR ./docker-compose.dev.yml ./reverseproxy.dev.conf + $CI_SSH_USER@$CI_SSH_DEV_HOST:/home/$CI_SSH_USER/docker/htwkalender/ + # ssh to the dev server and start the service + - > + ssh -p $CI_SSH_PORT -o StrictHostKeyChecking=no -o LogLevel=ERROR $CI_SSH_USER@$CI_SSH_DEV_HOST + "cd /home/$CI_SSH_USER/docker/htwkalender/ && + docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY && + docker compose -f ./docker-compose.dev.yml down && docker compose -f ./docker-compose.dev.yml up -d --remove-orphans && docker logout" + rules: + - if: $CI_COMMIT_BRANCH == "development" # Only execute for the development branch + + deploy-all: stage: deploy image: alpine:latest @@ -154,7 +180,7 @@ deploy-all: - ssh-add <(echo "$CI_SSH_KEY" | tr -d '\r') script: # replace some placeholders - - sed -i -e "s|DOCKER_REGISTRY_REPO|$CI_DOCKER_REGISTRY_REPO|" docker-compose.prod.yml + - sed -i -e "s|REGISTRY_REPO|$CI_REGISTRY_REPO|" docker-compose.prod.yml # upload necessary files to the server - > scp -P $CI_SSH_PORT -o StrictHostKeyChecking=no -o LogLevel=ERROR ./docker-compose.prod.yml ./reverseproxy.conf @@ -163,7 +189,7 @@ deploy-all: - > ssh -p $CI_SSH_PORT -o StrictHostKeyChecking=no -o LogLevel=ERROR $CI_SSH_USER@$CI_SSH_HOST "cd /home/$CI_SSH_USER/docker/htwkalender/ && - docker login -u $CI_DOCKER_REGISTRY_USER -p $CI_DOCKER_REGISTRY_PASSWORD $CI_DOCKER_REGISTRY && + docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY && docker compose -f ./docker-compose.prod.yml down && docker compose -f ./docker-compose.prod.yml up -d --remove-orphans && docker logout" rules: - if: $CI_COMMIT_BRANCH == "main" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..f0146e2 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,56 @@ +#Calendar implementation for the HTWK Leipzig timetable. Evaluation and display of the individual dates in iCal format. +#Copyright (C) 2024 HTWKalender support@htwkalender.de + +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. + +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. + +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . + +services: + htwkalender-backend: + image: DOCKER_REGISTRY_REPO:backend # DOCKER_REGISTRY_REPO will be replaced by CI + command: "--http=0.0.0.0:8090 --dir=/htwkalender/data/pb_data" + pull_policy: always + restart: always + volumes: + - pb_data:/htwkalender/data + networks: + - "net" + + htwkalender-frontend: + image: DOCKER_REGISTRY_REPO:frontend # DOCKER_REGISTRY_REPO will be replaced by CI + pull_policy: always + restart: always + depends_on: + - htwkalender-backend + networks: + - "net" + + rproxy: + image: bitnami/nginx:1.25 + restart: always + volumes: + - ./reverseproxy.dev.conf:/opt/bitnami/nginx/conf/nginx.conf + depends_on: + - htwkalender-backend + - htwkalender-frontend + ports: + - "443:443" + - "80:80" + networks: + - "net" + +volumes: + pb_data: + +networks: + net: + external: true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6b312f1..aa1b485 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,8 +23,8 @@ services: context: ./backend target: dev # prod command: "--http=0.0.0.0:8090 --dir=/htwkalender/data/pb_data" - ports: - - "8090:8090" + #ports: + # - "8090:8090" volumes: - pb_data:/htwkalender/data # for production with volume # - ./backend:/htwkalender/data # for development with bind mount from project directory @@ -33,8 +33,8 @@ services: build: dockerfile: Dockerfile context: ./frontend - target: dev - command: "npm run dev" + target: prod + # command: "npm run dev" # open port 8000 ports: - "8000:8000" @@ -44,7 +44,7 @@ services: rproxy: image: bitnami/nginx:1.25 volumes: - - ./reverseproxy.conf:/opt/bitnami/nginx/conf/nginx.conf + - ./reverseproxy.dev.conf:/opt/bitnami/nginx/conf/nginx.conf depends_on: - htwkalender-backend - htwkalender-frontend diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 8a87d80..5b3a522 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -18,9 +18,9 @@ FROM node:lts-alpine AS build WORKDIR /app -COPY frontend/package*.json ./ +COPY package*.json ./ RUN npm ci -COPY frontend/ ./ +COPY / ./ RUN npm run build # development stage @@ -37,6 +37,6 @@ FROM bitnami/nginx:1.25 AS prod # copy build files from build container COPY --from=build /app/dist /app -COPY ./frontend/nginx.conf /opt/bitnami/nginx/conf/nginx.conf +COPY ./nginx.conf /opt/bitnami/nginx/conf/nginx.conf EXPOSE 8000 diff --git a/reverseproxy.dev.conf b/reverseproxy.dev.conf new file mode 100644 index 0000000..de7c057 --- /dev/null +++ b/reverseproxy.dev.conf @@ -0,0 +1,235 @@ +#Calendar implementation for the HTWK Leipzig timetable. Evaluation and display of the individual dates in iCal format. +#Copyright (C) 2024 HTWKalender support@htwkalender.de + +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. + +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. + +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . + +worker_processes 4; + +error_log /opt/bitnami/nginx/logs/error.log; +pid /opt/bitnami/nginx/tmp/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + + # Cloudflare IP Ranges (https://www.cloudflare.com/ips/) + set_real_ip_from 173.245.48.0/20; + set_real_ip_from 103.21.244.0/22; + set_real_ip_from 103.22.200.0/22; + set_real_ip_from 103.31.4.0/22; + set_real_ip_from 141.101.64.0/18; + set_real_ip_from 108.162.192.0/18; + set_real_ip_from 190.93.240.0/20; + set_real_ip_from 188.114.96.0/20; + set_real_ip_from 197.234.240.0/22; + set_real_ip_from 198.41.128.0/17; + set_real_ip_from 162.158.0.0/15; + set_real_ip_from 104.16.0.0/13; + set_real_ip_from 104.24.0.0/14; + set_real_ip_from 172.64.0.0/13; + set_real_ip_from 131.0.72.0/22; + set_real_ip_from 2400:cb00::/32; + set_real_ip_from 2606:4700::/32; + set_real_ip_from 2803:f800::/32; + set_real_ip_from 2405:b500::/32; + set_real_ip_from 2405:8100::/32; + set_real_ip_from 2a06:98c0::/29; + set_real_ip_from 2c0f:f248::/32; + + # Docker IP Ranges (https://docs.docker.com/network/iptables/) + set_real_ip_from 172.16.0.0/12; + set_real_ip_from 141.57.0.0/16; + + real_ip_header CF-Connecting-IP; + + access_log /opt/bitnami/nginx/logs/proxy_access.log; + error_log /opt/bitnami/nginx/logs/proxy_error.log; + + sendfile on; + keepalive_timeout 180s; + send_timeout 180s; + + map $request_method $cache_bypass { + default 0; + POST 1; + PUT 1; + DELETE 1; + } + + client_body_temp_path /opt/bitnami/nginx/tmp/client_temp; + proxy_temp_path /opt/bitnami/nginx/tmp/proxy_temp_path; + fastcgi_temp_path /opt/bitnami/nginx/tmp/fastcgi_temp; + uwsgi_temp_path /opt/bitnami/nginx/tmp/uwsgi_temp; + scgi_temp_path /opt/bitnami/nginx/tmp/scgi_temp; + + proxy_cache_path /dev/shm levels=1:2 keys_zone=mcache:16m inactive=600s max_size=512m; + proxy_cache_methods GET HEAD; + proxy_cache_min_uses 1; + proxy_cache_key "$request_method$host$request_uri"; + proxy_cache_use_stale timeout updating; + proxy_ignore_headers Cache-Control Expires Set-Cookie; + + proxy_buffering on; + proxy_buffers 8 16k; + proxy_buffer_size 16k; + proxy_busy_buffers_size 64k; + proxy_temp_file_write_size 64k; + proxy_max_temp_file_size 1024m; + + geo $admin { + default 1; + 10.0.0.0/8 0; # Private Network + 127.0.0.0/8 0; # Localhost Network + 192.168.0.0/16 0; # Localhost Network + 141.57.0.0/16 0; # HTWK Leipzig Network + 172.16.0.0/12 0; # Private Network + } + + map $admin $limit_key { + 0 ""; + 1 $binary_remote_addr; + } + + # Limit the number of requests per IP + limit_req_zone $limit_key zone=feed:20m rate=20r/m; + limit_req_zone $limit_key zone=createFeed:10m rate=1r/m; + limit_req_zone $limit_key zone=modules:10m rate=30r/m; + + server { + listen 80; + listen [::]:80; + + location /api/feed { + proxy_pass http://htwkalender-backend:8090; + client_max_body_size 2m; + proxy_connect_timeout 600s; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + send_timeout 600s; + limit_req zone=feed burst=10 nodelay; + } + + location /api { + proxy_pass http://htwkalender-backend:8090; + client_max_body_size 20m; + proxy_connect_timeout 600s; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + send_timeout 600s; + } + + # Cache only specific URI + location /api/modules { + proxy_pass http://htwkalender-backend:8090; + client_max_body_size 20m; + proxy_connect_timeout 600s; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + send_timeout 600s; + proxy_cache_bypass 0; + proxy_no_cache 0; + proxy_cache mcache; # mcache=RAM + proxy_cache_valid 200 301 302 10m; + proxy_cache_valid 403 404 5m; + proxy_cache_lock on; + proxy_cache_use_stale timeout updating; + add_header X-Proxy-Cache $upstream_cache_status; + limit_req zone=modules burst=5 nodelay; + } + + location /api/rooms { + proxy_pass http://htwkalender-backend:8090; + client_max_body_size 20m; + proxy_connect_timeout 600s; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + send_timeout 600s; + proxy_cache_bypass 0; + proxy_no_cache 0; + proxy_cache mcache; # mcache=RAM + proxy_cache_valid 200 301 302 10m; + proxy_cache_valid 403 404 5m; + proxy_cache_lock on; + proxy_cache_use_stale timeout updating; + add_header X-Proxy-Cache $upstream_cache_status; + limit_req zone=modules burst=5 nodelay; + } + + location /api/schedule { + proxy_pass http://htwkalender-backend:8090; + client_max_body_size 20m; + proxy_connect_timeout 600s; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + send_timeout 600s; + proxy_cache_bypass 0; + proxy_no_cache 0; + proxy_cache mcache; # mcache=RAM + proxy_cache_valid 200 301 302 10m; + proxy_cache_valid 403 404 5m; + proxy_cache_lock on; + proxy_cache_use_stale timeout updating; + add_header X-Proxy-Cache $upstream_cache_status; + limit_req zone=modules burst=30 nodelay; + } + + location /api/courses { + proxy_pass http://htwkalender-backend:8090; + client_max_body_size 20m; + proxy_connect_timeout 600s; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + send_timeout 600s; + proxy_cache_bypass 0; + proxy_no_cache 0; + proxy_cache mcache; # mcache=RAM + proxy_cache_valid 200 301 302 10m; + proxy_cache_valid 403 404 5m; + proxy_cache_lock on; + proxy_cache_use_stale timeout updating; + add_header X-Proxy-Cache $upstream_cache_status; + limit_req zone=modules burst=5 nodelay; + } + + location /api/createFeed { + limit_req zone=createFeed nodelay; + # return limit request error + limit_req_status 429; + proxy_pass http://htwkalender-backend:8090; + client_max_body_size 2m; + proxy_connect_timeout 600s; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + send_timeout 600s; + } + + location /_ { + proxy_pass http://htwkalender-backend:8090; + # if user is not 0 in admin list, return 404 + if ($admin) { + return 404 "Not Found"; + } + # Increase upload file size + client_max_body_size 100m; + } + + location / { + proxy_pass http://htwkalender-frontend:8000; + } + } +}