HTWK/htwkalender-pwa
1
0
Fork 0
mirror of https://gitlab.dit.htwk-leipzig.de/htwk-software/htwkalender-pwa.git synced 2025-07-16 09:38:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '65-fix-proxy-block-and-fetcher-error' into 'main'

Browse Source 
65 fix proxy block and fetcher error

See merge request ekresse/htwkalender!26
This commit is contained in:
ekresse
2024-03-15 23:33:10 +00:00
parent ebc3ded952 55f1160723
commit 6e33a6b4fc
6 changed files with 65 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/service/addSchedule.go
View File

@ -5,7 +5,6 @@ import (
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/cron"
"htwkalender/service/course"
"htwkalender/service/events"
"htwkalender/service/feed"
"htwkalender/service/fetch/sport"
v2 "htwkalender/service/fetch/v2"
@ -34,7 +33,7 @@ func AddSchedules(app *pocketbase.PocketBase) {
feed.ClearFeeds(app.Dao(), 6, time.RealClock{})
})
// Every sunday at 2am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0"
// Every sunday at 3am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0"
scheduler.MustAdd("fetchSportEvents", "0 3 * * 0", func() {
slog.Info("Started fetching sport events schedule")
sportEvents, err := sport.FetchAndUpdateSportEvents(app)
@ -44,13 +43,8 @@ func AddSchedules(app *pocketbase.PocketBase) {
slog.Info("Successfully fetched " + strconv.FormatInt(int64(len(sportEvents)), 10) + " sport events")
})
//delete all events and then fetch all events from remote this should be done every sunday at 2am
//fetch all events for semester and delete from remote this should be done every sunday at 2am
scheduler.MustAdd("fetchEvents", "0 2 * * 0", func() {
err := events.DeleteAllEvents(app)
if err != nil {
slog.Error("Failed to delete all events: %v", err)
}
savedEvents, err := v2.FetchAllEventsAndSave(app, time.RealClock{})
if err != nil {
slog.Error("Failed to fetch and save events: %v", err)

21
backend/service/db/dbEvents.go
View File

@ -203,7 +203,7 @@ func GetAllModulesDistinctByNameAndCourse(app *pocketbase.PocketBase) ([]model.M
return modules, nil
}
func DeleteAllEventsForCourse(app *pocketbase.PocketBase, course string, semester string) error {
func DeleteAllEventsByCourse(app *pocketbase.PocketBase, course string, semester string) error {
_, err := app.Dao().DB().Delete("events", dbx.NewExp("course = {:course} AND semester = {:semester}", dbx.Params{"course": course, "semester": semester})).Execute()
if err != nil {
@ -212,6 +212,25 @@ func DeleteAllEventsForCourse(app *pocketbase.PocketBase, course string, semeste
return nil
}
func DeleteAllEventsBySemesterWithoutCourse(app *pocketbase.PocketBase, course string, semester string) error {
_, err := app.Dao().DB().Delete("events", dbx.NewExp("course != {:course} AND semester = {:semester}", dbx.Params{"course": course, "semester": semester})).Execute()
if err != nil {
return err
}
return nil
}
func DeleteAllEventsBySemester(app *pocketbase.PocketBase, semester string) error {
_, err := app.Dao().DB().Delete("events", dbx.NewExp("semester = {:semester}", dbx.Params{"semester": semester})).Execute()
if err != nil {
return err
}
return nil
}
func DeleteAllEvents(app *pocketbase.PocketBase) error {
_, err := app.Dao().DB().Delete("events", dbx.NewExp("1=1")).Execute()

2
backend/service/events/eventService.go
View File

@ -77,7 +77,7 @@ func GetModuleByUUID(app *pocketbase.PocketBase, uuid string) (model.Module, err
// If the deletion was successful, nil is returned
// If the deletion was not successful, an error is returned
func DeleteAllEventsByCourseAndSemester(app *pocketbase.PocketBase, course string, semester string) error {
err := db.DeleteAllEventsForCourse(app, course, semester)
err := db.DeleteAllEventsByCourse(app, course, semester)
if err != nil {
return err
} else {

2
backend/service/fetch/sport/sportFetcher.go
View File

@ -65,7 +65,7 @@ func FetchAndUpdateSportEvents(app *pocketbase.PocketBase) ([]model.Event, error
}
// @TODO: delete and save events in one transaction and it only should delete events that are not in the new events list and save events that are not in the database
err = db.DeleteAllEventsForCourse(app, "Sport", functions.GetCurrentSemesterString())
err = db.DeleteAllEventsByCourse(app, "Sport", functions.GetCurrentSemesterString())
if err != nil {
return nil, err
}

50
backend/service/fetch/v2/fetcher.go
View File

@ -12,6 +12,7 @@ import (
localTime "htwkalender/service/functions/time"
"log/slog"
"strings"
"time"
)
func ParseEventsFromRemote(app *pocketbase.PocketBase) (model.Events, error) {
@ -24,6 +25,7 @@ func ParseEventsFromRemote(app *pocketbase.PocketBase) (model.Events, error) {
func FetchAllEventsAndSave(app *pocketbase.PocketBase, clock localTime.Clock) ([]model.Event, error) {
var savedRecords []model.Event
var err error = nil
var stubUrl = [2]string{
"https://stundenplan.htwk-leipzig.de/",
@ -51,32 +53,46 @@ func FetchAllEventsAndSave(app *pocketbase.PocketBase, clock localTime.Clock) ([
"%0A?&template=sws_modul&weeks=1-65&combined=yes",
}
if (clock.Now().Month() >= 3) && (clock.Now().Month() <= 10) {
url := stubUrl[0] + "ss" + stubUrl[1]
events, err := parseEventForOneSemester(url)
if err != nil {
return nil, fmt.Errorf("failed to parse events for summmer semester: %w", err)
}
savedEvents, dbError := db.SaveEvents(events, app)
if dbError != nil {
return nil, fmt.Errorf("failed to save events: %w", dbError)
}
savedRecords = append(savedEvents, events...)
savedRecordsSs, errSs := fetchAndSaveAllEventsForSemester(app, time.March, time.October, "ss", clock, stubUrl)
if errSs == nil {
savedRecords = append(savedRecords, savedRecordsSs...)
}
savedRecordsWs, errWs := fetchAndSaveAllEventsForSemester(app, time.September, time.April, "ws", clock, stubUrl)
if errWs == nil {
savedRecords = append(savedRecords, savedRecordsWs...)
}
return savedRecords, err
}
if (clock.Now().Month() >= 9) || (clock.Now().Month() <= 4) {
url := stubUrl[0] + "ws" + stubUrl[1]
events, err := parseEventForOneSemester(url)
func fetchAndSaveAllEventsForSemester(
app *pocketbase.PocketBase,
before time.Month,
after time.Month,
semester string,
clock localTime.Clock,
stubUrl [2]string,
) ([]model.Event, error) {
var err error = nil
var savedRecords []model.Event
if (clock.Now().Month() >= after) || (clock.Now().Month() <= before) {
url := stubUrl[0] + semester + stubUrl[1]
var events []model.Event
events, err = parseEventForOneSemester(url)
if err != nil {
return nil, fmt.Errorf("failed to parse events for winter semester: %w", err)
return nil, fmt.Errorf("failed to parse events for "+semester+": %w", err)
}
err = db.DeleteAllEventsBySemesterWithoutCourse(app, "Sport", semester)
if err != nil {
return nil, fmt.Errorf("failed to delete all events for "+semester+": %w", err)
}
savedEvents, dbError := db.SaveEvents(events, app)
if dbError != nil {
return nil, fmt.Errorf("failed to save events: %w", dbError)
return nil, fmt.Errorf("failed to save events for "+semester+": %w", dbError)
}
savedRecords = append(savedRecords, savedEvents...)
}
return savedRecords, nil
return savedRecords, err
}
func parseEventForOneSemester(url string) ([]model.Event, error) {

15
reverseproxy.conf
View File

@ -37,6 +37,7 @@ http {
# Docker IP Ranges (https://docs.docker.com/network/iptables/)
set_real_ip_from 172.16.0.0/12;
set_real_ip_from 141.57.0.0/16;
real_ip_header CF-Connecting-IP;
@ -89,14 +90,14 @@ http {
}
# Limit the number of requests per IP
limit_req_zone $limit_key zone=feed:20m rate=10r/m;
limit_req_zone $limit_key zone=feed:20m rate=20r/m;
limit_req_zone $limit_key zone=createFeed:10m rate=1r/m;
limit_req_zone $limit_key zone=modules:10m rate=3r/m;
limit_req_zone $limit_key zone=modules:10m rate=30r/m;
server {
listen 80;
listen [::]:80;
server_name htwekalender.de www.htwekalender.de;
server_name htwkalender.de www.htwkalender.de;
location /api {
proxy_pass http://htwkalender-backend:8090;
@ -118,7 +119,7 @@ http {
proxy_cache_bypass 0;
proxy_no_cache 0;
proxy_cache mcache; # mcache=RAM
proxy_cache_valid 200 301 302 30m;
proxy_cache_valid 200 301 302 10m;
proxy_cache_valid 403 404 5m;
proxy_cache_lock on;
proxy_cache_use_stale timeout updating;
@ -136,7 +137,7 @@ http {
proxy_cache_bypass 0;
proxy_no_cache 0;
proxy_cache mcache; # mcache=RAM
proxy_cache_valid 200 301 302 30m;
proxy_cache_valid 200 301 302 10m;
proxy_cache_valid 403 404 5m;
proxy_cache_lock on;
proxy_cache_use_stale timeout updating;
@ -154,7 +155,7 @@ http {
proxy_cache_bypass 0;
proxy_no_cache 0;
proxy_cache mcache; # mcache=RAM
proxy_cache_valid 200 301 302 30m;
proxy_cache_valid 200 301 302 10m;
proxy_cache_valid 403 404 5m;
proxy_cache_lock on;
proxy_cache_use_stale timeout updating;
@ -172,7 +173,7 @@ http {
proxy_cache_bypass 0;
proxy_no_cache 0;
proxy_cache mcache; # mcache=RAM
proxy_cache_valid 200 301 302 30m;
proxy_cache_valid 200 301 302 10m;
proxy_cache_valid 403 404 5m;
proxy_cache_lock on;
proxy_cache_use_stale timeout updating;