diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index 422021e..1202c87 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -9,6 +9,7 @@ import ( "htwkalender/service/ical" "htwkalender/service/room" "io" + "log/slog" "net/http" "github.com/labstack/echo/v5" @@ -24,7 +25,13 @@ func AddRoutes(app *pocketbase.PocketBase) { Method: http.MethodGet, Path: "/api/fetch/events", Handler: func(c echo.Context) error { - return v2.ParseEventsFromRemote(c, app) + savedEvents, err := v2.ParseEventsFromRemote(app) + if err != nil { + slog.Error("Failed to parse events from remote: %v", err) + return c.JSON(http.StatusInternalServerError, "Failed to parse events from remote") + } else { + return c.JSON(http.StatusOK, savedEvents) + } }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), diff --git a/backend/service/addSchedule.go b/backend/service/addSchedule.go index b1263c5..8f85500 100644 --- a/backend/service/addSchedule.go +++ b/backend/service/addSchedule.go @@ -8,6 +8,7 @@ import ( "htwkalender/service/feed" "htwkalender/service/fetch/sport" "htwkalender/service/functions/time" + "log/slog" ) func AddSchedules(app *pocketbase.PocketBase) { @@ -19,17 +20,20 @@ func AddSchedules(app *pocketbase.PocketBase) { // Every three hours update all courses (5 segments - minute, hour, day, month, weekday) "0 */3 * * *" // Every 10 minutes update all courses (5 segments - minute, hour, day, month, weekday) "*/10 * * * *" scheduler.MustAdd("updateCourse", "0 */3 * * *", func() { + slog.Info("Started updating courses schedule") course.UpdateCourse(app) }) // Every sunday at 3am clean all courses (5 segments - minute, hour, day, month, weekday) "0 3 * * 0" scheduler.MustAdd("cleanFeeds", "0 3 * * 0", func() { // clean feeds older than 6 months + slog.Info("Started cleaning feeds schedule") feed.ClearFeeds(app.Dao(), 6, time.RealClock{}) }) // Every sunday at 2am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0" scheduler.MustAdd("fetchEvents", "0 2 * * 0", func() { + slog.Info("Started fetching sport events schedule") sport.FetchAndUpdateSportEvents(app) }) diff --git a/backend/service/course/courseFunctions.go b/backend/service/course/courseFunctions.go index 0d24700..144bd26 100644 --- a/backend/service/course/courseFunctions.go +++ b/backend/service/course/courseFunctions.go @@ -3,18 +3,18 @@ package course import ( "github.com/pocketbase/pocketbase" "htwkalender/service/events" - "log" + "log/slog" + "strconv" ) func UpdateCourse(app *pocketbase.PocketBase) { courses := events.GetAllCourses(app) for _, course := range courses { - err := events.UpdateModulesForCourse(app, course) + savedEvents, err := events.UpdateModulesForCourse(app, course) if err != nil { - log.Println("Update Course: " + course + " failed") - log.Println(err) + slog.Warn("Update Course: "+course+" failed: ", err.Error()) } else { - log.Println("Update Course: " + course + " successful") + slog.Info("Updated Course: " + course + " with " + strconv.FormatInt(int64(len(savedEvents)), 10) + " events") } } } diff --git a/backend/service/db/dbEvents.go b/backend/service/db/dbEvents.go index 5042c2d..10124c4 100644 --- a/backend/service/db/dbEvents.go +++ b/backend/service/db/dbEvents.go @@ -1,7 +1,9 @@ package db import ( + "fmt" "htwkalender/model" + "log/slog" "time" "github.com/pocketbase/dbx" @@ -187,8 +189,8 @@ func GetAllModulesForCourse(app *pocketbase.PocketBase, course string, semester // get all events from event records in the events collection err := app.Dao().DB().Select("*").From("events").Where(dbx.NewExp("course = {:course} AND semester = {:semester}", dbx.Params{"course": course, "semester": semester})).GroupBy("Name").Distinct(true).All(&events) if err != nil { - print("Error while getting events from database: ", err) - return nil, err + slog.Error("Error while getting events from database: ", err) + return nil, fmt.Errorf("error while getting events from database for course %s and semester %s", course, semester) } return events, nil @@ -199,7 +201,8 @@ func GetAllModulesDistinctByNameAndCourse(app *pocketbase.PocketBase) (model.Eve err := app.Dao().DB().Select("*").From("events").GroupBy("Name").Distinct(true).All(&events) if err != nil { - return nil, err + slog.Error("Error while getting events from database: ", err) + return nil, fmt.Errorf("error while getting events distinct by name and course from data") } return events, nil diff --git a/backend/service/events/eventService.go b/backend/service/events/eventService.go index d6dc97b..86eb613 100644 --- a/backend/service/events/eventService.go +++ b/backend/service/events/eventService.go @@ -3,7 +3,6 @@ package events import ( "github.com/labstack/echo/v5" "github.com/pocketbase/pocketbase" - "github.com/pocketbase/pocketbase/apis" "htwkalender/model" "htwkalender/service/db" "htwkalender/service/fetch/v1" @@ -89,7 +88,7 @@ func DeleteAllEvents(app *pocketbase.PocketBase) error { // 3. Save all events for the course and the semester // If the update was successful, nil is returned // If the update was not successful, an error is returned -func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) error { +func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Events, error) { //new string array with one element (course) var courses []string @@ -109,29 +108,32 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) error { //get all events for the course and the semester events, err := db.GetAllModulesForCourse(app, course, "ws") if err != nil { - return apis.NewNotFoundError("Events for winter semester could not be found", err) + return nil, err } // append all events for the course and the semester to the events array for ss summerEvents, err := db.GetAllModulesForCourse(app, course, "ss") if err != nil { - return apis.NewNotFoundError("Events for summer semester could not be found", err) + return nil, err } events = append(events, summerEvents...) //if there are no events in the database, save the new events if len(events) == 0 { - _, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) + events, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) if dbError != nil { - return apis.NewNotFoundError("Events could not be saved", dbError) + return nil, dbError } - return nil + return events, nil } //check if events in the seminarGroups Events are already in the database //if yes, keep the database as it is //if no, delete all events for the course and the semester and save the new events + + var savedEvents model.Events + for _, seminarGroup := range seminarGroups { for _, event := range seminarGroup.Events { // if the event is not in the database, delete all events for the course and the semester and save the new events @@ -139,25 +141,24 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) error { err = DeleteAllEventsByCourseAndSemester(app, course, "ws") if err != nil { - return err + return nil, err } err = DeleteAllEventsByCourseAndSemester(app, course, "ss") if err != nil { - return err + return nil, err } //save the new events - _, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) + savedEvent, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) if dbError != nil { - return apis.NewNotFoundError("Events could not be saved", dbError) + return nil, dbError } - return nil + savedEvents = append(savedEvents, savedEvent...) } } } - - return nil + return savedEvents, nil } func ContainsEvent(events model.Events, event model.Event) bool { diff --git a/backend/service/fetch/v1/fetchSeminarEventService.go b/backend/service/fetch/v1/fetchSeminarEventService.go index 6238c44..6fac5a3 100644 --- a/backend/service/fetch/v1/fetchSeminarEventService.go +++ b/backend/service/fetch/v1/fetchSeminarEventService.go @@ -30,7 +30,7 @@ func GetSeminarEvents(c echo.Context, app *pocketbase.PocketBase) error { seminarGroups = ReplaceEmptyEventNames(seminarGroups) - savedRecords, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) + dbError, savedRecords := db.SaveSeminarGroupEvents(seminarGroups, app) if dbError != nil { return apis.NewNotFoundError("Events could not be saved", dbError.Error()) diff --git a/backend/service/fetch/v2/fetcher.go b/backend/service/fetch/v2/fetcher.go index 6cafd63..93044c8 100644 --- a/backend/service/fetch/v2/fetcher.go +++ b/backend/service/fetch/v2/fetcher.go @@ -1,51 +1,54 @@ package v2 import ( + "fmt" "github.com/google/uuid" - "github.com/labstack/echo/v5" "github.com/pocketbase/pocketbase" "golang.org/x/net/html" "htwkalender/model" "htwkalender/service/db" "htwkalender/service/fetch" - "strconv" + localTime "htwkalender/service/functions/time" "strings" - "time" ) -func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error { - - err, savedRecords := FetchAllEventsAndSave(app) - +func ParseEventsFromRemote(app *pocketbase.PocketBase) (model.Events, error) { + savedRecords, err := FetchAllEventsAndSave(app, localTime.RealClock{}) if err != nil { - return err - } else { - savedRecordsLength := strconv.FormatInt(int64(len(savedRecords)), 10) - return c.JSON(200, "Successfully saved "+savedRecordsLength+" events") + return nil, err } + return savedRecords, nil } -func FetchAllEventsAndSave(app *pocketbase.PocketBase) (error, []model.Event) { - var err error +func FetchAllEventsAndSave(app *pocketbase.PocketBase, clock localTime.Clock) ([]model.Event, error) { var savedRecords []model.Event - var events []model.Event - if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) { + if (clock.Now().Month() >= 3) && (clock.Now().Month() <= 10) { url := "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" - events, err = parseEventForOneSemester(url) + events, err := parseEventForOneSemester(url) + if err != nil { + return nil, fmt.Errorf("failed to parse events for summmer semester: %w", err) + } savedEvents, dbError := db.SaveEvents(events, app) - err = dbError + if dbError != nil { + return nil, fmt.Errorf("failed to save events: %w", dbError) + } savedRecords = append(savedEvents, events...) } - if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) { + if (clock.Now().Month() >= 9) || (clock.Now().Month() <= 4) { url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" - events, err = parseEventForOneSemester(url) + events, err := parseEventForOneSemester(url) + if err != nil { + return nil, fmt.Errorf("failed to parse events for winter semester: %w", err) + } savedEvents, dbError := db.SaveEvents(events, app) - err = dbError + if dbError != nil { + return nil, fmt.Errorf("failed to save events: %w", dbError) + } savedRecords = append(savedEvents, events...) } - return err, savedRecords + return savedRecords, nil } func parseEventForOneSemester(url string) ([]model.Event, error) { @@ -56,25 +59,18 @@ func parseEventForOneSemester(url string) ([]model.Event, error) { } // Parse HTML to Node Tree - doc, err2 := parseHTML(err, webpage) - if err2 != nil { - return nil, err2 + var doc *html.Node + doc, err = parseHTML(webpage, err) + if err != nil { + return nil, err } // Get all event tables and all day labels eventTables := getEventTables(doc) allDayLabels := getAllDayLabels(doc) - if eventTables == nil || allDayLabels == nil { - return nil, err - } - eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels) - if eventsWithCombinedWeeks == nil { - return nil, err - } - splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks) events := splitEventsBySingleWeek(splitEventsByWeekVal) @@ -83,6 +79,11 @@ func parseEventForOneSemester(url string) ([]model.Event, error) { } table := findFirstTable(doc) + + if table == nil { + return nil, fmt.Errorf("failed to find first table") + } + semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data semester, year := extractSemesterAndYear(semesterString) events = convertWeeksToDates(events, semester, year) @@ -101,7 +102,7 @@ func parseEventForOneSemester(url string) ([]model.Event, error) { return events, nil } -func parseHTML(err error, webpage string) (*html.Node, error) { +func parseHTML(webpage string, err error) (*html.Node, error) { doc, err := html.Parse(strings.NewReader(webpage)) if err != nil { return nil, err diff --git a/backend/service/logger/logger.go b/backend/service/logger/logger.go new file mode 100644 index 0000000..90c66f6 --- /dev/null +++ b/backend/service/logger/logger.go @@ -0,0 +1 @@ +package logger diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b27fcd7..24a2a23 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -3588,9 +3588,9 @@ "dev": true }, "node_modules/vite": { - "version": "4.4.12", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.12.tgz", - "integrity": "sha512-KtPlUbWfxzGVul8Nut8Gw2Qe8sBzWY+8QVc5SL8iRFnpnrcoCaNlzO40c1R6hPmcdTwIPEDkq0Y9+27a5tVbdQ==", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", + "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", "dev": true, "dependencies": { "esbuild": "^0.18.10",