From ab22364ac82151878f1ff668d1ca2a17dc1ac365 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Fri, 5 Jul 2024 15:23:10 +0200 Subject: [PATCH] fix:#46 updated fetcher and empty course issue --- services/data-manager/service/addRoute.go | 8 ++- .../service/course/courseFunctions.go | 18 ++++-- services/data-manager/service/db/dbGroups.go | 44 +++++++++---- .../service/events/courseService.go | 3 +- .../service/events/eventService.go | 10 ++- .../fetch/v1/fetchSeminarEventService.go | 62 +++++-------------- .../fetch/v1/fetchSeminarGroupService.go | 3 +- 7 files changed, 76 insertions(+), 72 deletions(-) diff --git a/services/data-manager/service/addRoute.go b/services/data-manager/service/addRoute.go index 8dadd40..9c5c068 100644 --- a/services/data-manager/service/addRoute.go +++ b/services/data-manager/service/addRoute.go @@ -336,8 +336,12 @@ func AddRoutes(app *pocketbase.PocketBase) { courses := events.GetAllCourses(app) return c.JSON(200, courses) } else { - courses := events.GetAllCoursesForSemester(app, semester) - return c.JSON(200, courses) + seminarGroups := events.GetAllCoursesForSemester(app, semester) + courseStringList := make([]string, 0) + for _, seminarGroup := range seminarGroups { + courseStringList = append(courseStringList, seminarGroup.Course) + } + return c.JSON(200, courseStringList) } }, Middlewares: []echo.MiddlewareFunc{ diff --git a/services/data-manager/service/course/courseFunctions.go b/services/data-manager/service/course/courseFunctions.go index 0612d97..22febd4 100644 --- a/services/data-manager/service/course/courseFunctions.go +++ b/services/data-manager/service/course/courseFunctions.go @@ -18,16 +18,26 @@ package course import ( "github.com/pocketbase/pocketbase" + "htwkalender/data-manager/model" "htwkalender/data-manager/service/events" + "htwkalender/data-manager/service/functions" + "htwkalender/data-manager/service/functions/time" "log/slog" ) func UpdateCourse(app *pocketbase.PocketBase) { - courses := events.GetAllCourses(app) - for _, course := range courses { - _, err := events.UpdateModulesForCourse(app, course) + currentSemesters := functions.CalculateSemesterList(time.RealClock{}) + + var seminarGroups []model.SeminarGroup + + for _, semester := range currentSemesters { + seminarGroups = append(seminarGroups, events.GetAllCoursesForSemester(app, semester)...) + } + + for _, seminarGroup := range seminarGroups { + _, err := events.UpdateModulesForCourse(app, seminarGroup) if err != nil { - slog.Warn("Update Course: "+course+" failed:", "error", err) + slog.Warn("Update Course: "+seminarGroup.Course+" failed:", "error", err) } } } diff --git a/services/data-manager/service/db/dbGroups.go b/services/data-manager/service/db/dbGroups.go index 059fb65..5e70e21 100644 --- a/services/data-manager/service/db/dbGroups.go +++ b/services/data-manager/service/db/dbGroups.go @@ -20,6 +20,7 @@ import ( "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase/models" + "htwkalender/data-manager/model" "log/slog" ) @@ -38,6 +39,31 @@ func (s *SeminarGroup) TableName() string { return "groups" } +// UniqueKey Should be same as unique constraint in the database +func (s *SeminarGroup) UniqueKey() string { + return s.Course + s.Semester +} + +func (s *SeminarGroup) toSeminarGroupModel() model.SeminarGroup { + return model.SeminarGroup{ + University: s.University, + GroupShortcut: s.GroupShortcut, + GroupId: s.GroupId, + Course: s.Course, + Faculty: s.Faculty, + FacultyId: s.FacultyId, + Semester: s.Semester, + } +} + +func (s *SeminarGroups) toSeminarGroupModels() []model.SeminarGroup { + var seminarGroups []model.SeminarGroup + for _, group := range *s { + seminarGroups = append(seminarGroups, group.toSeminarGroupModel()) + } + return seminarGroups +} + type SeminarGroups []*SeminarGroup func SaveGroups(seminarGroups SeminarGroups, app *pocketbase.PocketBase) (SeminarGroups, error) { @@ -84,26 +110,18 @@ func GetAllCourses(app *pocketbase.PocketBase) []string { return courseArray } -func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []string { +func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []model.SeminarGroup { - var courses []struct { - CourseShortcut string `db:"course" json:"course"` - } + var courses SeminarGroups // get all courses for a specific semester - err := app.Dao().DB().Select("course").From("groups").Where(dbx.NewExp("semester = {:semester}", dbx.Params{"semester": semester})).All(&courses) + err := app.Dao().DB().Select("*").From("groups").Where(dbx.NewExp("semester = {:semester}", dbx.Params{"semester": semester})).All(&courses) if err != nil { slog.Error("Error while getting groups from database: ", "error", err) - return []string{} + return nil } - var courseArray []string - - for _, course := range courses { - courseArray = append(courseArray, course.CourseShortcut) - } - - return courseArray + return courses.toSeminarGroupModels() } diff --git a/services/data-manager/service/events/courseService.go b/services/data-manager/service/events/courseService.go index 389a428..ea291fd 100644 --- a/services/data-manager/service/events/courseService.go +++ b/services/data-manager/service/events/courseService.go @@ -18,6 +18,7 @@ package events import ( "github.com/pocketbase/pocketbase" + "htwkalender/data-manager/model" "htwkalender/data-manager/service/db" "htwkalender/data-manager/service/functions" ) @@ -26,7 +27,7 @@ func GetAllCourses(app *pocketbase.PocketBase) []string { return db.GetAllCourses(app) } -func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []string { +func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []model.SeminarGroup { return db.GetAllCoursesForSemester(app, semester) } diff --git a/services/data-manager/service/events/eventService.go b/services/data-manager/service/events/eventService.go index afa318a..ab82d52 100644 --- a/services/data-manager/service/events/eventService.go +++ b/services/data-manager/service/events/eventService.go @@ -120,14 +120,12 @@ func DeleteAllEvents(app *pocketbase.PocketBase) error { // 3. Save all events for the course and the semester // If the update was successful, nil is returned // If the update was not successful, an error is returned -func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Events, error) { +func UpdateModulesForCourse(app *pocketbase.PocketBase, seminarGroup model.SeminarGroup) (model.Events, error) { - seminarGroup, err := v1.GetSeminarGroupEventsFromHTML(course) + seminarGroup, err := v1.FetchAndParse(seminarGroup.Semester, seminarGroup.Course) if err != nil { return nil, err } - - seminarGroup = v1.ClearEmptySeminarGroups(seminarGroup) seminarGroup = v1.ReplaceEmptyEventNames(seminarGroup) //check if events in the seminarGroups Events are already in the database @@ -136,7 +134,7 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Ev //if there are no events in the database, save the new events //get all events for the course and the semester - dbEvents, err := db.GetAllEventsForCourse(app, course) + dbEvents, err := db.GetAllEventsForCourse(app, seminarGroup.Course) if err != nil { return nil, err } @@ -182,7 +180,7 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Ev return nil, err } - slog.Info("Course: " + course + " - Event changes: " + strconv.FormatInt(int64(len(insertList)), 10) + " new events, " + strconv.FormatInt(int64(len(deleteList)), 10) + " deleted events") + slog.Info("Course: " + seminarGroup.Course + " - Event changes: " + strconv.FormatInt(int64(len(insertList)), 10) + " new events, " + strconv.FormatInt(int64(len(deleteList)), 10) + " deleted events") return savedEvents, nil } diff --git a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go index e2a8780..4a02776 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go @@ -41,16 +41,12 @@ func ReplaceEmptyEventNames(group model.SeminarGroup) model.SeminarGroup { return group } -func ClearEmptySeminarGroups(seminarGroup model.SeminarGroup) model.SeminarGroup { - var newSeminarGroup = model.SeminarGroup{} - - if len(seminarGroup.Events) > 0 && seminarGroup.Course != "" { - newSeminarGroup = seminarGroup - } - return newSeminarGroup -} - func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) { + // check that semester and seminarGroupLabel are not empty + if semester == "" || seminarGroupLabel == "" { + return "", fmt.Errorf("semester or seminarGroupLabel is empty") + } + url := "https://stundenplan.htwk-leipzig.de/" + semester + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65" result, err := fetch.GetHTML(url) if err != nil { @@ -60,22 +56,6 @@ func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) { return result, nil } -func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) (model.SeminarGroup, error) { - var seminarGroup [2]model.SeminarGroup - var errSS, errWS error - - currentMonth := time.Now().Month() - - if isSummerSemester(currentMonth) { - seminarGroup[0], errSS = fetchAndParse("ss", seminarGroupLabel) - } - if isWinterSemester(currentMonth) { - seminarGroup[1], errWS = fetchAndParse("ws", seminarGroupLabel) - } - - return checkForSuccessfulFetch(errSS, errWS, seminarGroup) -} - func isSummerSemester(month time.Month) bool { return month >= 3 && month <= 10 } @@ -84,7 +64,7 @@ func isWinterSemester(month time.Month) bool { return month >= 9 || month <= 4 } -func fetchAndParse(season, label string) (model.SeminarGroup, error) { +func FetchAndParse(season, label string) (model.SeminarGroup, error) { result, err := fetchHTMLFromURL(season, label) if err != nil { return model.SeminarGroup{}, err @@ -92,20 +72,6 @@ func fetchAndParse(season, label string) (model.SeminarGroup, error) { return parseSeminarGroup(result), nil } -func checkForSuccessfulFetch(errSS error, errWS error, seminarGroup [2]model.SeminarGroup) (model.SeminarGroup, error) { - switch { - case errSS != nil && errWS != nil: - return model.SeminarGroup{}, errWS - case errSS != nil: - return seminarGroup[1], nil - case errWS != nil: - return seminarGroup[0], nil - default: - seminarGroup[0].Events = append(seminarGroup[0].Events, seminarGroup[1].Events...) - return seminarGroup[0], nil - } -} - func SplitEventType(events []model.Event) ([]model.Event, error) { re, err := regexp.Compile("^([VPS])([wp])$") if err != nil { @@ -135,15 +101,21 @@ func parseSeminarGroup(result string) model.SeminarGroup { eventTables := getEventTables(doc) allDayLabels := getAllDayLabels(doc) - if eventTables == nil || allDayLabels == nil { - return model.SeminarGroup{} - } course := findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data + semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data + semester, year := extractSemesterAndYear(semesterString) + + if eventTables == nil || allDayLabels == nil { + return model.SeminarGroup{ + University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data, + Course: course, + Events: []model.Event{}, + } + } + eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels, course) splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks) events := splitEventsBySingleWeek(splitEventsByWeekVal) - semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data - semester, year := extractSemesterAndYear(semesterString) events = convertWeeksToDates(events, semester, year) events = generateUUIDs(events, course) events, err = SplitEventType(events) diff --git a/services/data-manager/service/fetch/v1/fetchSeminarGroupService.go b/services/data-manager/service/fetch/v1/fetchSeminarGroupService.go index 316ff57..692847c 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarGroupService.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarGroupService.go @@ -89,8 +89,9 @@ func removeDuplicates(groups db.SeminarGroups) db.SeminarGroups { uniqueGroups := make(db.SeminarGroups, 0, len(groups)) seen := make(map[string]struct{}) // Use an empty struct to minimize memory usage + // unique Identifier is the course and semester for _, group := range groups { - key := group.Course + group.Semester + key := group.UniqueKey() if _, exists := seen[key]; !exists { seen[key] = struct{}{} uniqueGroups = append(uniqueGroups, group)