HTWK/htwkalender
1
1
Fork 0
mirror of https://gitlab.dit.htwk-leipzig.de/htwk-software/htwkalender.git synced 2025-07-16 09:38:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix:#46 updated fetcher and empty course issue

Browse Source
This commit is contained in:
Elmar Kresse
2024-07-05 15:23:10 +02:00
parent a5a8c2a58d
commit ab22364ac8
7 changed files with 76 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
services/data-manager/service/addRoute.go
View File

@ -336,8 +336,12 @@ func AddRoutes(app *pocketbase.PocketBase) {
courses := events.GetAllCourses(app)
return c.JSON(200, courses)
} else {
courses := events.GetAllCoursesForSemester(app, semester)
return c.JSON(200, courses)
seminarGroups := events.GetAllCoursesForSemester(app, semester)
courseStringList := make([]string, 0)
for _, seminarGroup := range seminarGroups {
courseStringList = append(courseStringList, seminarGroup.Course)
}
return c.JSON(200, courseStringList)
}
},
Middlewares: []echo.MiddlewareFunc{

18
services/data-manager/service/course/courseFunctions.go
View File

@ -18,16 +18,26 @@ package course
import (
"github.com/pocketbase/pocketbase"
"htwkalender/data-manager/model"
"htwkalender/data-manager/service/events"
"htwkalender/data-manager/service/functions"
"htwkalender/data-manager/service/functions/time"
"log/slog"
)
func UpdateCourse(app *pocketbase.PocketBase) {
courses := events.GetAllCourses(app)
for _, course := range courses {
_, err := events.UpdateModulesForCourse(app, course)
currentSemesters := functions.CalculateSemesterList(time.RealClock{})
var seminarGroups []model.SeminarGroup
for _, semester := range currentSemesters {
seminarGroups = append(seminarGroups, events.GetAllCoursesForSemester(app, semester)...)
}
for _, seminarGroup := range seminarGroups {
_, err := events.UpdateModulesForCourse(app, seminarGroup)
if err != nil {
slog.Warn("Update Course: "+course+" failed:", "error", err)
slog.Warn("Update Course: "+seminarGroup.Course+" failed:", "error", err)
}
}
}

44
services/data-manager/service/db/dbGroups.go
View File

@ -20,6 +20,7 @@ import (
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase"
"github.com/pocketbase/pocketbase/models"
"htwkalender/data-manager/model"
"log/slog"
)
@ -38,6 +39,31 @@ func (s *SeminarGroup) TableName() string {
return "groups"
}
// UniqueKey Should be same as unique constraint in the database
func (s *SeminarGroup) UniqueKey() string {
return s.Course + s.Semester
}
func (s *SeminarGroup) toSeminarGroupModel() model.SeminarGroup {
return model.SeminarGroup{
University: s.University,
GroupShortcut: s.GroupShortcut,
GroupId: s.GroupId,
Course: s.Course,
Faculty: s.Faculty,
FacultyId: s.FacultyId,
Semester: s.Semester,
}
}
func (s *SeminarGroups) toSeminarGroupModels() []model.SeminarGroup {
var seminarGroups []model.SeminarGroup
for _, group := range *s {
seminarGroups = append(seminarGroups, group.toSeminarGroupModel())
}
return seminarGroups
}
type SeminarGroups []*SeminarGroup
func SaveGroups(seminarGroups SeminarGroups, app *pocketbase.PocketBase) (SeminarGroups, error) {
@ -84,26 +110,18 @@ func GetAllCourses(app *pocketbase.PocketBase) []string {
return courseArray
}
func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []string {
func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []model.SeminarGroup {
var courses []struct {
CourseShortcut string `db:"course" json:"course"`
}
var courses SeminarGroups
// get all courses for a specific semester
err := app.Dao().DB().Select("course").From("groups").Where(dbx.NewExp("semester = {:semester}", dbx.Params{"semester": semester})).All(&courses)
err := app.Dao().DB().Select("*").From("groups").Where(dbx.NewExp("semester = {:semester}", dbx.Params{"semester": semester})).All(&courses)
if err != nil {
slog.Error("Error while getting groups from database: ", "error", err)
return []string{}
return nil
}
var courseArray []string
for _, course := range courses {
courseArray = append(courseArray, course.CourseShortcut)
}
return courseArray
return courses.toSeminarGroupModels()
}

3
services/data-manager/service/events/courseService.go
View File

@ -18,6 +18,7 @@ package events
import (
"github.com/pocketbase/pocketbase"
"htwkalender/data-manager/model"
"htwkalender/data-manager/service/db"
"htwkalender/data-manager/service/functions"
)
@ -26,7 +27,7 @@ func GetAllCourses(app *pocketbase.PocketBase) []string {
return db.GetAllCourses(app)
}
func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []string {
func GetAllCoursesForSemester(app *pocketbase.PocketBase, semester string) []model.SeminarGroup {
return db.GetAllCoursesForSemester(app, semester)
}

10
services/data-manager/service/events/eventService.go
View File

@ -120,14 +120,12 @@ func DeleteAllEvents(app *pocketbase.PocketBase) error {
// 3. Save all events for the course and the semester
// If the update was successful, nil is returned
// If the update was not successful, an error is returned
func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Events, error) {
func UpdateModulesForCourse(app *pocketbase.PocketBase, seminarGroup model.SeminarGroup) (model.Events, error) {
seminarGroup, err := v1.GetSeminarGroupEventsFromHTML(course)
seminarGroup, err := v1.FetchAndParse(seminarGroup.Semester, seminarGroup.Course)
if err != nil {
return nil, err
}
seminarGroup = v1.ClearEmptySeminarGroups(seminarGroup)
seminarGroup = v1.ReplaceEmptyEventNames(seminarGroup)
//check if events in the seminarGroups Events are already in the database
@ -136,7 +134,7 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Ev
//if there are no events in the database, save the new events
//get all events for the course and the semester
dbEvents, err := db.GetAllEventsForCourse(app, course)
dbEvents, err := db.GetAllEventsForCourse(app, seminarGroup.Course)
if err != nil {
return nil, err
}
@ -182,7 +180,7 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Ev
return nil, err
}
slog.Info("Course: " + course + " - Event changes: " + strconv.FormatInt(int64(len(insertList)), 10) + " new events, " + strconv.FormatInt(int64(len(deleteList)), 10) + " deleted events")
slog.Info("Course: " + seminarGroup.Course + " - Event changes: " + strconv.FormatInt(int64(len(insertList)), 10) + " new events, " + strconv.FormatInt(int64(len(deleteList)), 10) + " deleted events")
return savedEvents, nil
}

62
services/data-manager/service/fetch/v1/fetchSeminarEventService.go
View File

@ -41,16 +41,12 @@ func ReplaceEmptyEventNames(group model.SeminarGroup) model.SeminarGroup {
return group
}
func ClearEmptySeminarGroups(seminarGroup model.SeminarGroup) model.SeminarGroup {
var newSeminarGroup = model.SeminarGroup{}
if len(seminarGroup.Events) > 0 && seminarGroup.Course != "" {
newSeminarGroup = seminarGroup
}
return newSeminarGroup
}
func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) {
// check that semester and seminarGroupLabel are not empty
if semester == "" || seminarGroupLabel == "" {
return "", fmt.Errorf("semester or seminarGroupLabel is empty")
}
url := "https://stundenplan.htwk-leipzig.de/" + semester + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65"
result, err := fetch.GetHTML(url)
if err != nil {
@ -60,22 +56,6 @@ func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) {
return result, nil
}
func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) (model.SeminarGroup, error) {
var seminarGroup [2]model.SeminarGroup
var errSS, errWS error
currentMonth := time.Now().Month()
if isSummerSemester(currentMonth) {
seminarGroup[0], errSS = fetchAndParse("ss", seminarGroupLabel)
}
if isWinterSemester(currentMonth) {
seminarGroup[1], errWS = fetchAndParse("ws", seminarGroupLabel)
}
return checkForSuccessfulFetch(errSS, errWS, seminarGroup)
}
func isSummerSemester(month time.Month) bool {
return month >= 3 && month <= 10
}
@ -84,7 +64,7 @@ func isWinterSemester(month time.Month) bool {
return month >= 9 || month <= 4
}
func fetchAndParse(season, label string) (model.SeminarGroup, error) {
func FetchAndParse(season, label string) (model.SeminarGroup, error) {
result, err := fetchHTMLFromURL(season, label)
if err != nil {
return model.SeminarGroup{}, err
@ -92,20 +72,6 @@ func fetchAndParse(season, label string) (model.SeminarGroup, error) {
return parseSeminarGroup(result), nil
}
func checkForSuccessfulFetch(errSS error, errWS error, seminarGroup [2]model.SeminarGroup) (model.SeminarGroup, error) {
switch {
case errSS != nil && errWS != nil:
return model.SeminarGroup{}, errWS
case errSS != nil:
return seminarGroup[1], nil
case errWS != nil:
return seminarGroup[0], nil
default:
seminarGroup[0].Events = append(seminarGroup[0].Events, seminarGroup[1].Events...)
return seminarGroup[0], nil
}
}
func SplitEventType(events []model.Event) ([]model.Event, error) {
re, err := regexp.Compile("^([VPS])([wp])$")
if err != nil {
@ -135,15 +101,21 @@ func parseSeminarGroup(result string) model.SeminarGroup {
eventTables := getEventTables(doc)
allDayLabels := getAllDayLabels(doc)
if eventTables == nil || allDayLabels == nil {
return model.SeminarGroup{}
}
course := findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data
semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data
semester, year := extractSemesterAndYear(semesterString)
if eventTables == nil || allDayLabels == nil {
return model.SeminarGroup{
University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data,
Course: course,
Events: []model.Event{},
}
}
eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels, course)
splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks)
events := splitEventsBySingleWeek(splitEventsByWeekVal)
semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data
semester, year := extractSemesterAndYear(semesterString)
events = convertWeeksToDates(events, semester, year)
events = generateUUIDs(events, course)
events, err = SplitEventType(events)

3
services/data-manager/service/fetch/v1/fetchSeminarGroupService.go
View File

@ -89,8 +89,9 @@ func removeDuplicates(groups db.SeminarGroups) db.SeminarGroups {
uniqueGroups := make(db.SeminarGroups, 0, len(groups))
seen := make(map[string]struct{}) // Use an empty struct to minimize memory usage
// unique Identifier is the course and semester
for _, group := range groups {
key := group.Course + group.Semester
key := group.UniqueKey()
if _, exists := seen[key]; !exists {
seen[key] = struct{}{}
uniqueGroups = append(uniqueGroups, group)