From a79f9c3eb9fe43d4390a2afc04ae84a3d191c01a Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:39:45 +0100 Subject: [PATCH 01/11] feat:#104 added fetcher for exams --- backend/service/addRoute.go | 4 +-- backend/service/addSchedule.go | 21 ++++++++++++ backend/service/fetch/v2/fetcher.go | 45 ++++++++++++++++++++++--- frontend/src/api/fetchCourse.ts | 2 ++ frontend/src/model/module.ts | 1 + frontend/src/view/AdditionalModules.vue | 19 +++++++++++ 6 files changed, 86 insertions(+), 6 deletions(-) diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index 79877fd..4dd7e7e 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -27,7 +27,7 @@ func AddRoutes(app *pocketbase.PocketBase) { }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), - apis.RequireAdminAuth(), + //apis.RequireAdminAuth(), }, }) if err != nil { @@ -45,7 +45,7 @@ func AddRoutes(app *pocketbase.PocketBase) { }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), - apis.RequireAdminAuth(), + //apis.RequireAdminAuth(), }, }) if err != nil { diff --git a/backend/service/addSchedule.go b/backend/service/addSchedule.go index bd0cd96..661c256 100644 --- a/backend/service/addSchedule.go +++ b/backend/service/addSchedule.go @@ -5,8 +5,12 @@ import ( "github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/tools/cron" "htwkalender/service/course" + "htwkalender/service/events" "htwkalender/service/feed" + v2 "htwkalender/service/fetch/v2" "htwkalender/service/functions/time" + "log" + "strconv" ) func AddSchedules(app *pocketbase.PocketBase) { @@ -26,6 +30,23 @@ func AddSchedules(app *pocketbase.PocketBase) { // clean feeds older than 6 months feed.ClearFeeds(app.Dao(), 6, time.RealClock{}) }) + + //delete all events and then fetch all events from remote this should be done every day at 4am + scheduler.MustAdd("fetchEvents", "0 4 * * *", func() { + err := events.DeleteAllEvents(app) + if err != nil { + log.Println(err) + } + + err, savedEvents := v2.FetchAllEventsAndSave(app) + if err != nil { + log.Println(err) + } else { + log.Println("Successfully saved: " + strconv.FormatInt(int64(len(savedEvents)), 10) + " events") + } + + }) + scheduler.Start() return nil }) diff --git a/backend/service/fetch/v2/fetcher.go b/backend/service/fetch/v2/fetcher.go index 6cafd63..72320bf 100644 --- a/backend/service/fetch/v2/fetcher.go +++ b/backend/service/fetch/v2/fetcher.go @@ -30,20 +30,45 @@ func FetchAllEventsAndSave(app *pocketbase.PocketBase) (error, []model.Event) { var savedRecords []model.Event var events []model.Event + var stubUrl = [2]string{ + "https://stundenplan.htwk-leipzig.de/", + "/Berichte/Text-Listen;Veranstaltungsarten;name;" + + "Vp%0A" + + "Vw%0A" + + "V%0A" + + "Sp%0A" + + "Sw%0A" + + "S%0A" + + "Pp%0A" + + "Pw%0A" + + "P%0A" + + "ZV%0A" + + "Tut%0A" + + "Sperr%0A" + + "pf%0A" + + "wpf%0A" + + "fak%0A" + + "Pruefung%0A" + + "Vertretung%0A" + + "Fremdveranst.%0A" + + "Buchen%0A" + + "%0A?&template=sws_modul&weeks=1-65&combined=yes", + } + if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) { - url := "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" + url := stubUrl[0] + "ss" + stubUrl[1] events, err = parseEventForOneSemester(url) savedEvents, dbError := db.SaveEvents(events, app) err = dbError - savedRecords = append(savedEvents, events...) + savedRecords = append(savedRecords, savedEvents...) } if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) { - url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" + url := stubUrl[0] + "ws" + stubUrl[1] events, err = parseEventForOneSemester(url) savedEvents, dbError := db.SaveEvents(events, app) err = dbError - savedRecords = append(savedEvents, events...) + savedRecords = append(savedRecords, savedEvents...) } return err, savedRecords } @@ -89,6 +114,8 @@ func parseEventForOneSemester(url string) ([]model.Event, error) { events = generateUUIDs(events) events = splitEventType(events) + events = switchNameAndNotesForPruefung(events) + var seminarGroup = model.SeminarGroup{ University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data, Events: events, @@ -101,6 +128,16 @@ func parseEventForOneSemester(url string) ([]model.Event, error) { return events, nil } +func switchNameAndNotesForPruefung(events []model.Event) []model.Event { + for i, event := range events { + if event.EventType == "Pruefung" { + events[i].Name = event.Notes + events[i].Notes = event.Name + } + } + return events +} + func parseHTML(err error, webpage string) (*html.Node, error) { doc, err := html.Parse(strings.NewReader(webpage)) if err != nil { diff --git a/frontend/src/api/fetchCourse.ts b/frontend/src/api/fetchCourse.ts index b1e164b..809a2c2 100644 --- a/frontend/src/api/fetchCourse.ts +++ b/frontend/src/api/fetchCourse.ts @@ -36,6 +36,7 @@ export async function fetchModulesByCourseAndSemester( module.uuid, module.name, course, + module.eventType, module.name, module.prof, semester, @@ -61,6 +62,7 @@ export async function fetchAllModules(): Promise { module.uuid, module.name, module.course, + module.eventType, module.name, module.prof, module.semester, diff --git a/frontend/src/model/module.ts b/frontend/src/model/module.ts index e08c18b..0b051e1 100644 --- a/frontend/src/model/module.ts +++ b/frontend/src/model/module.ts @@ -5,6 +5,7 @@ export class Module { public uuid: string, public name: string, public course: string, + public eventType: string, public userDefinedName: string, public prof: string, public semester: string, diff --git a/frontend/src/view/AdditionalModules.vue b/frontend/src/view/AdditionalModules.vue index 79284b3..a02ca54 100644 --- a/frontend/src/view/AdditionalModules.vue +++ b/frontend/src/view/AdditionalModules.vue @@ -33,6 +33,10 @@ const filters = ref({ value: null, matchMode: FilterMatchMode.CONTAINS, }, + eventType: { + value: null, + matchMode: FilterMatchMode.CONTAINS, + }, prof: { value: null, matchMode: FilterMatchMode.CONTAINS, @@ -156,6 +160,21 @@ function unselectModule(event: DataTableRowUnselectEvent) { /> + + + Date: Fri, 8 Dec 2023 11:41:14 +0100 Subject: [PATCH 02/11] feat:#104 added eventType to model --- frontend/src/api/fetchModule.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/api/fetchModule.ts b/frontend/src/api/fetchModule.ts index 85e843d..48da22d 100644 --- a/frontend/src/api/fetchModule.ts +++ b/frontend/src/api/fetchModule.ts @@ -14,6 +14,7 @@ export async function fetchModule(module: Module): Promise { module.uuid, module.name, module.course, + module.eventType, module.name, module.prof, module.semester, From ab84c2880f2df8fddd7aed5cc2843f8836a0b878 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:44:41 +0100 Subject: [PATCH 03/11] fix:#104 resecure api endpoint --- backend/service/addRoute.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index 4dd7e7e..79877fd 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -27,7 +27,7 @@ func AddRoutes(app *pocketbase.PocketBase) { }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), - //apis.RequireAdminAuth(), + apis.RequireAdminAuth(), }, }) if err != nil { @@ -45,7 +45,7 @@ func AddRoutes(app *pocketbase.PocketBase) { }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), - //apis.RequireAdminAuth(), + apis.RequireAdminAuth(), }, }) if err != nil { From 78e2602b19c3d98d804f79fda0d1090355695377 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:23:24 +0100 Subject: [PATCH 04/11] feat:#104 added additional check for switch --- backend/service/fetch/v2/fetcher.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/service/fetch/v2/fetcher.go b/backend/service/fetch/v2/fetcher.go index 72320bf..f04d8c5 100644 --- a/backend/service/fetch/v2/fetcher.go +++ b/backend/service/fetch/v2/fetcher.go @@ -114,7 +114,7 @@ func parseEventForOneSemester(url string) ([]model.Event, error) { events = generateUUIDs(events) events = splitEventType(events) - events = switchNameAndNotesForPruefung(events) + events = switchNameAndNotesForExam(events) var seminarGroup = model.SeminarGroup{ University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data, @@ -128,11 +128,14 @@ func parseEventForOneSemester(url string) ([]model.Event, error) { return events, nil } -func switchNameAndNotesForPruefung(events []model.Event) []model.Event { +// switch name and notes for Pruefung events when Note is not empty and Name starts with "Prüfungen" and contains email +func switchNameAndNotesForExam(events []model.Event) []model.Event { for i, event := range events { if event.EventType == "Pruefung" { - events[i].Name = event.Notes - events[i].Notes = event.Name + if event.Notes != "" && strings.HasPrefix(event.Name, "Prüfungen") && strings.Contains(event.Name, "@") { + events[i].Name = event.Notes + events[i].Notes = event.Name + } } } return events From 75417021476ac7a271efe2b0f649fcb143a9377b Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:27:55 +0100 Subject: [PATCH 05/11] test:#104 added test for name description switch for exam --- backend/service/fetch/v2/fetcher_test.go | 83 ++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 backend/service/fetch/v2/fetcher_test.go diff --git a/backend/service/fetch/v2/fetcher_test.go b/backend/service/fetch/v2/fetcher_test.go new file mode 100644 index 0000000..c0acff6 --- /dev/null +++ b/backend/service/fetch/v2/fetcher_test.go @@ -0,0 +1,83 @@ +package v2 + +import ( + "htwkalender/model" + "reflect" + "testing" +) + +func Test_switchNameAndNotesForExam(t *testing.T) { + type args struct { + events []model.Event + } + tests := []struct { + name string + args args + want []model.Event + }{ + { + name: "switch name and notes for exam", + args: args{ + events: []model.Event{ + { + EventType: "Pruefung", + Name: "Prüfungen FING/EIT WiSe (pruefungsamt.fing-eit@htwk-leipzig.de)", + Notes: "Computer Vision II - Räume/Zeit unter Vorbehalt- (Raum W111.1)", + }, + }, + }, + want: []model.Event{ + { + EventType: "Pruefung", + Name: "Computer Vision II - Räume/Zeit unter Vorbehalt- (Raum W111.1)", + Notes: "Prüfungen FING/EIT WiSe (pruefungsamt.fing-eit@htwk-leipzig.de)", + }, + }, + }, + { + name: "dont switch name and notes for exam", + args: args{ + events: []model.Event{ + { + EventType: "Pruefung", + Name: "i054 Umweltschutz und Recycling DPB & VNB 7.FS (wpf)", + Notes: "Prüfung", + }, + }, + }, + want: []model.Event{ + { + EventType: "Pruefung", + Notes: "Prüfung", + Name: "i054 Umweltschutz und Recycling DPB & VNB 7.FS (wpf)", + }, + }, + }, + { + name: "dont switch name and notes for exam", + args: args{ + events: []model.Event{ + { + EventType: "Pruefung", + Name: "Prüfungen FING/ME WiSe (pruefungsamt.fing-me@htwk-leipzig.de)", + Notes: "", + }, + }, + }, + want: []model.Event{ + { + EventType: "Pruefung", + Notes: "", + Name: "Prüfungen FING/ME WiSe (pruefungsamt.fing-me@htwk-leipzig.de)", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := switchNameAndNotesForExam(tt.args.events); !reflect.DeepEqual(got, tt.want) { + t.Errorf("switchNameAndNotesForExam() = %v, want %v", got, tt.want) + } + }) + } +} From 2e9dd764fe4e4508b551a64510f451b818afcf3d Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:43:37 +0100 Subject: [PATCH 06/11] lang:#104 added translation --- frontend/src/i18n/translations/de.json | 3 ++- frontend/src/i18n/translations/en.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/i18n/translations/de.json b/frontend/src/i18n/translations/de.json index a0b7b56..40893af 100644 --- a/frontend/src/i18n/translations/de.json +++ b/frontend/src/i18n/translations/de.json @@ -93,7 +93,8 @@ "from": "", "to": " bis ", "of": " von insgesamt " - } + }, + "eventType": "Ereignistyp" }, "renameModules": { "reminder": "Erinnerung", diff --git a/frontend/src/i18n/translations/en.json b/frontend/src/i18n/translations/en.json index e594cbc..5fba29a 100644 --- a/frontend/src/i18n/translations/en.json +++ b/frontend/src/i18n/translations/en.json @@ -93,7 +93,8 @@ "from": "", "to": " to ", "of": " of " - } + }, + "eventType": "event type" }, "renameModules": { "reminder": "reminder", From 6fa607fdd15c7ccf96b3fccbb17ea26ce180fd1c Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Thu, 28 Dec 2023 23:34:03 +0100 Subject: [PATCH 07/11] feat:#104 added whitespace trim for course --- backend/service/fetch/v2/eventParser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/service/fetch/v2/eventParser.go b/backend/service/fetch/v2/eventParser.go index 752bb24..d775603 100644 --- a/backend/service/fetch/v2/eventParser.go +++ b/backend/service/fetch/v2/eventParser.go @@ -35,7 +35,7 @@ func toEvents(tables [][]*html.Node, days []string) []model.Event { Prof: getTextContent(tableData[6]), Rooms: getTextContent(tableData[8]), BookedAt: getTextContent(tableData[10]), - Course: course, + Course: strings.TrimSpace(course), }) } } From b23326cfe74c24863ea57c20acb85f747e6e2348 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Thu, 28 Dec 2023 23:58:45 +0100 Subject: [PATCH 08/11] feat:#104 sport event name needs only title and id --- backend/service/fetch/sport/sportFetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go index 47ba277..62d91cf 100644 --- a/backend/service/fetch/sport/sportFetcher.go +++ b/backend/service/fetch/sport/sportFetcher.go @@ -89,7 +89,7 @@ func formatEntriesToEvents(entries []model.SportEntry) []model.Event { Week: strconv.Itoa(23), Start: start, End: end, - Name: entry.Title + " " + entry.Details.Type + " (" + entry.ID + ")", + Name: entry.Title + " (" + entry.ID + ")", EventType: entry.Details.Type, Prof: entry.Details.CourseLead.Name, Rooms: entry.Details.Location.Name, From e29ed1f12feeda902024372bec8eee79e1021df2 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Thu, 28 Dec 2023 23:59:09 +0100 Subject: [PATCH 09/11] feat:#104 moved eventTyp colum to new component --- .../src/components/AdditionalModuleTable.vue | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/frontend/src/components/AdditionalModuleTable.vue b/frontend/src/components/AdditionalModuleTable.vue index 7c0bf0f..3d38780 100644 --- a/frontend/src/components/AdditionalModuleTable.vue +++ b/frontend/src/components/AdditionalModuleTable.vue @@ -33,6 +33,10 @@ const filters = ref({ value: null, matchMode: FilterMatchMode.CONTAINS, }, + eventType: { + value: null, + matchMode: FilterMatchMode.CONTAINS, + }, prof: { value: null, matchMode: FilterMatchMode.CONTAINS, @@ -146,6 +150,21 @@ function unselectModule(event: DataTableRowUnselectEvent) { /> + + + Date: Fri, 29 Dec 2023 00:05:39 +0100 Subject: [PATCH 10/11] feat:#104 changed event selection to uuid --- backend/service/db/dbEvents.go | 10 ++-------- backend/service/db/dbEvents_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/backend/service/db/dbEvents.go b/backend/service/db/dbEvents.go index e301f00..37ed4cf 100644 --- a/backend/service/db/dbEvents.go +++ b/backend/service/db/dbEvents.go @@ -125,20 +125,14 @@ func buildIcalQueryForModules(modules []model.FeedCollection) dbx.Expression { //second check if modules has only one element if len(modules) == 1 { - return dbx.And( - dbx.HashExp{"Name": modules[0].Name}, - dbx.HashExp{"course": modules[0].Course}, - ) + return dbx.HashExp{"uuid": modules[0].UUID} } //third check if modules has more than one element var wheres []dbx.Expression for _, module := range modules { - where := dbx.And( - dbx.HashExp{"Name": module.Name}, - dbx.HashExp{"course": module.Course}, - ) + where := dbx.HashExp{"uuid": module.UUID} wheres = append(wheres, where) } diff --git a/backend/service/db/dbEvents_test.go b/backend/service/db/dbEvents_test.go index 8d46131..b8fa224 100644 --- a/backend/service/db/dbEvents_test.go +++ b/backend/service/db/dbEvents_test.go @@ -23,13 +23,13 @@ func Test_buildIcalQueryForModules(t *testing.T) { }, { name: "one module", - args: args{modules: []model.FeedCollection{{Name: "test", Course: "test"}}}, - want: dbx.And(dbx.HashExp{"Name": "test"}, dbx.HashExp{"course": "test"}), + args: args{modules: []model.FeedCollection{{Name: "test", Course: "test", UUID: "test"}}}, + want: dbx.HashExp{"uuid": "test"}, }, { name: "two modules", - args: args{modules: []model.FeedCollection{{Name: "test", Course: "test"}, {Name: "test2", Course: "test2"}}}, - want: dbx.Or(dbx.And(dbx.HashExp{"Name": "test"}, dbx.HashExp{"course": "test"}), dbx.And(dbx.HashExp{"Name": "test2"}, dbx.HashExp{"course": "test2"})), + args: args{modules: []model.FeedCollection{{Name: "test", Course: "test", UUID: "test"}, {Name: "test2", Course: "test2", UUID: "test2"}}}, + want: dbx.Or(dbx.HashExp{"uuid": "test"}, dbx.HashExp{"uuid": "test2"}), }, } for _, tt := range tests { From 8a1214e80c25482ec847ed83d165b341bc8ad689 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Fri, 29 Dec 2023 03:29:05 +0100 Subject: [PATCH 11/11] feat:#104 reordered schedule --- backend/service/addSchedule.go | 13 ++++++------- backend/service/fetch/sport/sportFetcher.go | 6 ++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/service/addSchedule.go b/backend/service/addSchedule.go index 9009776..56763a7 100644 --- a/backend/service/addSchedule.go +++ b/backend/service/addSchedule.go @@ -26,20 +26,19 @@ func AddSchedules(app *pocketbase.PocketBase) { course.UpdateCourse(app) }) - // Every sunday at 3am clean all courses (5 segments - minute, hour, day, month, weekday) "0 3 * * 0" - scheduler.MustAdd("cleanFeeds", "0 3 * * 0", func() { + // Every sunday at 1am clean all courses (5 segments - minute, hour, day, month, weekday) "0 3 * * 0" + scheduler.MustAdd("cleanFeeds", "0 1 * * 0", func() { // clean feeds older than 6 months feed.ClearFeeds(app.Dao(), 6, time.RealClock{}) }) - // Every sunday at 2am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0" - scheduler.MustAdd("fetchEvents", "0 2 * * 0", func() { + // Every sunday at 3am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0" + scheduler.MustAdd("fetchSportEvents", "0 3 * * 0", func() { sport.FetchAndUpdateSportEvents(app) }) - - //delete all events and then fetch all events from remote this should be done every day at 4am - scheduler.MustAdd("fetchEvents", "0 4 * * *", func() { + //delete all events and then fetch all events from remote this should be done every sunday at 2am + scheduler.MustAdd("fetchEvents", "0 2 * * 0", func() { err := events.DeleteAllEvents(app) if err != nil { log.Println(err) diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go index 62d91cf..1bff08a 100644 --- a/backend/service/fetch/sport/sportFetcher.go +++ b/backend/service/fetch/sport/sportFetcher.go @@ -18,8 +18,9 @@ import ( "github.com/PuerkitoBio/goquery" ) -// @TODO: add tests -// @TODO: make it like a cron job to fetch the sport courses once a week +// FetchAndUpdateSportEvents fetches all sport events from the HTWK sport website +// it deletes them first and then saves them to the database +// It returns all saved events func FetchAndUpdateSportEvents(app *pocketbase.PocketBase) []model.Event { var sportCourseLinks = fetchAllAvailableSportCourses() @@ -56,6 +57,7 @@ func FetchAndUpdateSportEvents(app *pocketbase.PocketBase) []model.Event { } } + // @TODO: delete and save events in one transaction and it only should delete events that are not in the new events list and save events that are not in the database err = db.DeleteAllEventsForCourse(app, "Sport", functions.GetCurrentSemesterString()) if err != nil { return nil