From c9caa2aedc29f7cfc3c29fb64613e26a0af2f858 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Thu, 30 Nov 2023 23:31:04 +0100 Subject: [PATCH] feat:#52 added ss and ws month fix for fetching --- README.md | 6 +-- backend/service/addRoute.go | 59 ++++++++++------------------- backend/service/addSchedule.go | 2 +- backend/service/fetch/v2/fetcher.go | 58 +++++++++++++++++++--------- 4 files changed, 66 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 7d59d51..67fa014 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,15 @@ This should be done quick in a few seconds (0-5s). When you execute the command again, it will update the groups in the database and only return new added groups. -http://127.0.0.1/api/fetchGroups +http://127.0.0.1/api/fetch/groups For fetching the plans, you can use the following command. This will fetch all plans for all groups and store the events in the database. It's done for all current existing events (ws/ss). -The whole process takes a while (1-5min), depending on the amount of groups and events. +The whole process takes a while (30s-2min), depending on the amount of groups and events. Stay for this time on the page and wait for the response. -http://127.0.0.1/api/fetchPlans +http://127.0.0.1/api/fetch/events ### View/Filter/Search in Admin UI diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index b662273..82c33de 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -2,7 +2,7 @@ package service import ( "htwkalender/service/events" - "htwkalender/service/fetch/v1" + v1 "htwkalender/service/fetch/v1" v2 "htwkalender/service/fetch/v2" "htwkalender/service/ical" "htwkalender/service/room" @@ -20,45 +20,10 @@ func AddRoutes(app *pocketbase.PocketBase) { app.OnBeforeServe().Add(func(e *core.ServeEvent) error { _, err := e.Router.AddRoute(echo.Route{ Method: http.MethodGet, - Path: "/api/fetchPlans", - Handler: func(c echo.Context) error { - return v1.GetSeminarEvents(c, app) - }, - Middlewares: []echo.MiddlewareFunc{ - apis.ActivityLogger(app), - //apis.RequireAdminAuth(), - }, - }) - if err != nil { - return err - } - return nil - }) - - app.OnBeforeServe().Add(func(e *core.ServeEvent) error { - _, err := e.Router.AddRoute(echo.Route{ - Method: http.MethodGet, - Path: "/api/v2/fetch", + Path: "/api/fetch/events", Handler: func(c echo.Context) error { return v2.ParseEventsFromRemote(c, app) }, - Middlewares: []echo.MiddlewareFunc{ - apis.ActivityLogger(app), - }, - }) - if err != nil { - return err - } - return nil - }) - - app.OnBeforeServe().Add(func(e *core.ServeEvent) error { - _, err := e.Router.AddRoute(echo.Route{ - Method: http.MethodDelete, - Path: "/api/modules", - Handler: func(c echo.Context) error { - return events.DeleteAllEvents(app) - }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), apis.RequireAdminAuth(), @@ -73,7 +38,7 @@ func AddRoutes(app *pocketbase.PocketBase) { app.OnBeforeServe().Add(func(e *core.ServeEvent) error { _, err := e.Router.AddRoute(echo.Route{ Method: http.MethodGet, - Path: "/api/fetchGroups", + Path: "/api/fetch/groups", Handler: func(c echo.Context) error { return v1.SeminarGroups(c, app) }, @@ -88,6 +53,24 @@ func AddRoutes(app *pocketbase.PocketBase) { return nil }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + _, err := e.Router.AddRoute(echo.Route{ + Method: http.MethodDelete, + Path: "/api/modules", + Handler: func(c echo.Context) error { + return events.DeleteAllEvents(app) + }, + Middlewares: []echo.MiddlewareFunc{ + apis.ActivityLogger(app), + apis.RequireAdminAuth(), + }, + }) + if err != nil { + return err + } + return nil + }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { _, err := e.Router.AddRoute(echo.Route{ Method: http.MethodGet, diff --git a/backend/service/addSchedule.go b/backend/service/addSchedule.go index bd0cd96..e3df317 100644 --- a/backend/service/addSchedule.go +++ b/backend/service/addSchedule.go @@ -17,7 +17,7 @@ func AddSchedules(app *pocketbase.PocketBase) { // Every hour update all courses (5 segments - minute, hour, day, month, weekday) "0 * * * *" // Every three hours update all courses (5 segments - minute, hour, day, month, weekday) "0 */3 * * *" // Every 10 minutes update all courses (5 segments - minute, hour, day, month, weekday) "*/10 * * * *" - scheduler.MustAdd("updateCourse", "0 */3 * * *", func() { + scheduler.MustAdd("updateCourse", "*/10 * * * *", func() { course.UpdateCourse(app) }) diff --git a/backend/service/fetch/v2/fetcher.go b/backend/service/fetch/v2/fetcher.go index 6205927..b163d54 100644 --- a/backend/service/fetch/v2/fetcher.go +++ b/backend/service/fetch/v2/fetcher.go @@ -10,22 +10,55 @@ import ( "htwkalender/service/fetch" "strconv" "strings" + "time" ) func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error { - url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" + err, savedRecords := FetchAllEventsAndSave(app) + if err != nil { + return err + } else { + savedRecordsLength := strconv.FormatInt(int64(len(savedRecords)), 10) + return c.JSON(200, "Successfully saved "+savedRecordsLength+" events") + } +} + +func FetchAllEventsAndSave(app *pocketbase.PocketBase) (error, []model.Event) { + var err error + var savedRecords []model.Event + var events []model.Event + + if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) { + url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" + events, err = parseEventForOneSemester(url) + savedEvents, dbError := db.SaveEvents(events, app) + err = dbError + savedRecords = append(savedEvents, events...) + } + + if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) { + url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" + events, err = parseEventForOneSemester(url) + savedEvents, dbError := db.SaveEvents(events, app) + err = dbError + savedRecords = append(savedEvents, events...) + } + return err, savedRecords +} + +func parseEventForOneSemester(url string) ([]model.Event, error) { // Fetch Webpage from URL webpage, err := fetch.GetHTML(url) if err != nil { - return err + return nil, err } // Parse HTML to Node Tree doc, err2 := parseHTML(err, webpage) if err2 != nil { - return err2 + return nil, err2 } // Get all event tables and all day labels @@ -33,26 +66,24 @@ func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error { allDayLabels := getAllDayLabels(doc) if eventTables == nil || allDayLabels == nil { - return err + return nil, err } eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels) if eventsWithCombinedWeeks == nil { - return err + return nil, err } splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks) events := splitEventsBySingleWeek(splitEventsByWeekVal) if events == nil { - return err + return nil, err } - // Search Semester String in Page Head table := findFirstTable(doc) semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data - semester, year := extractSemesterAndYear(semesterString) events = convertWeeksToDates(events, semester, year) events = generateUUIDs(events) @@ -64,17 +95,10 @@ func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error { } if seminarGroup.Events == nil && seminarGroup.University == "" { - return err + return nil, err } - savedRecords, dbError := db.SaveEvents(events, app) - - if dbError != nil { - return dbError - } else { - savedRecordsLength := strconv.FormatInt(int64(len(savedRecords)), 10) - return c.JSON(200, "Successfully saved "+savedRecordsLength+" events") - } + return events, nil } func parseHTML(err error, webpage string) (*html.Node, error) {