mirror of
https://gitlab.dit.htwk-leipzig.de/htwk-software/htwkalender-pwa.git
synced 2025-08-07 04:09:17 +02:00
feat:#52 added ss and ws month fix for fetching
This commit is contained in:
@@ -48,15 +48,15 @@ This should be done quick in a few seconds (0-5s).
|
|||||||
When you execute the command again, it will update the groups in the
|
When you execute the command again, it will update the groups in the
|
||||||
database and only return new added groups.
|
database and only return new added groups.
|
||||||
|
|
||||||
http://127.0.0.1/api/fetchGroups
|
http://127.0.0.1/api/fetch/groups
|
||||||
|
|
||||||
For fetching the plans, you can use the following command.
|
For fetching the plans, you can use the following command.
|
||||||
This will fetch all plans for all groups and store the events in the database.
|
This will fetch all plans for all groups and store the events in the database.
|
||||||
It's done for all current existing events (ws/ss).
|
It's done for all current existing events (ws/ss).
|
||||||
The whole process takes a while (1-5min), depending on the amount of groups and events.
|
The whole process takes a while (30s-2min), depending on the amount of groups and events.
|
||||||
Stay for this time on the page and wait for the response.
|
Stay for this time on the page and wait for the response.
|
||||||
|
|
||||||
http://127.0.0.1/api/fetchPlans
|
http://127.0.0.1/api/fetch/events
|
||||||
|
|
||||||
|
|
||||||
### View/Filter/Search in Admin UI
|
### View/Filter/Search in Admin UI
|
||||||
|
@@ -2,7 +2,7 @@ package service
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"htwkalender/service/events"
|
"htwkalender/service/events"
|
||||||
"htwkalender/service/fetch/v1"
|
v1 "htwkalender/service/fetch/v1"
|
||||||
v2 "htwkalender/service/fetch/v2"
|
v2 "htwkalender/service/fetch/v2"
|
||||||
"htwkalender/service/ical"
|
"htwkalender/service/ical"
|
||||||
"htwkalender/service/room"
|
"htwkalender/service/room"
|
||||||
@@ -20,45 +20,10 @@ func AddRoutes(app *pocketbase.PocketBase) {
|
|||||||
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
||||||
_, err := e.Router.AddRoute(echo.Route{
|
_, err := e.Router.AddRoute(echo.Route{
|
||||||
Method: http.MethodGet,
|
Method: http.MethodGet,
|
||||||
Path: "/api/fetchPlans",
|
Path: "/api/fetch/events",
|
||||||
Handler: func(c echo.Context) error {
|
|
||||||
return v1.GetSeminarEvents(c, app)
|
|
||||||
},
|
|
||||||
Middlewares: []echo.MiddlewareFunc{
|
|
||||||
apis.ActivityLogger(app),
|
|
||||||
//apis.RequireAdminAuth(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
|
||||||
_, err := e.Router.AddRoute(echo.Route{
|
|
||||||
Method: http.MethodGet,
|
|
||||||
Path: "/api/v2/fetch",
|
|
||||||
Handler: func(c echo.Context) error {
|
Handler: func(c echo.Context) error {
|
||||||
return v2.ParseEventsFromRemote(c, app)
|
return v2.ParseEventsFromRemote(c, app)
|
||||||
},
|
},
|
||||||
Middlewares: []echo.MiddlewareFunc{
|
|
||||||
apis.ActivityLogger(app),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
|
||||||
_, err := e.Router.AddRoute(echo.Route{
|
|
||||||
Method: http.MethodDelete,
|
|
||||||
Path: "/api/modules",
|
|
||||||
Handler: func(c echo.Context) error {
|
|
||||||
return events.DeleteAllEvents(app)
|
|
||||||
},
|
|
||||||
Middlewares: []echo.MiddlewareFunc{
|
Middlewares: []echo.MiddlewareFunc{
|
||||||
apis.ActivityLogger(app),
|
apis.ActivityLogger(app),
|
||||||
apis.RequireAdminAuth(),
|
apis.RequireAdminAuth(),
|
||||||
@@ -73,7 +38,7 @@ func AddRoutes(app *pocketbase.PocketBase) {
|
|||||||
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
||||||
_, err := e.Router.AddRoute(echo.Route{
|
_, err := e.Router.AddRoute(echo.Route{
|
||||||
Method: http.MethodGet,
|
Method: http.MethodGet,
|
||||||
Path: "/api/fetchGroups",
|
Path: "/api/fetch/groups",
|
||||||
Handler: func(c echo.Context) error {
|
Handler: func(c echo.Context) error {
|
||||||
return v1.SeminarGroups(c, app)
|
return v1.SeminarGroups(c, app)
|
||||||
},
|
},
|
||||||
@@ -88,6 +53,24 @@ func AddRoutes(app *pocketbase.PocketBase) {
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
||||||
|
_, err := e.Router.AddRoute(echo.Route{
|
||||||
|
Method: http.MethodDelete,
|
||||||
|
Path: "/api/modules",
|
||||||
|
Handler: func(c echo.Context) error {
|
||||||
|
return events.DeleteAllEvents(app)
|
||||||
|
},
|
||||||
|
Middlewares: []echo.MiddlewareFunc{
|
||||||
|
apis.ActivityLogger(app),
|
||||||
|
apis.RequireAdminAuth(),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
||||||
_, err := e.Router.AddRoute(echo.Route{
|
_, err := e.Router.AddRoute(echo.Route{
|
||||||
Method: http.MethodGet,
|
Method: http.MethodGet,
|
||||||
|
@@ -17,7 +17,7 @@ func AddSchedules(app *pocketbase.PocketBase) {
|
|||||||
// Every hour update all courses (5 segments - minute, hour, day, month, weekday) "0 * * * *"
|
// Every hour update all courses (5 segments - minute, hour, day, month, weekday) "0 * * * *"
|
||||||
// Every three hours update all courses (5 segments - minute, hour, day, month, weekday) "0 */3 * * *"
|
// Every three hours update all courses (5 segments - minute, hour, day, month, weekday) "0 */3 * * *"
|
||||||
// Every 10 minutes update all courses (5 segments - minute, hour, day, month, weekday) "*/10 * * * *"
|
// Every 10 minutes update all courses (5 segments - minute, hour, day, month, weekday) "*/10 * * * *"
|
||||||
scheduler.MustAdd("updateCourse", "0 */3 * * *", func() {
|
scheduler.MustAdd("updateCourse", "*/10 * * * *", func() {
|
||||||
course.UpdateCourse(app)
|
course.UpdateCourse(app)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@@ -10,22 +10,55 @@ import (
|
|||||||
"htwkalender/service/fetch"
|
"htwkalender/service/fetch"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error {
|
func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error {
|
||||||
|
|
||||||
url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes"
|
err, savedRecords := FetchAllEventsAndSave(app)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
} else {
|
||||||
|
savedRecordsLength := strconv.FormatInt(int64(len(savedRecords)), 10)
|
||||||
|
return c.JSON(200, "Successfully saved "+savedRecordsLength+" events")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func FetchAllEventsAndSave(app *pocketbase.PocketBase) (error, []model.Event) {
|
||||||
|
var err error
|
||||||
|
var savedRecords []model.Event
|
||||||
|
var events []model.Event
|
||||||
|
|
||||||
|
if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) {
|
||||||
|
url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes"
|
||||||
|
events, err = parseEventForOneSemester(url)
|
||||||
|
savedEvents, dbError := db.SaveEvents(events, app)
|
||||||
|
err = dbError
|
||||||
|
savedRecords = append(savedEvents, events...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) {
|
||||||
|
url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes"
|
||||||
|
events, err = parseEventForOneSemester(url)
|
||||||
|
savedEvents, dbError := db.SaveEvents(events, app)
|
||||||
|
err = dbError
|
||||||
|
savedRecords = append(savedEvents, events...)
|
||||||
|
}
|
||||||
|
return err, savedRecords
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseEventForOneSemester(url string) ([]model.Event, error) {
|
||||||
// Fetch Webpage from URL
|
// Fetch Webpage from URL
|
||||||
webpage, err := fetch.GetHTML(url)
|
webpage, err := fetch.GetHTML(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse HTML to Node Tree
|
// Parse HTML to Node Tree
|
||||||
doc, err2 := parseHTML(err, webpage)
|
doc, err2 := parseHTML(err, webpage)
|
||||||
if err2 != nil {
|
if err2 != nil {
|
||||||
return err2
|
return nil, err2
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get all event tables and all day labels
|
// Get all event tables and all day labels
|
||||||
@@ -33,26 +66,24 @@ func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error {
|
|||||||
allDayLabels := getAllDayLabels(doc)
|
allDayLabels := getAllDayLabels(doc)
|
||||||
|
|
||||||
if eventTables == nil || allDayLabels == nil {
|
if eventTables == nil || allDayLabels == nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels)
|
eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels)
|
||||||
|
|
||||||
if eventsWithCombinedWeeks == nil {
|
if eventsWithCombinedWeeks == nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks)
|
splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks)
|
||||||
events := splitEventsBySingleWeek(splitEventsByWeekVal)
|
events := splitEventsBySingleWeek(splitEventsByWeekVal)
|
||||||
|
|
||||||
if events == nil {
|
if events == nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search Semester String in Page Head
|
|
||||||
table := findFirstTable(doc)
|
table := findFirstTable(doc)
|
||||||
semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data
|
semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data
|
||||||
|
|
||||||
semester, year := extractSemesterAndYear(semesterString)
|
semester, year := extractSemesterAndYear(semesterString)
|
||||||
events = convertWeeksToDates(events, semester, year)
|
events = convertWeeksToDates(events, semester, year)
|
||||||
events = generateUUIDs(events)
|
events = generateUUIDs(events)
|
||||||
@@ -64,17 +95,10 @@ func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if seminarGroup.Events == nil && seminarGroup.University == "" {
|
if seminarGroup.Events == nil && seminarGroup.University == "" {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
savedRecords, dbError := db.SaveEvents(events, app)
|
return events, nil
|
||||||
|
|
||||||
if dbError != nil {
|
|
||||||
return dbError
|
|
||||||
} else {
|
|
||||||
savedRecordsLength := strconv.FormatInt(int64(len(savedRecords)), 10)
|
|
||||||
return c.JSON(200, "Successfully saved "+savedRecordsLength+" events")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseHTML(err error, webpage string) (*html.Node, error) {
|
func parseHTML(err error, webpage string) (*html.Node, error) {
|
||||||
|
Reference in New Issue
Block a user