feature:#150 added slog and changed signatures

This commit is contained in:
Elmar Kresse
2024-01-21 17:53:49 +01:00
parent a9e76ee3db
commit 031fa717d7
9 changed files with 77 additions and 60 deletions

View File

@@ -9,6 +9,7 @@ import (
"htwkalender/service/ical" "htwkalender/service/ical"
"htwkalender/service/room" "htwkalender/service/room"
"io" "io"
"log/slog"
"net/http" "net/http"
"github.com/labstack/echo/v5" "github.com/labstack/echo/v5"
@@ -24,7 +25,13 @@ func AddRoutes(app *pocketbase.PocketBase) {
Method: http.MethodGet, Method: http.MethodGet,
Path: "/api/fetch/events", Path: "/api/fetch/events",
Handler: func(c echo.Context) error { Handler: func(c echo.Context) error {
return v2.ParseEventsFromRemote(c, app) savedEvents, err := v2.ParseEventsFromRemote(app)
if err != nil {
slog.Error("Failed to parse events from remote: %v", err)
return c.JSON(http.StatusInternalServerError, "Failed to parse events from remote")
} else {
return c.JSON(http.StatusOK, savedEvents)
}
}, },
Middlewares: []echo.MiddlewareFunc{ Middlewares: []echo.MiddlewareFunc{
apis.ActivityLogger(app), apis.ActivityLogger(app),

View File

@@ -8,6 +8,7 @@ import (
"htwkalender/service/feed" "htwkalender/service/feed"
"htwkalender/service/fetch/sport" "htwkalender/service/fetch/sport"
"htwkalender/service/functions/time" "htwkalender/service/functions/time"
"log/slog"
) )
func AddSchedules(app *pocketbase.PocketBase) { func AddSchedules(app *pocketbase.PocketBase) {
@@ -19,17 +20,20 @@ func AddSchedules(app *pocketbase.PocketBase) {
// Every three hours update all courses (5 segments - minute, hour, day, month, weekday) "0 */3 * * *" // Every three hours update all courses (5 segments - minute, hour, day, month, weekday) "0 */3 * * *"
// Every 10 minutes update all courses (5 segments - minute, hour, day, month, weekday) "*/10 * * * *" // Every 10 minutes update all courses (5 segments - minute, hour, day, month, weekday) "*/10 * * * *"
scheduler.MustAdd("updateCourse", "0 */3 * * *", func() { scheduler.MustAdd("updateCourse", "0 */3 * * *", func() {
slog.Info("Started updating courses schedule")
course.UpdateCourse(app) course.UpdateCourse(app)
}) })
// Every sunday at 3am clean all courses (5 segments - minute, hour, day, month, weekday) "0 3 * * 0" // Every sunday at 3am clean all courses (5 segments - minute, hour, day, month, weekday) "0 3 * * 0"
scheduler.MustAdd("cleanFeeds", "0 3 * * 0", func() { scheduler.MustAdd("cleanFeeds", "0 3 * * 0", func() {
// clean feeds older than 6 months // clean feeds older than 6 months
slog.Info("Started cleaning feeds schedule")
feed.ClearFeeds(app.Dao(), 6, time.RealClock{}) feed.ClearFeeds(app.Dao(), 6, time.RealClock{})
}) })
// Every sunday at 2am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0" // Every sunday at 2am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0"
scheduler.MustAdd("fetchEvents", "0 2 * * 0", func() { scheduler.MustAdd("fetchEvents", "0 2 * * 0", func() {
slog.Info("Started fetching sport events schedule")
sport.FetchAndUpdateSportEvents(app) sport.FetchAndUpdateSportEvents(app)
}) })

View File

@@ -3,18 +3,18 @@ package course
import ( import (
"github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase"
"htwkalender/service/events" "htwkalender/service/events"
"log" "log/slog"
"strconv"
) )
func UpdateCourse(app *pocketbase.PocketBase) { func UpdateCourse(app *pocketbase.PocketBase) {
courses := events.GetAllCourses(app) courses := events.GetAllCourses(app)
for _, course := range courses { for _, course := range courses {
err := events.UpdateModulesForCourse(app, course) savedEvents, err := events.UpdateModulesForCourse(app, course)
if err != nil { if err != nil {
log.Println("Update Course: " + course + " failed") slog.Warn("Update Course: "+course+" failed: ", err.Error())
log.Println(err)
} else { } else {
log.Println("Update Course: " + course + " successful") slog.Info("Updated Course: " + course + " with " + strconv.FormatInt(int64(len(savedEvents)), 10) + " events")
} }
} }
} }

View File

@@ -1,7 +1,9 @@
package db package db
import ( import (
"fmt"
"htwkalender/model" "htwkalender/model"
"log/slog"
"time" "time"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
@@ -187,8 +189,8 @@ func GetAllModulesForCourse(app *pocketbase.PocketBase, course string, semester
// get all events from event records in the events collection // get all events from event records in the events collection
err := app.Dao().DB().Select("*").From("events").Where(dbx.NewExp("course = {:course} AND semester = {:semester}", dbx.Params{"course": course, "semester": semester})).GroupBy("Name").Distinct(true).All(&events) err := app.Dao().DB().Select("*").From("events").Where(dbx.NewExp("course = {:course} AND semester = {:semester}", dbx.Params{"course": course, "semester": semester})).GroupBy("Name").Distinct(true).All(&events)
if err != nil { if err != nil {
print("Error while getting events from database: ", err) slog.Error("Error while getting events from database: ", err)
return nil, err return nil, fmt.Errorf("error while getting events from database for course %s and semester %s", course, semester)
} }
return events, nil return events, nil
@@ -199,7 +201,8 @@ func GetAllModulesDistinctByNameAndCourse(app *pocketbase.PocketBase) (model.Eve
err := app.Dao().DB().Select("*").From("events").GroupBy("Name").Distinct(true).All(&events) err := app.Dao().DB().Select("*").From("events").GroupBy("Name").Distinct(true).All(&events)
if err != nil { if err != nil {
return nil, err slog.Error("Error while getting events from database: ", err)
return nil, fmt.Errorf("error while getting events distinct by name and course from data")
} }
return events, nil return events, nil

View File

@@ -3,7 +3,6 @@ package events
import ( import (
"github.com/labstack/echo/v5" "github.com/labstack/echo/v5"
"github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase"
"github.com/pocketbase/pocketbase/apis"
"htwkalender/model" "htwkalender/model"
"htwkalender/service/db" "htwkalender/service/db"
"htwkalender/service/fetch/v1" "htwkalender/service/fetch/v1"
@@ -89,7 +88,7 @@ func DeleteAllEvents(app *pocketbase.PocketBase) error {
// 3. Save all events for the course and the semester // 3. Save all events for the course and the semester
// If the update was successful, nil is returned // If the update was successful, nil is returned
// If the update was not successful, an error is returned // If the update was not successful, an error is returned
func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) error { func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Events, error) {
//new string array with one element (course) //new string array with one element (course)
var courses []string var courses []string
@@ -109,29 +108,32 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) error {
//get all events for the course and the semester //get all events for the course and the semester
events, err := db.GetAllModulesForCourse(app, course, "ws") events, err := db.GetAllModulesForCourse(app, course, "ws")
if err != nil { if err != nil {
return apis.NewNotFoundError("Events for winter semester could not be found", err) return nil, err
} }
// append all events for the course and the semester to the events array for ss // append all events for the course and the semester to the events array for ss
summerEvents, err := db.GetAllModulesForCourse(app, course, "ss") summerEvents, err := db.GetAllModulesForCourse(app, course, "ss")
if err != nil { if err != nil {
return apis.NewNotFoundError("Events for summer semester could not be found", err) return nil, err
} }
events = append(events, summerEvents...) events = append(events, summerEvents...)
//if there are no events in the database, save the new events //if there are no events in the database, save the new events
if len(events) == 0 { if len(events) == 0 {
_, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) events, dbError := db.SaveSeminarGroupEvents(seminarGroups, app)
if dbError != nil { if dbError != nil {
return apis.NewNotFoundError("Events could not be saved", dbError) return nil, dbError
} }
return nil return events, nil
} }
//check if events in the seminarGroups Events are already in the database //check if events in the seminarGroups Events are already in the database
//if yes, keep the database as it is //if yes, keep the database as it is
//if no, delete all events for the course and the semester and save the new events //if no, delete all events for the course and the semester and save the new events
var savedEvents model.Events
for _, seminarGroup := range seminarGroups { for _, seminarGroup := range seminarGroups {
for _, event := range seminarGroup.Events { for _, event := range seminarGroup.Events {
// if the event is not in the database, delete all events for the course and the semester and save the new events // if the event is not in the database, delete all events for the course and the semester and save the new events
@@ -139,25 +141,24 @@ func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) error {
err = DeleteAllEventsByCourseAndSemester(app, course, "ws") err = DeleteAllEventsByCourseAndSemester(app, course, "ws")
if err != nil { if err != nil {
return err return nil, err
} }
err = DeleteAllEventsByCourseAndSemester(app, course, "ss") err = DeleteAllEventsByCourseAndSemester(app, course, "ss")
if err != nil { if err != nil {
return err return nil, err
} }
//save the new events //save the new events
_, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) savedEvent, dbError := db.SaveSeminarGroupEvents(seminarGroups, app)
if dbError != nil { if dbError != nil {
return apis.NewNotFoundError("Events could not be saved", dbError) return nil, dbError
} }
return nil savedEvents = append(savedEvents, savedEvent...)
} }
} }
} }
return savedEvents, nil
return nil
} }
func ContainsEvent(events model.Events, event model.Event) bool { func ContainsEvent(events model.Events, event model.Event) bool {

View File

@@ -30,7 +30,7 @@ func GetSeminarEvents(c echo.Context, app *pocketbase.PocketBase) error {
seminarGroups = ReplaceEmptyEventNames(seminarGroups) seminarGroups = ReplaceEmptyEventNames(seminarGroups)
savedRecords, dbError := db.SaveSeminarGroupEvents(seminarGroups, app) dbError, savedRecords := db.SaveSeminarGroupEvents(seminarGroups, app)
if dbError != nil { if dbError != nil {
return apis.NewNotFoundError("Events could not be saved", dbError.Error()) return apis.NewNotFoundError("Events could not be saved", dbError.Error())

View File

@@ -1,51 +1,54 @@
package v2 package v2
import ( import (
"fmt"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/labstack/echo/v5"
"github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase"
"golang.org/x/net/html" "golang.org/x/net/html"
"htwkalender/model" "htwkalender/model"
"htwkalender/service/db" "htwkalender/service/db"
"htwkalender/service/fetch" "htwkalender/service/fetch"
"strconv" localTime "htwkalender/service/functions/time"
"strings" "strings"
"time"
) )
func ParseEventsFromRemote(c echo.Context, app *pocketbase.PocketBase) error { func ParseEventsFromRemote(app *pocketbase.PocketBase) (model.Events, error) {
savedRecords, err := FetchAllEventsAndSave(app, localTime.RealClock{})
err, savedRecords := FetchAllEventsAndSave(app)
if err != nil { if err != nil {
return err return nil, err
} else {
savedRecordsLength := strconv.FormatInt(int64(len(savedRecords)), 10)
return c.JSON(200, "Successfully saved "+savedRecordsLength+" events")
} }
return savedRecords, nil
} }
func FetchAllEventsAndSave(app *pocketbase.PocketBase) (error, []model.Event) { func FetchAllEventsAndSave(app *pocketbase.PocketBase, clock localTime.Clock) ([]model.Event, error) {
var err error
var savedRecords []model.Event var savedRecords []model.Event
var events []model.Event
if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) { if (clock.Now().Month() >= 3) && (clock.Now().Month() <= 10) {
url := "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" url := "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes"
events, err = parseEventForOneSemester(url) events, err := parseEventForOneSemester(url)
if err != nil {
return nil, fmt.Errorf("failed to parse events for summmer semester: %w", err)
}
savedEvents, dbError := db.SaveEvents(events, app) savedEvents, dbError := db.SaveEvents(events, app)
err = dbError if dbError != nil {
return nil, fmt.Errorf("failed to save events: %w", dbError)
}
savedRecords = append(savedEvents, events...) savedRecords = append(savedEvents, events...)
} }
if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) { if (clock.Now().Month() >= 9) || (clock.Now().Month() <= 4) {
url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes" url := "https://stundenplan.htwk-leipzig.de/ws/Berichte/Text-Listen;Veranstaltungsarten;name;Vp%0AVw%0AV%0ASp%0ASw%0AS%0APp%0APw%0AP%0AZV%0ATut%0ASperr%0Apf%0Awpf%0Afak%0A%0A?&template=sws_modul&weeks=1-65&combined=yes"
events, err = parseEventForOneSemester(url) events, err := parseEventForOneSemester(url)
if err != nil {
return nil, fmt.Errorf("failed to parse events for winter semester: %w", err)
}
savedEvents, dbError := db.SaveEvents(events, app) savedEvents, dbError := db.SaveEvents(events, app)
err = dbError if dbError != nil {
return nil, fmt.Errorf("failed to save events: %w", dbError)
}
savedRecords = append(savedEvents, events...) savedRecords = append(savedEvents, events...)
} }
return err, savedRecords return savedRecords, nil
} }
func parseEventForOneSemester(url string) ([]model.Event, error) { func parseEventForOneSemester(url string) ([]model.Event, error) {
@@ -56,25 +59,18 @@ func parseEventForOneSemester(url string) ([]model.Event, error) {
} }
// Parse HTML to Node Tree // Parse HTML to Node Tree
doc, err2 := parseHTML(err, webpage) var doc *html.Node
if err2 != nil { doc, err = parseHTML(webpage, err)
return nil, err2 if err != nil {
return nil, err
} }
// Get all event tables and all day labels // Get all event tables and all day labels
eventTables := getEventTables(doc) eventTables := getEventTables(doc)
allDayLabels := getAllDayLabels(doc) allDayLabels := getAllDayLabels(doc)
if eventTables == nil || allDayLabels == nil {
return nil, err
}
eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels) eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels)
if eventsWithCombinedWeeks == nil {
return nil, err
}
splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks) splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks)
events := splitEventsBySingleWeek(splitEventsByWeekVal) events := splitEventsBySingleWeek(splitEventsByWeekVal)
@@ -83,6 +79,11 @@ func parseEventForOneSemester(url string) ([]model.Event, error) {
} }
table := findFirstTable(doc) table := findFirstTable(doc)
if table == nil {
return nil, fmt.Errorf("failed to find first table")
}
semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data
semester, year := extractSemesterAndYear(semesterString) semester, year := extractSemesterAndYear(semesterString)
events = convertWeeksToDates(events, semester, year) events = convertWeeksToDates(events, semester, year)
@@ -101,7 +102,7 @@ func parseEventForOneSemester(url string) ([]model.Event, error) {
return events, nil return events, nil
} }
func parseHTML(err error, webpage string) (*html.Node, error) { func parseHTML(webpage string, err error) (*html.Node, error) {
doc, err := html.Parse(strings.NewReader(webpage)) doc, err := html.Parse(strings.NewReader(webpage))
if err != nil { if err != nil {
return nil, err return nil, err

View File

@@ -0,0 +1 @@
package logger

View File

@@ -3588,9 +3588,9 @@
"dev": true "dev": true
}, },
"node_modules/vite": { "node_modules/vite": {
"version": "4.4.12", "version": "4.5.2",
"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.12.tgz", "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz",
"integrity": "sha512-KtPlUbWfxzGVul8Nut8Gw2Qe8sBzWY+8QVc5SL8iRFnpnrcoCaNlzO40c1R6hPmcdTwIPEDkq0Y9+27a5tVbdQ==", "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"esbuild": "^0.18.10", "esbuild": "^0.18.10",