From 0686b9397f02adde66a3a4d3510881ccd17f0a35 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 23 Aug 2023 19:16:23 +0200 Subject: [PATCH] update fetch and fixed bug --- addRoute.go | 25 +++- pb_public/index.html | 10 ++ pb_schema.json | 105 ++++++++++++++++- service/db/dbEvents.go | 72 ++++++++++++ service/db/dbGroups.go | 45 ++++++++ service/fetchSeminarGroupService.go | 24 +++- service/fetchService.go | 171 ++++++++++++++++++---------- service/roomService.go | 13 +++ 8 files changed, 399 insertions(+), 66 deletions(-) create mode 100644 pb_public/index.html create mode 100644 service/db/dbEvents.go create mode 100644 service/db/dbGroups.go create mode 100644 service/roomService.go diff --git a/addRoute.go b/addRoute.go index 130c8f0..2e037db 100644 --- a/addRoute.go +++ b/addRoute.go @@ -7,10 +7,16 @@ import ( "github.com/pocketbase/pocketbase/core" "htwk-planner/service" "net/http" + "os" ) func addRoutes(app *pocketbase.PocketBase) { + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + e.Router.GET("/*", apis.StaticDirectoryHandler(os.DirFS("./pb_public"), false)) + return nil + }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { _, err := e.Router.AddRoute(echo.Route{ Method: http.MethodGet, @@ -25,7 +31,6 @@ func addRoutes(app *pocketbase.PocketBase) { if err != nil { return err } - return nil }) @@ -34,7 +39,7 @@ func addRoutes(app *pocketbase.PocketBase) { Method: http.MethodGet, Path: "/api/fetchGroups", Handler: func(c echo.Context) error { - return service.FetchSeminarGroups(c) + return service.FetchSeminarGroups(c, app) }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), @@ -43,7 +48,23 @@ func addRoutes(app *pocketbase.PocketBase) { if err != nil { return err } + return nil + }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + _, err := e.Router.AddRoute(echo.Route{ + Method: http.MethodGet, + Path: "/api/rooms", + Handler: func(c echo.Context) error { + return service.GetRooms(c, app) + }, + Middlewares: []echo.MiddlewareFunc{ + apis.ActivityLogger(app), + }, + }) + if err != nil { + return err + } return nil }) diff --git a/pb_public/index.html b/pb_public/index.html new file mode 100644 index 0000000..d4285af --- /dev/null +++ b/pb_public/index.html @@ -0,0 +1,10 @@ + + + + + HTWKalender + + + + + \ No newline at end of file diff --git a/pb_schema.json b/pb_schema.json index 57920b1..957637a 100644 --- a/pb_schema.json +++ b/pb_schema.json @@ -55,6 +55,95 @@ "requireEmail": false } }, + { + "id": "cfq9mqlmd97v8z5", + "name": "groups", + "type": "base", + "system": false, + "schema": [ + { + "id": "85msl21p", + "name": "university", + "type": "text", + "system": false, + "required": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } + }, + { + "id": "2sii4dtp", + "name": "shortcut", + "type": "text", + "system": false, + "required": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } + }, + { + "id": "uiwgo28f", + "name": "groupId", + "type": "text", + "system": false, + "required": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } + }, + { + "id": "y0l1lrzs", + "name": "course", + "type": "text", + "system": false, + "required": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } + }, + { + "id": "kr62mhbz", + "name": "faculty", + "type": "text", + "system": false, + "required": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } + }, + { + "id": "ya6znpez", + "name": "facultyId", + "type": "text", + "system": false, + "required": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } + } + ], + "indexes": [ + "CREATE UNIQUE INDEX `idx_rcaN2Oq` ON `groups` (`course`)" + ], + "listRule": null, + "viewRule": null, + "createRule": null, + "updateRule": null, + "deleteRule": null, + "options": {} + }, { "id": "7her4515qsmrxe8", "name": "events", @@ -180,9 +269,23 @@ "max": null, "pattern": "" } + }, + { + "id": "vyyefxp7", + "name": "course", + "type": "text", + "system": false, + "required": false, + "options": { + "min": null, + "max": null, + "pattern": "" + } } ], - "indexes": [], + "indexes": [ + "CREATE UNIQUE INDEX `idx_orp1NWL` ON `events` (\n `Day`,\n `Week`,\n `Start`,\n `End`,\n `Name`,\n `course`\n)" + ], "listRule": null, "viewRule": null, "createRule": null, diff --git a/service/db/dbEvents.go b/service/db/dbEvents.go new file mode 100644 index 0000000..888b1bb --- /dev/null +++ b/service/db/dbEvents.go @@ -0,0 +1,72 @@ +package db + +import ( + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/models" + "htwk-planner/model" + "strings" +) + +func SaveEvents(seminarGroup []model.SeminarGroup, collection *models.Collection, app *pocketbase.PocketBase) error { + + for _, seminarGroup := range seminarGroup { + for _, event := range seminarGroup.Events { + var err error = nil + record := models.NewRecord(collection) + record.Set("Day", event.Day) + record.Set("Week", event.Week) + record.Set("Start", event.Start) + record.Set("End", event.End) + record.Set("Name", event.Name) + record.Set("EventType", event.EventType) + record.Set("Prof", event.Prof) + record.Set("Rooms", event.Rooms) + record.Set("Notes", event.Notes) + record.Set("BookedAt", event.BookedAt) + record.Set("course", seminarGroup.Course) + err = app.Dao().SaveRecord(record) + if err != nil { + println("Error while saving record: ", err.Error()) + } + } + } + return nil +} + +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +// GetRooms function to get all rooms from database that are stored as a string in the Events struct +func GetRooms(app *pocketbase.PocketBase) []string { + + var events []struct { + Rooms string `db:"Rooms" json:"Rooms"` + } + + // get all rooms from event records in the events collection + err := app.Dao().DB().Select("Rooms").From("events").All(&events) + if err != nil { + print("Error while getting rooms from database: ", err) + return nil + } + + var roomArray []string + + for _, event := range events { + var room = strings.Split(event.Rooms, " ") + //split string room by space and add each room to array if it is not already in there + for _, r := range room { + var text = strings.TrimSpace(r) + if !contains(roomArray, text) && !strings.Contains(text, " ") && len(text) >= 1 { + roomArray = append(roomArray, text) + } + } + } + return roomArray +} diff --git a/service/db/dbGroups.go b/service/db/dbGroups.go new file mode 100644 index 0000000..fa2363c --- /dev/null +++ b/service/db/dbGroups.go @@ -0,0 +1,45 @@ +package db + +import ( + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/models" + "htwk-planner/model" +) + +func SaveGroups(seminarGroup []model.SeminarGroup, collection *models.Collection, app *pocketbase.PocketBase) error { + for _, group := range seminarGroup { + record := models.NewRecord(collection) + record.Set("university", group.University) + record.Set("shortcut", group.GroupShortcut) + record.Set("groupId", group.GroupId) + record.Set("course", group.Course) + record.Set("faculty", group.Faculty) + record.Set("facultyId", group.FacultyId) + if err := app.Dao().SaveRecord(record); err != nil { + return err + } + } + return nil +} + +func GetAllCourses(app *pocketbase.PocketBase) []string { + + var courses []struct { + Rooms string `db:"course" json:"course"` + } + + // get all rooms from event records in the events collection + err := app.Dao().DB().Select("course").From("groups").All(&courses) + if err != nil { + print("Error while getting groups from database: ", err) + return nil + } + + var courseArray []string + + for _, room := range courses { + courseArray = append(courseArray, room.Rooms) + } + + return courseArray +} diff --git a/service/fetchSeminarGroupService.go b/service/fetchSeminarGroupService.go index 02de4d3..1c815b6 100644 --- a/service/fetchSeminarGroupService.go +++ b/service/fetchSeminarGroupService.go @@ -3,11 +3,13 @@ package service import ( "encoding/xml" "fmt" + "github.com/labstack/echo/v5" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/apis" "htwk-planner/model" + "htwk-planner/service/db" "io" "net/http" - - "github.com/labstack/echo/v5" ) func getSeminarHTML() (string, error) { @@ -38,15 +40,25 @@ func getSeminarHTML() (string, error) { } -func FetchSeminarGroups(c echo.Context) error { +func FetchSeminarGroups(c echo.Context, app *pocketbase.PocketBase) error { result, _ := getSeminarHTML() - var studium []model.SeminarGroup + var groups []model.SeminarGroup - studium = parseSeminarGroups(result) + groups = parseSeminarGroups(result) - return c.JSON(http.StatusOK, studium) + collection, dbError := findCollection(app, "groups") + if dbError != nil { + return apis.NewNotFoundError("Collection not found", dbError) + } + + dbError = db.SaveGroups(groups, collection, app) + if dbError != nil { + return apis.NewApiError(400, "Could not save Events into database", dbError) + } + + return c.JSON(http.StatusOK, groups) } func parseSeminarGroups(result string) []model.SeminarGroup { diff --git a/service/fetchService.go b/service/fetchService.go index 383acfd..bad24cd 100644 --- a/service/fetchService.go +++ b/service/fetchService.go @@ -8,8 +8,10 @@ import ( "github.com/pocketbase/pocketbase/models" "golang.org/x/net/html" "htwk-planner/model" + "htwk-planner/service/db" "io" "net/http" + "strconv" "strings" ) @@ -17,73 +19,64 @@ func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error { var seminarGroups []model.SeminarGroup - seminarGroupsLabel := []string{ - "22INB-3", - "22INB-2", - "22INB-1", - "21INB-2", - "21INB-1", - "22INM", - //"21INM (Masterarbeit)", - "22MIB-BIN", - "22MIB-2", - "22MIB-1", - "21MIB-BIN", - "21MIB-1", - "21MIB-2", - //"20MIB (Praxis/Bachelorarbeit)", - "22MIM", - //"21MIM (Masterarbeit)" - } + //seminarGroupsLabel := []string{ + // "22EIM-MET", + //} + seminarGroupsLabel := db.GetAllCourses(app) for _, seminarGroupLabel := range seminarGroupsLabel { - result, err := getPlanHTML("ss", seminarGroupLabel) + result, getError := getPlanHTML("ws", seminarGroupLabel) - seminarGroup := parseSeminarGroup(result) - - if err != nil { - return apis.NewNotFoundError("The Data could not be fetched", err) + if getError == nil { + seminarGroup := parseSeminarGroup(result) + seminarGroups = append(seminarGroups, seminarGroup) } + } - collection, err := app.Dao().FindCollectionByNameOrId("events") - if err != nil { - return err - } + //Save SeminarGroups to DB + print("Saving SeminarGroups to DB...\n") - for _, event := range seminarGroup.Events { - record := models.NewRecord(collection) - record.Set("Day", event.Day) - record.Set("Week", event.Week) - record.Set("Start", event.Start) - record.Set("End", event.End) - record.Set("Name", event.Name) - record.Set("EventType", event.EventType) - record.Set("Prof", event.Prof) - record.Set("Rooms", event.Rooms) - record.Set("Notes", event.Notes) - record.Set("BookedAt", event.BookedAt) - if err := app.Dao().SaveRecord(record); err != nil { - return err - } - } - - seminarGroups = append(seminarGroups, seminarGroup) + collection, dbError := findCollection(app, "events") + if dbError != nil { + return apis.NewNotFoundError("Collection not found", dbError) + } + dbError = db.SaveEvents(seminarGroups, collection, app) + if dbError != nil { + return apis.NewApiError(400, "Could not save Events into database", dbError) } return c.JSON(http.StatusOK, seminarGroups) } -func parseSeminarGroup(result string) model.SeminarGroup { - doc, _ := html.Parse(strings.NewReader(result)) - table := findFirstTable(doc) - events := toEvents(getEventTables(doc), getAllDayLabels(doc)) +func findCollection(app *pocketbase.PocketBase, collectionName string) (*models.Collection, error) { + collection, dbError := app.Dao().FindCollectionByNameOrId(collectionName) + return collection, dbError +} +func parseSeminarGroup(result string) model.SeminarGroup { + doc, err := html.Parse(strings.NewReader(result)) + if err != nil { + fmt.Printf("Error occurred while parsing the HTML document: %s\n", err.Error()) + return model.SeminarGroup{} + } + + table := findFirstTable(doc) + eventTables := getEventTables(doc) + allDayLabels := getAllDayLabels(doc) + + if eventTables == nil || allDayLabels == nil { + return model.SeminarGroup{} + } + + eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels) + splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks) + events := splitEventsBySingleWeek(splitEventsByWeekVal) var seminarGroup = model.SeminarGroup{ - University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data, - GroupShortcut: findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data, - Events: events, + University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data, + Course: findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data, + Events: events, } return seminarGroup @@ -117,6 +110,59 @@ func toEvents(tables [][]*html.Node, days []string) []model.Events { return events } +func splitEventsByWeek(events []model.Events) []model.Events { + var newEvents []model.Events + + for _, event := range events { + weeks := strings.Split(event.Week, ",") + for _, week := range weeks { + newEvent := event + newEvent.Week = strings.TrimSpace(week) + newEvents = append(newEvents, newEvent) + } + } + return newEvents +} + +func splitEventsBySingleWeek(events []model.Events) []model.Events { + var newEvents []model.Events + + for _, event := range events { + if strings.Contains(event.Week, "-") { + weeks := splitWeekRange(event.Week) + for _, week := range weeks { + newEvent := event + newEvent.Week = week + newEvents = append(newEvents, newEvent) + } + } else { + newEvents = append(newEvents, event) + } + } + return newEvents +} + +func splitWeekRange(weekRange string) []string { + parts := strings.Split(weekRange, "-") + if len(parts) != 2 { + return nil // Invalid format + } + + start, errStart := strconv.Atoi(strings.TrimSpace(parts[0])) + end, errEnd := strconv.Atoi(strings.TrimSpace(parts[1])) + + if errStart != nil || errEnd != nil { + return nil // Error converting to integers + } + + var weeks []string + for i := start; i <= end; i++ { + weeks = append(weeks, strconv.Itoa(i)) + } + + return weeks +} + func toUtf8(iso88591Buf []byte) string { buf := make([]rune, len(iso88591Buf)) for i, b := range iso88591Buf { @@ -202,11 +248,14 @@ func getEventTables(node *html.Node) [][]*html.Node { var eventTables [][]*html.Node tables := findTables(node) - + // get all tables with events for events := range tables { rows := findTableRows(tables[events]) - rows = rows[1:] - eventTables = append(eventTables, rows) + // check that a first row exists + if len(rows) > 0 { + rows = rows[1:] + eventTables = append(eventTables, rows) + } } return eventTables @@ -256,11 +305,19 @@ func findTableRows(node *html.Node) []*html.Node { } } + // Traverse child nodes recursively for child := node.FirstChild; child != nil; child = child.NextSibling { - tableRows = append(tableRows, findTableRows(child)...) + var tableRowElement = findTableRows(child) + if tableRowElement != nil { + tableRows = append(tableRows, tableRowElement...) + } } - return tableRows + if tableRows == nil { + return []*html.Node{} + } else { + return tableRows + } } // Find all

elements in the HTML document diff --git a/service/roomService.go b/service/roomService.go new file mode 100644 index 0000000..96a330d --- /dev/null +++ b/service/roomService.go @@ -0,0 +1,13 @@ +package service + +import ( + "github.com/labstack/echo/v5" + "github.com/pocketbase/pocketbase" + "htwk-planner/service/db" + "net/http" +) + +func GetRooms(c echo.Context, app *pocketbase.PocketBase) error { + rooms := db.GetRooms(app) + return c.JSON(http.StatusOK, rooms) +}