From 82cf9ce48c74ef53f638c957526181ca86039cc0 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Sun, 24 Mar 2024 00:15:49 +0100 Subject: [PATCH] feat:#39 added module and prof fetch --- backend/service/fetch/v3/apiModel.go | 5 +- backend/service/fetch/v3/fetchEventType.go | 50 +++++++ backend/service/fetch/v3/fetchEvents.go | 21 ++- backend/service/fetch/v3/fetchFaculty.go | 2 +- backend/service/fetch/v3/fetchLunaApi.go | 141 ++++++++++++++---- backend/service/fetch/v3/fetchModule.go | 114 ++++++++++++++ backend/service/fetch/v3/fetchProfs.go | 102 +++++++++++++ backend/service/fetch/v3/fetchRooms.go | 93 ++++++++++++ .../fetch/v3/fetchSeminarGroupService.go | 2 +- backend/service/fetch/v3/fetchStudyTypes.go | 2 +- backend/service/fetch/v3/paginatedFetch.go | 1 - backend/service/functions/string.go | 3 + 12 files changed, 504 insertions(+), 32 deletions(-) create mode 100644 backend/service/fetch/v3/fetchEventType.go create mode 100644 backend/service/fetch/v3/fetchModule.go create mode 100644 backend/service/fetch/v3/fetchProfs.go create mode 100644 backend/service/fetch/v3/fetchRooms.go diff --git a/backend/service/fetch/v3/apiModel.go b/backend/service/fetch/v3/apiModel.go index 8b71fc3..40e6384 100644 --- a/backend/service/fetch/v3/apiModel.go +++ b/backend/service/fetch/v3/apiModel.go @@ -51,9 +51,9 @@ type Events struct { type Event struct { ID string `json:"id"` Faculty string `json:"fakultaet"` - SeminarGroups []string `json:"seminargruppen"` + SeminarGroups []string `json:"studierendengruppen"` Flags []string `json:"flags"` - Modul string `json:"modul"` + Module string `json:"modul"` EventType string `json:"veranstaltungstyp"` Professors []string `json:"dozierende"` Rooms []string `json:"raeume"` @@ -64,4 +64,5 @@ type Event struct { EndTime string `json:"endAt"` CalendarWeeks []int `json:"kalenderwochen"` Semester string `json:"semester"` + BookedAt string `json:"internalUpdatedAt"` } diff --git a/backend/service/fetch/v3/fetchEventType.go b/backend/service/fetch/v3/fetchEventType.go new file mode 100644 index 0000000..c291951 --- /dev/null +++ b/backend/service/fetch/v3/fetchEventType.go @@ -0,0 +1,50 @@ +package v3 + +/* + +{ + "hydra:member": [ + { + "@context": "string", + "@id": "string", + "@type": "string", + "id": "string" + } + ], + "hydra:totalItems": 0, + "hydra:view": { + "@id": "string", + "type": "string", + "hydra:first": "string", + "hydra:last": "string", + "hydra:previous": "string", + "hydra:next": "string" + }, + "hydra:search": { + "@type": "string", + "hydra:template": "string", + "hydra:variableRepresentation": "string", + "hydra:mapping": [ + { + "@type": "string", + "variable": "string", + "property": "string", + "required": true + } + ] + } +} + +*/ + +// EventType Model for fetching json data + +type EventTypes struct { + TotalItems int `json:"hydra:totalItems"` + EventTypes []EventType `json:"hydra:member"` +} + +type EventType struct { + ID string `json:"id"` + Type string `json:"@type"` +} diff --git a/backend/service/fetch/v3/fetchEvents.go b/backend/service/fetch/v3/fetchEvents.go index 4da310f..9cd1f5d 100644 --- a/backend/service/fetch/v3/fetchEvents.go +++ b/backend/service/fetch/v3/fetchEvents.go @@ -2,6 +2,7 @@ package v3 import ( "encoding/json" + "htwkalender/service/functions" "log/slog" "net/http" ) @@ -13,7 +14,7 @@ func parseEvents(url string, client *http.Client) (Events, error) { // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response var fetchedEvents Events - var itemsPerPage = 100 + var itemsPerPage = 999 responses, err := paginatedFetch(url, itemsPerPage, client) @@ -30,6 +31,24 @@ func parseEvents(url string, client *http.Client) (Events, error) { return Events{}, err } + // cut api iri prefix + for i, event := range events.Events { + + events.Events[i].EventType = functions.RemoveIriPrefix(event.EventType, 32) + events.Events[i].Faculty = functions.RemoveIriPrefix(event.Faculty, 32) + events.Events[i].Module = functions.RemoveIriPrefix(event.Module, 32) + events.Events[i].Semester = functions.RemoveIriPrefix(event.Semester, 2) + for j, professors := range event.Professors { + events.Events[i].Professors[j] = functions.RemoveIriPrefix(professors, 32) + } + for j, rooms := range event.Rooms { + events.Events[i].Rooms[j] = functions.RemoveIriPrefix(rooms, 32) + } + for j, courses := range event.Courses { + events.Events[i].Courses[j] = functions.RemoveIriPrefix(courses, 32) + } + } + fetchedEvents.Events = append(fetchedEvents.Events, events.Events...) fetchedEvents.TotalItems = events.TotalItems } diff --git a/backend/service/fetch/v3/fetchFaculty.go b/backend/service/fetch/v3/fetchFaculty.go index 93c126b..bcdf7a5 100644 --- a/backend/service/fetch/v3/fetchFaculty.go +++ b/backend/service/fetch/v3/fetchFaculty.go @@ -13,7 +13,7 @@ func parseFaculties(url string, client *http.Client) (faculties, error) { // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response var fetchedFaculties []faculty - var itemsPerPage = 100 + var itemsPerPage = 999 responses, err := paginatedFetch(url, itemsPerPage, client) diff --git a/backend/service/fetch/v3/fetchLunaApi.go b/backend/service/fetch/v3/fetchLunaApi.go index b9ff769..36c2fe7 100644 --- a/backend/service/fetch/v3/fetchLunaApi.go +++ b/backend/service/fetch/v3/fetchLunaApi.go @@ -2,16 +2,15 @@ package v3 import ( "github.com/pocketbase/pocketbase" - "github.com/pocketbase/pocketbase/models" + "github.com/pocketbase/pocketbase/tools/types" "golang.org/x/net/http2" "htwkalender/model" - "htwkalender/service/db" "log/slog" "net/http" "strconv" ) -func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) { +func FetchAllResources(app *pocketbase.PocketBase) (*[]model.Event, error) { var groups []model.SeminarGroup client := &http.Client{} client.Transport = &http2.Transport{} @@ -21,7 +20,7 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) { seminarUrl := apiUrl + "studierendengruppen" parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client) - + slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems)) if err != nil { slog.Error("Error while fetching seminar groups", err) return nil, err @@ -30,7 +29,7 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) { studyTypeUrl := apiUrl + "studiengaenge" parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client) - + slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes))) if err != nil { slog.Error("Error while fetching study types", err) return nil, err @@ -38,15 +37,31 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) { facultyUrl := apiUrl + "fakultaeten" parsedFaculties, err := parseFaculties(facultyUrl, client) - + slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties))) if err != nil { slog.Error("Error while fetching faculties", err) return nil, err } - slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes))) - slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems)) - slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties))) + moduleUrl := apiUrl + "module" + parsedModules, err := parseModules(moduleUrl, client) + slog.Info("Fetched modules: " + strconv.Itoa(len(parsedModules.Modules))) + if err != nil { + slog.Error("Error while fetching modules", err) + return nil, err + } + + roomUrl := apiUrl + "raeume" + parsedRooms, err := parseRooms(roomUrl, client) + slog.Info("Fetched rooms: " + strconv.Itoa(len(parsedRooms.Rooms))) + if err != nil { + slog.Error("Error while fetching rooms", err) + return nil, err + } + + profUrl := apiUrl + "dozierende" + parsedProfs, err := parseProfs(profUrl, client) + slog.Info("Fetched profs: " + strconv.Itoa(len(parsedProfs.Professors))) // map seminar groups to model seminar groups for _, group := range parsedSeminarGroups.Groups { @@ -76,30 +91,106 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) { groups = append(groups, newGroup) } - collection, dbError := db.FindCollection(app, "groups") - if dbError != nil { - slog.Error("Error while searching collection groups", dbError) - return nil, err - } - var insertedGroups []*models.Record - - insertedGroups, dbError = db.SaveGroups(groups, collection, app) - if dbError != nil { - slog.Error("Error while saving groups", dbError) - return nil, err - } - //Now fetch all events - eventUrl := apiUrl + "veranstaltungen" parsedEvents, err := parseEvents(eventUrl, client) + slog.Info("Fetched events: " + strconv.Itoa(len(parsedEvents.Events)) + " of " + strconv.Itoa(parsedEvents.TotalItems)) if err != nil { slog.Error("Error while fetching events", err) return nil, err } - slog.Info("Fetched events: " + strconv.Itoa(len(parsedEvents.Events)) + " of " + strconv.Itoa(parsedEvents.TotalItems)) + //resolve course id in parsedEvents to course name + for i, event := range parsedEvents.Events { + for j, course := range event.Courses { + for _, group := range parsedStudyTypes { + if group.ID == course { + parsedEvents.Events[i].Courses[j] = group.GroupID + } + } + } - return insertedGroups, nil + for _, group := range parsedSeminarGroups.Groups { + for courseItemId, seminarGroupItem := range event.SeminarGroups { + if group.ID == seminarGroupItem { + parsedEvents.Events[i].SeminarGroups[courseItemId] = group.SeminarGroup + } + } + } + + for _, facultyItem := range parsedFaculties.Faculties { + if facultyItem.ID == event.Faculty { + parsedEvents.Events[i].Faculty = facultyItem.ShortCut + } + } + + for _, module := range parsedModules.Modules { + if module.ID == event.Module { + parsedEvents.Events[i].Module = module.Name + } + } + + for _, room := range parsedRooms.Rooms { + for roomPlace, roomID := range event.Rooms { + if room.ID == roomID { + parsedEvents.Events[i].Rooms[roomPlace] = room.ShortCut + } + } + } + + for _, prof := range parsedProfs.Professors { + for profPlace, profID := range event.Professors { + if prof.ID == profID { + parsedEvents.Events[i].Professors[profPlace] = prof.Description1 + } + } + } + } + + // map events to model events + var events []model.Event + + for _, event := range parsedEvents.Events { + for _, week := range event.CalendarWeeks { + for _, course := range event.Courses { + var newEvent model.Event + newEvent.UUID = "" + newEvent.Day = event.Day + newEvent.Week = strconv.Itoa(week) + newEvent.Start, _ = types.ParseDateTime(event.StartTime) + newEvent.End, _ = types.ParseDateTime(event.EndTime) + newEvent.Name = event.Module + newEvent.EventType = event.EventType + newEvent.Compulsory = "" + newEvent.Prof = concatStringArray(event.Professors, "|") + newEvent.Rooms = concatStringArray(event.Rooms, " ") + newEvent.Notes = event.Description + newEvent.BookedAt = event.BookedAt + newEvent.Course = course + newEvent.Semester = event.Semester + events = append(events, newEvent) + } + } + } + + // return events + return &events, nil +} + +func concatStringArray(array []string, concatChar string) string { + var result string + + if len(array) == 0 { + return result + } + + if len(array) > 1 { + for _, item := range array[:len(array)-1] { + result += item + concatChar + } + } + result += array[len(array)-1] + + return result } diff --git a/backend/service/fetch/v3/fetchModule.go b/backend/service/fetch/v3/fetchModule.go new file mode 100644 index 0000000..b8d000b --- /dev/null +++ b/backend/service/fetch/v3/fetchModule.go @@ -0,0 +1,114 @@ +package v3 + +import ( + "encoding/json" + "htwkalender/service/functions" + "log/slog" + "net/http" +) + +/* + +{ + "@context": "/api/contexts/Modul", + "@id": "/api/module", + "@type": "hydra:Collection", + "hydra:totalItems": 1678, + "hydra:member": [ + { + "@id": "/api/module/0055CE2545A3411A00C32F30A645463B", + "@type": "Modul", + "id": "0055CE2545A3411A00C32F30A645463B", + "semester": "/api/semester/ws", + "fakultaet": "/api/fakultaeten/9B89016985E5156B1C033BB0FD3AF9B4", + "kuerzel": "PPKMSBM&SMM&STM3", + "bezeichnung": "W833 Produkt- und Prozesskostenmanagement SBM & SMM & STM 3. FS (wpf)", + "hinweisOne": "", + "internalUpdatedAt": "2023-03-07T07:35:34+01:00" + } + ], + "hydra:view": { + "@id": "/api/module?itemsPerPage=100&page=1", + "@type": "hydra:PartialCollectionView", + "hydra:first": "/api/module?itemsPerPage=100&page=1", + "hydra:last": "/api/module?itemsPerPage=100&page=17", + "hydra:next": "/api/module?itemsPerPage=100&page=2" + }, + "hydra:search": { + "@type": "hydra:IriTemplate", + "hydra:template": "/api/module{?veranstaltungen,veranstaltungen[]}", + "hydra:variableRepresentation": "BasicRepresentation", + "hydra:mapping": [ + { + "@type": "IriTemplateMapping", + "variable": "veranstaltungen", + "property": "veranstaltungen", + "required": false + }, + { + "@type": "IriTemplateMapping", + "variable": "veranstaltungen[]", + "property": "veranstaltungen", + "required": false + } + ] + } +} + + +*/ + +// Module Model for fetching json data + +type Modules struct { + TotalItems int `json:"hydra:totalItems"` + Modules []Module `json:"hydra:member"` +} + +type Module struct { + ID string `json:"id"` + Semester string `json:"semester"` + Faculty string `json:"fakultaet"` + ShortCut string `json:"kuerzel"` + Name string `json:"bezeichnung"` + Note string `json:"hinweisOne"` + Internal string `json:"internalUpdatedAt"` +} + +func parseModules(url string, client *http.Client) (Modules, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/module?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedModules Modules + var itemsPerPage = 999 + + responses, err := paginatedFetch(url, itemsPerPage, client) + + if err != nil { + slog.Error("Error while fetching modules", err) + return Modules{}, err + } + + for _, response := range responses { + var modules Modules + err = json.Unmarshal([]byte(response), &modules) + if err != nil { + slog.Error("Error while unmarshalling modules", err) + return Modules{}, err + } + + // cut api iri prefix + for i, module := range modules.Modules { + + modules.Modules[i].Faculty = functions.RemoveIriPrefix(module.Faculty, 32) + modules.Modules[i].Semester = functions.RemoveIriPrefix(module.Semester, 2) + } + + fetchedModules.Modules = append(fetchedModules.Modules, modules.Modules...) + fetchedModules.TotalItems = modules.TotalItems + } + + return fetchedModules, nil +} diff --git a/backend/service/fetch/v3/fetchProfs.go b/backend/service/fetch/v3/fetchProfs.go new file mode 100644 index 0000000..59a6459 --- /dev/null +++ b/backend/service/fetch/v3/fetchProfs.go @@ -0,0 +1,102 @@ +package v3 + +import ( + "encoding/json" + "log/slog" + "net/http" +) + +/* + +{ + "hydra:member": [ + { + "@context": "string", + "@id": "string", + "@type": "string", + "id": "string", + "kuerzel": "string", + "nachname": "string", + "fakultaet": "https://example.com/", + "bezeichnung": "string", + "bezeichnungOne": "string" + } + ], + "hydra:totalItems": 0, + "hydra:view": { + "@id": "string", + "type": "string", + "hydra:first": "string", + "hydra:last": "string", + "hydra:previous": "string", + "hydra:next": "string" + }, + "hydra:search": { + "@type": "string", + "hydra:template": "string", + "hydra:variableRepresentation": "string", + "hydra:mapping": [ + { + "@type": "string", + "variable": "string", + "property": "string", + "required": true + } + ] + } +} + +*/ + +// professor Model for fetching json data + +type Professors struct { + TotalItems int `json:"hydra:totalItems"` + Professors []Professor `json:"hydra:member"` +} + +type Professor struct { + ID string `json:"id"` + ShortCut string `json:"kuerzel"` + LastName string `json:"nachname"` + Faculty string `json:"fakultaet"` + Description string `json:"bezeichnung"` + Description1 string `json:"bezeichnungOne"` +} + +func parseProfs(url string, client *http.Client) (Professors, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/dozierende?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedProfs Professors + var itemsPerPage = 999 + + responses, err := paginatedFetch(url, itemsPerPage, client) + + if err != nil { + slog.Error("Error while fetching paginated api", err) + return fetchedProfs, err + } + + for _, response := range responses { + var hydra hydraResponse + err = json.Unmarshal([]byte(response), &hydra) + if err != nil { + slog.Error("Error while unmarshalling hydra response", err, url) + return fetchedProfs, err + } + + var professors Professors + err = json.Unmarshal([]byte(response), &professors) + if err != nil { + slog.Error("Error while unmarshalling professors", err) + return fetchedProfs, err + } + fetchedProfs.TotalItems = professors.TotalItems + fetchedProfs.Professors = append(fetchedProfs.Professors, professors.Professors...) + } + + return fetchedProfs, nil +} diff --git a/backend/service/fetch/v3/fetchRooms.go b/backend/service/fetch/v3/fetchRooms.go new file mode 100644 index 0000000..013d471 --- /dev/null +++ b/backend/service/fetch/v3/fetchRooms.go @@ -0,0 +1,93 @@ +package v3 + +import ( + "encoding/json" + "log/slog" + "net/http" +) + +/* + +{ + "hydra:member": [ + { + "@context": "string", + "@id": "string", + "@type": "string", + "id": "string", + "kuerzel": "string", + "beschreibung": "string", + "hinweisOne": "string", + "hinweisTwo": "string", + "plaetze": 0 + } + ], + "hydra:totalItems": 0, + "hydra:view": { + "@id": "string", + "type": "string", + "hydra:first": "string", + "hydra:last": "string", + "hydra:previous": "string", + "hydra:next": "string" + }, + "hydra:search": { + "@type": "string", + "hydra:template": "string", + "hydra:variableRepresentation": "string", + "hydra:mapping": [ + { + "@type": "string", + "variable": "string", + "property": "string", + "required": true + } + ] + } +} + +*/ + +// room Model for fetching json data + +type Rooms struct { + TotalItems int `json:"hydra:totalItems"` + Rooms []Room `json:"hydra:member"` +} + +type Room struct { + ID string `json:"id"` + ShortCut string `json:"kuerzel"` + Description string `json:"beschreibung"` + Seats int `json:"plaetze"` + Note1 string `json:"hinweisOne"` + Note2 string `json:"hinweisTwo"` +} + +func parseRooms(url string, client *http.Client) (Rooms, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/raum?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedRooms Rooms + var itemsPerPage = 999 + + responses, err := paginatedFetch(url, itemsPerPage, client) + + if err != nil { + slog.Error("Error while fetching paginated api", err) + return fetchedRooms, err + } + + for _, response := range responses { + var rooms Rooms + err = json.Unmarshal([]byte(response), &rooms) + if err != nil { + slog.Error("Error while unmarshalling rooms", err) + return fetchedRooms, err + } + fetchedRooms.Rooms = append(fetchedRooms.Rooms, rooms.Rooms...) + } + return fetchedRooms, nil +} diff --git a/backend/service/fetch/v3/fetchSeminarGroupService.go b/backend/service/fetch/v3/fetchSeminarGroupService.go index 8955c27..d12454e 100644 --- a/backend/service/fetch/v3/fetchSeminarGroupService.go +++ b/backend/service/fetch/v3/fetchSeminarGroupService.go @@ -16,7 +16,7 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) var fetchedSeminarGroups []seminarGroup var page = 1 - var itemsPerPage = 100 + var itemsPerPage = 999 var totalItems = 0 responses, err := paginatedFetch(url, itemsPerPage, client) diff --git a/backend/service/fetch/v3/fetchStudyTypes.go b/backend/service/fetch/v3/fetchStudyTypes.go index cab4081..01e5be0 100644 --- a/backend/service/fetch/v3/fetchStudyTypes.go +++ b/backend/service/fetch/v3/fetchStudyTypes.go @@ -13,7 +13,7 @@ func parseStudyTypes(url string, client *http.Client) ([]studyType, error) { // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response var fetchedStudyTypes []studyType - var itemsPerPage = 100 + var itemsPerPage = 999 responses, err := paginatedFetch(url, itemsPerPage, client) diff --git a/backend/service/fetch/v3/paginatedFetch.go b/backend/service/fetch/v3/paginatedFetch.go index 64eb8a1..47f8537 100644 --- a/backend/service/fetch/v3/paginatedFetch.go +++ b/backend/service/fetch/v3/paginatedFetch.go @@ -63,7 +63,6 @@ func paginatedFetch(url string, itemsPerPage int, client *http.Client) ([]string for i := 0; i < maxThreads; i++ { go func(i int) { for j := i; j < len(links); j += maxThreads { - slog.Info("Fetching page: " + strconv.Itoa(j) + " of " + strconv.Itoa(len(links))) doc, err := requestPage(links[j], client) if err == nil { htmlPageArray[j] = doc diff --git a/backend/service/functions/string.go b/backend/service/functions/string.go index a874320..e8c74c8 100644 --- a/backend/service/functions/string.go +++ b/backend/service/functions/string.go @@ -46,5 +46,8 @@ func SeperateRoomString(rooms string) []string { } func RemoveIriPrefix(iri string, idSize int) string { + if len(iri) < idSize { + return "" + } return iri[len(iri)-idSize:] }