diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index d4bd49f..226c538 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -69,10 +69,11 @@ func AddRoutes(app *pocketbase.PocketBase) { app.OnBeforeServe().Add(func(e *core.ServeEvent) error { _, err := e.Router.AddRoute(echo.Route{ Method: http.MethodGet, - Path: "/api/fetch/v3/groups", + Path: "/api/fetch/v3", Handler: func(c echo.Context) error { - groups, err := v3.FetchSeminarGroups(app) + groups, err := v3.FetchAllResources(app) if err != nil { + slog.Error("Failed to fetch seminar groups: %v", err) return c.JSON(http.StatusBadRequest, "Failed to fetch seminar groups") } return c.JSON(http.StatusOK, groups) diff --git a/backend/service/fetch/v3/apiModel.go b/backend/service/fetch/v3/apiModel.go index d324dac..8b71fc3 100644 --- a/backend/service/fetch/v3/apiModel.go +++ b/backend/service/fetch/v3/apiModel.go @@ -42,3 +42,26 @@ type faculty struct { Description string `json:"bezeichnung"` Internal string `json:"internal"` } + +type Events struct { + TotalItems int `json:"hydra:totalItems"` + Events []Event `json:"hydra:member"` +} + +type Event struct { + ID string `json:"id"` + Faculty string `json:"fakultaet"` + SeminarGroups []string `json:"seminargruppen"` + Flags []string `json:"flags"` + Modul string `json:"modul"` + EventType string `json:"veranstaltungstyp"` + Professors []string `json:"dozierende"` + Rooms []string `json:"raeume"` + Courses []string `json:"studiengaenge"` + Description string `json:"bezeichnung"` + Day string `json:"wochentag"` + StartTime string `json:"beginAt"` + EndTime string `json:"endAt"` + CalendarWeeks []int `json:"kalenderwochen"` + Semester string `json:"semester"` +} diff --git a/backend/service/fetch/v3/fetchEvents.go b/backend/service/fetch/v3/fetchEvents.go new file mode 100644 index 0000000..4da310f --- /dev/null +++ b/backend/service/fetch/v3/fetchEvents.go @@ -0,0 +1,38 @@ +package v3 + +import ( + "encoding/json" + "log/slog" + "net/http" +) + +func parseEvents(url string, client *http.Client) (Events, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/veranstaltungen?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedEvents Events + var itemsPerPage = 100 + + responses, err := paginatedFetch(url, itemsPerPage, client) + + if err != nil { + slog.Error("Error while fetching events", err) + return Events{}, err + } + + for _, response := range responses { + var events Events + err = json.Unmarshal([]byte(response), &events) + if err != nil { + slog.Error("Error while unmarshalling events", err) + return Events{}, err + } + + fetchedEvents.Events = append(fetchedEvents.Events, events.Events...) + fetchedEvents.TotalItems = events.TotalItems + } + + return fetchedEvents, nil +} diff --git a/backend/service/fetch/v3/fetchFaculty.go b/backend/service/fetch/v3/fetchFaculty.go new file mode 100644 index 0000000..93c126b --- /dev/null +++ b/backend/service/fetch/v3/fetchFaculty.go @@ -0,0 +1,39 @@ +package v3 + +import ( + "encoding/json" + "log/slog" + "net/http" +) + +func parseFaculties(url string, client *http.Client) (faculties, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/fakultaeten?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedFaculties []faculty + var itemsPerPage = 100 + + responses, err := paginatedFetch(url, itemsPerPage, client) + + if err != nil { + slog.Error("Error while fetching faculties", err) + return faculties{}, err + } + + for _, response := range responses { + var facs faculties + err = json.Unmarshal([]byte(response), &facs) + if err != nil { + slog.Error("Error while unmarshalling faculties", err) + return faculties{}, err + } + + fetchedFaculties = append(fetchedFaculties, facs.Faculties...) + } + + return faculties{ + Faculties: fetchedFaculties, + }, nil +} diff --git a/backend/service/fetch/v3/fetchLunaApi.go b/backend/service/fetch/v3/fetchLunaApi.go new file mode 100644 index 0000000..ca240ba --- /dev/null +++ b/backend/service/fetch/v3/fetchLunaApi.go @@ -0,0 +1,100 @@ +package v3 + +import ( + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/models" + "golang.org/x/net/http2" + "htwkalender/model" + "htwkalender/service/db" + "log/slog" + "net/http" + "strconv" +) + +func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) { + var groups []model.SeminarGroup + client := &http.Client{} + client.Transport = &http2.Transport{} + + apiUrl := "https://luna.htwk-leipzig.de/api/" + + seminarUrl := apiUrl + "studierendengruppen" + + parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client) + + if err != nil { + slog.Error("Error while fetching seminar groups", err) + return nil, err + } + + studyTypeUrl := apiUrl + "studiengaenge" + + parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client) + + if err != nil { + slog.Error("Error while fetching study types", err) + return nil, err + } + + facultyUrl := apiUrl + "fakultaeten" + parsedFaculties, err := parseFaculties(facultyUrl, client) + + if err != nil { + slog.Error("Error while fetching faculties", err) + return nil, err + } + + slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes))) + slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems)) + slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties))) + + // map seminar groups to model seminar groups + for _, group := range parsedSeminarGroups.Groups { + var newGroup model.SeminarGroup + newGroup.University = "HTWK Leipzig" + newGroup.Course = group.SeminarGroup + newGroup.Faculty = group.Faculty + newGroup.Semester = group.Semester + + // find corresponding study type by studiengang in parsedStudyTypes + for _, studyTypeItem := range parsedStudyTypes { + if studyTypeItem.ID == group.Studiengang { + newGroup.GroupShortcut = studyTypeItem.Description + newGroup.GroupId = studyTypeItem.GroupID + break + } + } + + for _, facultyItem := range parsedFaculties.Faculties { + if facultyItem.ID == group.Faculty { + newGroup.FacultyId = facultyItem.ShortCut + newGroup.Faculty = facultyItem.Description + break + } + } + + groups = append(groups, newGroup) + } + + collection, dbError := db.FindCollection(app, "groups") + if dbError != nil { + slog.Error("Error while searching collection groups", dbError) + return nil, err + } + var insertedGroups []*models.Record + + insertedGroups, dbError = db.SaveGroups(groups, collection, app) + if dbError != nil { + slog.Error("Error while saving groups", dbError) + return nil, err + } + + //Now fetch all events + + eventUrl := apiUrl + "veranstaltungen" + parsedEvents, err := parseEvents(eventUrl, client) + + slog.Info("Fetched events: " + strconv.Itoa(len(parsedEvents.Events)) + " of " + strconv.Itoa(parsedEvents.TotalItems)) + + return insertedGroups, nil +} diff --git a/backend/service/fetch/v3/fetchSeminarGroupLD.go b/backend/service/fetch/v3/fetchSeminarGroupLD.go deleted file mode 100644 index ebf3582..0000000 --- a/backend/service/fetch/v3/fetchSeminarGroupLD.go +++ /dev/null @@ -1,4 +0,0 @@ -package v3 - -// seminarGroups Model for fetching json data -// url - https://luna.htwk-leipzig.de/api/studierendengruppen diff --git a/backend/service/fetch/v3/fetchSeminarGroupService.go b/backend/service/fetch/v3/fetchSeminarGroupService.go index a213ff7..8955c27 100644 --- a/backend/service/fetch/v3/fetchSeminarGroupService.go +++ b/backend/service/fetch/v3/fetchSeminarGroupService.go @@ -2,163 +2,12 @@ package v3 import ( "encoding/json" - "github.com/pocketbase/pocketbase" - "github.com/pocketbase/pocketbase/models" - "golang.org/x/net/http2" - "htwkalender/model" + "htwkalender/service/functions" "net/http" - "htwkalender/service/db" "log/slog" - "strconv" ) -func FetchSeminarGroups(app *pocketbase.PocketBase) ([]*models.Record, error) { - var groups []model.SeminarGroup - client := &http.Client{} - client.Transport = &http2.Transport{} - - apiUrl := "https://luna.htwk-leipzig.de/api/" - - seminarUrl := apiUrl + "studierendengruppen" - - parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client) - - studyTypeUrl := apiUrl + "studiengaenge" - - parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client) - - if err != nil { - slog.Error("Error while fetching study types", err) - return nil, err - } - - facultyUrl := apiUrl + "fakultaeten" - parsedFaculties, err := parseFaculties(facultyUrl, client) - - slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes))) - slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems)) - slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties))) - - // map seminar groups to model seminar groups - for _, group := range parsedSeminarGroups.Groups { - var newGroup model.SeminarGroup - newGroup.University = "HTWK Leipzig" - newGroup.Course = group.SeminarGroup - newGroup.Faculty = group.Faculty - newGroup.Semester = group.Semester - - // find corresponding study type by studiengang in parsedStudyTypes - for _, studyTypeItem := range parsedStudyTypes { - if studyTypeItem.ID == group.Studiengang { - newGroup.GroupShortcut = studyTypeItem.Description - newGroup.GroupId = studyTypeItem.GroupID - break - } - } - - for _, facultyItem := range parsedFaculties.Faculties { - if facultyItem.ID == group.Faculty { - newGroup.FacultyId = facultyItem.ShortCut - newGroup.Faculty = facultyItem.Description - break - } - } - - groups = append(groups, newGroup) - } - - collection, dbError := db.FindCollection(app, "groups") - if dbError != nil { - slog.Error("Error while searching collection groups", dbError) - return nil, err - } - var insertedGroups []*models.Record - - insertedGroups, dbError = db.SaveGroups(groups, collection, app) - if dbError != nil { - slog.Error("Error while saving groups", dbError) - return nil, err - } - - return insertedGroups, nil -} - -func parseFaculties(url string, client *http.Client) (faculties, error) { - - // the url is paginated, so we need to fetch all pages - // example url: https://luna.htwk-leipzig.de/api/fakultaeten?page=1&itemsPerPage=100 - // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response - - var fetchedFaculties []faculty - var page = 1 - var itemsPerPage = 100 - - for { - requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) - response, err := requestJSON(requestUrl, client) - if err != nil { - slog.Error("Error while fetching faculties", err) - return faculties{}, err - } - - var facs faculties - err = json.Unmarshal([]byte(response), &facs) - if err != nil { - slog.Error("Error while unmarshalling faculties", err) - return faculties{}, err - } - - fetchedFaculties = append(fetchedFaculties, facs.Faculties...) - - if len(facs.Faculties) == 0 { - break - } - page++ - } - - return faculties{ - Faculties: fetchedFaculties, - }, nil - -} - -func parseStudyTypes(url string, client *http.Client) ([]studyType, error) { - - // the url is paginated, so we need to fetch all pages - // example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100 - // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response - - var fetchedStudyTypes []studyType - var page = 1 - var itemsPerPage = 100 - - for { - requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) - response, err := requestJSON(requestUrl, client) - if err != nil { - slog.Error("Error while fetching study types", err) - return nil, err - } - - var types studyTypes - err = json.Unmarshal([]byte(response), &types) - if err != nil { - slog.Error("Error while unmarshalling study types", err) - return nil, err - } - - fetchedStudyTypes = append(fetchedStudyTypes, types.Types...) - - if len(types.Types) == 0 { - break - } - page++ - } - - return fetchedStudyTypes, nil -} - func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) { // the url is paginated, so we need to fetch all pages @@ -170,14 +19,14 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) var itemsPerPage = 100 var totalItems = 0 - for { - requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) - response, err := requestJSON(requestUrl, client) - if err != nil { - slog.Error("Error while fetching seminar groups", err) - return seminarGroups{}, err - } + responses, err := paginatedFetch(url, itemsPerPage, client) + if err != nil { + slog.Error("Error while fetching seminar groups", err) + return seminarGroups{}, err + } + + for _, response := range responses { var groups seminarGroups err = json.Unmarshal([]byte(response), &groups) if err != nil { @@ -188,9 +37,9 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) // cut api iri prefix for i, group := range groups.Groups { //keep the last 32 characters of the string - groups.Groups[i].Faculty = group.Faculty[len(group.Faculty)-32:] - groups.Groups[i].Studiengang = group.Studiengang[len(group.Studiengang)-32:] - groups.Groups[i].Semester = group.Semester[len(group.Semester)-2:] + groups.Groups[i].Faculty = functions.RemoveIriPrefix(group.Faculty, 32) + groups.Groups[i].Studiengang = functions.RemoveIriPrefix(group.Studiengang, 32) + groups.Groups[i].Semester = functions.RemoveIriPrefix(group.Semester, 2) } totalItems = groups.TotalItems @@ -208,22 +57,3 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) }, nil } - -func removeDuplicates(groups []model.SeminarGroup) []model.SeminarGroup { - var uniqueGroups []model.SeminarGroup - for _, group := range groups { - if !contains(uniqueGroups, group) { - uniqueGroups = append(uniqueGroups, group) - } - } - return uniqueGroups -} - -func contains(groups []model.SeminarGroup, group model.SeminarGroup) bool { - for _, a := range groups { - if (a.Course == group.Course) && (a.Semester == group.Semester) { - return true - } - } - return false -} diff --git a/backend/service/fetch/v3/fetchStudyTypes.go b/backend/service/fetch/v3/fetchStudyTypes.go new file mode 100644 index 0000000..cab4081 --- /dev/null +++ b/backend/service/fetch/v3/fetchStudyTypes.go @@ -0,0 +1,37 @@ +package v3 + +import ( + "encoding/json" + "log/slog" + "net/http" +) + +func parseStudyTypes(url string, client *http.Client) ([]studyType, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedStudyTypes []studyType + var itemsPerPage = 100 + + responses, err := paginatedFetch(url, itemsPerPage, client) + + if err != nil { + slog.Error("Error while fetching study types", err) + return nil, err + } + + for _, response := range responses { + var types studyTypes + err = json.Unmarshal([]byte(response), &types) + if err != nil { + slog.Error("Error while unmarshalling study types", err) + return nil, err + } + + fetchedStudyTypes = append(fetchedStudyTypes, types.Types...) + } + + return fetchedStudyTypes, nil +} diff --git a/backend/service/fetch/v3/paginatedFetch.go b/backend/service/fetch/v3/paginatedFetch.go new file mode 100644 index 0000000..64eb8a1 --- /dev/null +++ b/backend/service/fetch/v3/paginatedFetch.go @@ -0,0 +1,104 @@ +package v3 + +import ( + "encoding/json" + "log/slog" + "net/http" + "strconv" + "strings" + "sync" +) + +type hydraResponse struct { + TotalItems int `json:"totalItems"` + View hydraView `json:"hydra:view"` +} + +type hydraView struct { + First string `json:"hydra:first"` + Last string `json:"hydra:last"` + Next string `json:"hydra:next"` +} + +func paginatedFetch(url string, itemsPerPage int, client *http.Client) ([]string, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var firstPage = 1 + var responses []string + link := url + "?page=" + strconv.Itoa(firstPage) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) + response, err := requestPage(link, client) + + if err != nil { + slog.Error("Error while fetching paginated api", err) + return nil, err + } + + //extract the first and last page from the response + var hydra hydraResponse + err = json.Unmarshal([]byte(response), &hydra) + if err != nil { + slog.Error("Error while unmarshalling hydra response", err, link) + return nil, err + } + var lastPage = extractPageNumber(hydra.View.Last) + responses = append(responses, response) + + // prepare the links for the multithreaded requests + var links []string + for i := firstPage + 1; i <= lastPage; i++ { + link := url + "?page=" + strconv.Itoa(i) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) + links = append(links, link) + } + + //multithreading webpage requests to speed up the process + + var maxThreads = 20 + var htmlPageArray = make([]string, len(links)) + + var wg sync.WaitGroup + wg.Add(maxThreads) + for i := 0; i < maxThreads; i++ { + go func(i int) { + for j := i; j < len(links); j += maxThreads { + slog.Info("Fetching page: " + strconv.Itoa(j) + " of " + strconv.Itoa(len(links))) + doc, err := requestPage(links[j], client) + if err == nil { + htmlPageArray[j] = doc + } + } + wg.Done() + }(i) + } + wg.Wait() + + responses = append(responses, htmlPageArray...) + + return responses, nil +} + +func requestPage(url string, client *http.Client) (string, error) { + response, err := requestJSON(url, client) + if err != nil { + slog.Error("Error while fetching paginated api", err) + return "", err + } + return response, nil +} + +func extractPageNumber(url string) int { + + if url == "" { + return 0 + } + split := strings.Split(url, "page=") + lastPart := split[len(split)-1] + pageNumber, err := strconv.Atoi(lastPart) + if err != nil { + slog.Error("Error while extracting page number", err) + return 0 + } + return pageNumber +} diff --git a/backend/service/functions/string.go b/backend/service/functions/string.go index 7975b6f..1a1efb0 100644 --- a/backend/service/functions/string.go +++ b/backend/service/functions/string.go @@ -51,3 +51,7 @@ func SeperateRoomString(rooms string) []string { []rune{',', '\t', '\n', '\r', ';', ' ', '\u00A0'}), ) } + +func RemoveIriPrefix(iri string, idSize int) string { + return iri[len(iri)-idSize:] +}