diff --git a/backend/go.mod b/backend/go.mod index 7a92cc2..5977758 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/PuerkitoBio/goquery v1.8.1 + github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73 github.com/google/uuid v1.5.0 github.com/jordic/goics v0.0.0-20210404174824-5a0337b716a0 github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 diff --git a/backend/go.sum b/backend/go.sum index ad5fe1f..e4b90d7 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -89,6 +89,8 @@ github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uq github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ganigeorgiev/fexpr v0.4.0 h1:ojitI+VMNZX/odeNL1x3RzTTE8qAIVvnSSYPNAnQFDI= github.com/ganigeorgiev/fexpr v0.4.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= +github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73 h1:GMKIYXyXPGIp+hYiWOhfqK4A023HdgisDT4YGgf99mw= +github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73/go.mod h1:jyveZeGw5LaADntW+UEsMjl3IlIwk+DxlYNsbofQkGA= github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index 6a2bd28..d4bd49f 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -5,6 +5,7 @@ import ( "htwkalender/service/fetch/sport" v1 "htwkalender/service/fetch/v1" v2 "htwkalender/service/fetch/v2" + v3 "htwkalender/service/fetch/v3" "htwkalender/service/functions/time" "htwkalender/service/ical" "htwkalender/service/room" @@ -65,6 +66,27 @@ func AddRoutes(app *pocketbase.PocketBase) { return nil }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + _, err := e.Router.AddRoute(echo.Route{ + Method: http.MethodGet, + Path: "/api/fetch/v3/groups", + Handler: func(c echo.Context) error { + groups, err := v3.FetchSeminarGroups(app) + if err != nil { + return c.JSON(http.StatusBadRequest, "Failed to fetch seminar groups") + } + return c.JSON(http.StatusOK, groups) + }, + Middlewares: []echo.MiddlewareFunc{ + apis.ActivityLogger(app), + }, + }) + if err != nil { + return err + } + return nil + }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { _, err := e.Router.AddRoute(echo.Route{ Method: http.MethodGet, diff --git a/backend/service/fetch/v3/apiModel.go b/backend/service/fetch/v3/apiModel.go new file mode 100644 index 0000000..d324dac --- /dev/null +++ b/backend/service/fetch/v3/apiModel.go @@ -0,0 +1,44 @@ +package v3 + +// seminarGroups Model for fetching json data +type seminarGroups struct { + TotalItems int `json:"hydra:totalItems"` + Groups []seminarGroup `json:"hydra:member"` +} + +type seminarGroup struct { + Type string `json:"@type"` + ID string `json:"id"` + Semester string `json:"semester"` + Faculty string `json:"fakultaet"` + Studiengang string `json:"studiengang"` + SeminarGroup string `json:"kuerzel"` +} + +type studyTypes struct { + TotalItems int `json:"hydra:totalItems"` + Types []studyType `json:"hydra:member"` +} + +type studyType struct { + Type string `json:"@type"` + ID string `json:"id"` + Faculty string `json:"fakultaet"` + Semester string `json:"semester"` + ShortCut string `json:"kuerzel"` + Description string `json:"bezeichnung"` + GroupID string `json:"studiengangskuerzel"` +} + +type faculties struct { + TotalItems int `json:"hydra:totalItems"` + Faculties []faculty `json:"hydra:member"` +} + +type faculty struct { + ID string `json:"id"` + StudyTypes []string `json:"studiengaenge"` + ShortCut string `json:"kuerzel"` + Description string `json:"bezeichnung"` + Internal string `json:"internal"` +} diff --git a/backend/service/fetch/v3/fetchClient.go b/backend/service/fetch/v3/fetchClient.go new file mode 100644 index 0000000..d413f8b --- /dev/null +++ b/backend/service/fetch/v3/fetchClient.go @@ -0,0 +1,41 @@ +package v3 + +import ( + "fmt" + "io" + "net/http" +) + +func requestJSON(url string, client *http.Client) (string, error) { + // Send GET request with the given URL and special headers + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + fmt.Printf("Error occurred while creating the request: %s\n", err.Error()) + return "", err + } + // add header -H 'accept: application/ld+json' + req.Header.Add("accept", "application/ld+json") + + response, err := client.Do(req) + + if err != nil { + fmt.Printf("Error occurred while making the request: %s\n", err.Error()) + return "", err + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + return + } + }(response.Body) + + // Read the response body + body, err := io.ReadAll(response.Body) + + if err != nil { + fmt.Printf("Error occurred while reading the response: %s\n", err.Error()) + return "", err + } + return string(body), err +} diff --git a/backend/service/fetch/v3/fetchSeminarGroupLD.go b/backend/service/fetch/v3/fetchSeminarGroupLD.go new file mode 100644 index 0000000..ebf3582 --- /dev/null +++ b/backend/service/fetch/v3/fetchSeminarGroupLD.go @@ -0,0 +1,4 @@ +package v3 + +// seminarGroups Model for fetching json data +// url - https://luna.htwk-leipzig.de/api/studierendengruppen diff --git a/backend/service/fetch/v3/fetchSeminarGroupService.go b/backend/service/fetch/v3/fetchSeminarGroupService.go new file mode 100644 index 0000000..a213ff7 --- /dev/null +++ b/backend/service/fetch/v3/fetchSeminarGroupService.go @@ -0,0 +1,229 @@ +package v3 + +import ( + "encoding/json" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/models" + "golang.org/x/net/http2" + "htwkalender/model" + "net/http" + + "htwkalender/service/db" + "log/slog" + "strconv" +) + +func FetchSeminarGroups(app *pocketbase.PocketBase) ([]*models.Record, error) { + var groups []model.SeminarGroup + client := &http.Client{} + client.Transport = &http2.Transport{} + + apiUrl := "https://luna.htwk-leipzig.de/api/" + + seminarUrl := apiUrl + "studierendengruppen" + + parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client) + + studyTypeUrl := apiUrl + "studiengaenge" + + parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client) + + if err != nil { + slog.Error("Error while fetching study types", err) + return nil, err + } + + facultyUrl := apiUrl + "fakultaeten" + parsedFaculties, err := parseFaculties(facultyUrl, client) + + slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes))) + slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems)) + slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties))) + + // map seminar groups to model seminar groups + for _, group := range parsedSeminarGroups.Groups { + var newGroup model.SeminarGroup + newGroup.University = "HTWK Leipzig" + newGroup.Course = group.SeminarGroup + newGroup.Faculty = group.Faculty + newGroup.Semester = group.Semester + + // find corresponding study type by studiengang in parsedStudyTypes + for _, studyTypeItem := range parsedStudyTypes { + if studyTypeItem.ID == group.Studiengang { + newGroup.GroupShortcut = studyTypeItem.Description + newGroup.GroupId = studyTypeItem.GroupID + break + } + } + + for _, facultyItem := range parsedFaculties.Faculties { + if facultyItem.ID == group.Faculty { + newGroup.FacultyId = facultyItem.ShortCut + newGroup.Faculty = facultyItem.Description + break + } + } + + groups = append(groups, newGroup) + } + + collection, dbError := db.FindCollection(app, "groups") + if dbError != nil { + slog.Error("Error while searching collection groups", dbError) + return nil, err + } + var insertedGroups []*models.Record + + insertedGroups, dbError = db.SaveGroups(groups, collection, app) + if dbError != nil { + slog.Error("Error while saving groups", dbError) + return nil, err + } + + return insertedGroups, nil +} + +func parseFaculties(url string, client *http.Client) (faculties, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/fakultaeten?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedFaculties []faculty + var page = 1 + var itemsPerPage = 100 + + for { + requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) + response, err := requestJSON(requestUrl, client) + if err != nil { + slog.Error("Error while fetching faculties", err) + return faculties{}, err + } + + var facs faculties + err = json.Unmarshal([]byte(response), &facs) + if err != nil { + slog.Error("Error while unmarshalling faculties", err) + return faculties{}, err + } + + fetchedFaculties = append(fetchedFaculties, facs.Faculties...) + + if len(facs.Faculties) == 0 { + break + } + page++ + } + + return faculties{ + Faculties: fetchedFaculties, + }, nil + +} + +func parseStudyTypes(url string, client *http.Client) ([]studyType, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedStudyTypes []studyType + var page = 1 + var itemsPerPage = 100 + + for { + requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) + response, err := requestJSON(requestUrl, client) + if err != nil { + slog.Error("Error while fetching study types", err) + return nil, err + } + + var types studyTypes + err = json.Unmarshal([]byte(response), &types) + if err != nil { + slog.Error("Error while unmarshalling study types", err) + return nil, err + } + + fetchedStudyTypes = append(fetchedStudyTypes, types.Types...) + + if len(types.Types) == 0 { + break + } + page++ + } + + return fetchedStudyTypes, nil +} + +func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) { + + // the url is paginated, so we need to fetch all pages + // example url: https://luna.htwk-leipzig.de/api/studierendengruppen?page=1&itemsPerPage=100 + // the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response + + var fetchedSeminarGroups []seminarGroup + var page = 1 + var itemsPerPage = 100 + var totalItems = 0 + + for { + requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage) + response, err := requestJSON(requestUrl, client) + if err != nil { + slog.Error("Error while fetching seminar groups", err) + return seminarGroups{}, err + } + + var groups seminarGroups + err = json.Unmarshal([]byte(response), &groups) + if err != nil { + slog.Error("Error while unmarshalling seminar groups", err) + return seminarGroups{}, err + } + + // cut api iri prefix + for i, group := range groups.Groups { + //keep the last 32 characters of the string + groups.Groups[i].Faculty = group.Faculty[len(group.Faculty)-32:] + groups.Groups[i].Studiengang = group.Studiengang[len(group.Studiengang)-32:] + groups.Groups[i].Semester = group.Semester[len(group.Semester)-2:] + } + + totalItems = groups.TotalItems + fetchedSeminarGroups = append(fetchedSeminarGroups, groups.Groups...) + + if len(groups.Groups) == 0 { + break + } + page++ + } + + return seminarGroups{ + TotalItems: totalItems, + Groups: fetchedSeminarGroups, + }, nil + +} + +func removeDuplicates(groups []model.SeminarGroup) []model.SeminarGroup { + var uniqueGroups []model.SeminarGroup + for _, group := range groups { + if !contains(uniqueGroups, group) { + uniqueGroups = append(uniqueGroups, group) + } + } + return uniqueGroups +} + +func contains(groups []model.SeminarGroup, group model.SeminarGroup) bool { + for _, a := range groups { + if (a.Course == group.Course) && (a.Semester == group.Semester) { + return true + } + } + return false +}