feat:#39 added fetcher for SeminarGroups

backend/go.mod

@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.1
+	github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73
 	github.com/google/uuid v1.5.0
 	github.com/jordic/goics v0.0.0-20210404174824-5a0337b716a0
 	github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61

backend/go.sum

@@ -89,6 +89,8 @@ github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uq
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/ganigeorgiev/fexpr v0.4.0 h1:ojitI+VMNZX/odeNL1x3RzTTE8qAIVvnSSYPNAnQFDI=
 github.com/ganigeorgiev/fexpr v0.4.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE=
+github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73 h1:GMKIYXyXPGIp+hYiWOhfqK4A023HdgisDT4YGgf99mw=
+github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73/go.mod h1:jyveZeGw5LaADntW+UEsMjl3IlIwk+DxlYNsbofQkGA=
 github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es=
 github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=

backend/service/addRoute.go

@@ -5,6 +5,7 @@ import (
 	"htwkalender/service/fetch/sport"
 	v1 "htwkalender/service/fetch/v1"
 	v2 "htwkalender/service/fetch/v2"
+	v3 "htwkalender/service/fetch/v3"
 	"htwkalender/service/functions/time"
 	"htwkalender/service/ical"
 	"htwkalender/service/room"
@@ -65,6 +66,27 @@ func AddRoutes(app *pocketbase.PocketBase) {
 		return nil
 	})
 
+	app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
+		_, err := e.Router.AddRoute(echo.Route{
+			Method: http.MethodGet,
+			Path:   "/api/fetch/v3/groups",
+			Handler: func(c echo.Context) error {
+				groups, err := v3.FetchSeminarGroups(app)
+				if err != nil {
+					return c.JSON(http.StatusBadRequest, "Failed to fetch seminar groups")
+				}
+				return c.JSON(http.StatusOK, groups)
+			},
+			Middlewares: []echo.MiddlewareFunc{
+				apis.ActivityLogger(app),
+			},
+		})
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
 	app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
 		_, err := e.Router.AddRoute(echo.Route{
 			Method: http.MethodGet,

backend/service/fetch/v3/apiModel.go

@@ -0,0 +1,44 @@
+package v3
+
+// seminarGroups Model for fetching json data
+type seminarGroups struct {
+	TotalItems int            `json:"hydra:totalItems"`
+	Groups     []seminarGroup `json:"hydra:member"`
+}
+
+type seminarGroup struct {
+	Type         string `json:"@type"`
+	ID           string `json:"id"`
+	Semester     string `json:"semester"`
+	Faculty      string `json:"fakultaet"`
+	Studiengang  string `json:"studiengang"`
+	SeminarGroup string `json:"kuerzel"`
+}
+
+type studyTypes struct {
+	TotalItems int         `json:"hydra:totalItems"`
+	Types      []studyType `json:"hydra:member"`
+}
+
+type studyType struct {
+	Type        string `json:"@type"`
+	ID          string `json:"id"`
+	Faculty     string `json:"fakultaet"`
+	Semester    string `json:"semester"`
+	ShortCut    string `json:"kuerzel"`
+	Description string `json:"bezeichnung"`
+	GroupID     string `json:"studiengangskuerzel"`
+}
+
+type faculties struct {
+	TotalItems int       `json:"hydra:totalItems"`
+	Faculties  []faculty `json:"hydra:member"`
+}
+
+type faculty struct {
+	ID          string   `json:"id"`
+	StudyTypes  []string `json:"studiengaenge"`
+	ShortCut    string   `json:"kuerzel"`
+	Description string   `json:"bezeichnung"`
+	Internal    string   `json:"internal"`
+}

backend/service/fetch/v3/fetchClient.go

@@ -0,0 +1,41 @@
+package v3
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+func requestJSON(url string, client *http.Client) (string, error) {
+	// Send GET request with the given URL and special headers
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		fmt.Printf("Error occurred while creating the request: %s\n", err.Error())
+		return "", err
+	}
+	// add header -H 'accept: application/ld+json'
+	req.Header.Add("accept", "application/ld+json")
+
+	response, err := client.Do(req)
+
+	if err != nil {
+		fmt.Printf("Error occurred while making the request: %s\n", err.Error())
+		return "", err
+	}
+	defer func(Body io.ReadCloser) {
+		err := Body.Close()
+		if err != nil {
+			return
+		}
+	}(response.Body)
+
+	// Read the response body
+	body, err := io.ReadAll(response.Body)
+
+	if err != nil {
+		fmt.Printf("Error occurred while reading the response: %s\n", err.Error())
+		return "", err
+	}
+	return string(body), err
+}

backend/service/fetch/v3/fetchSeminarGroupLD.go

@@ -0,0 +1,4 @@
+package v3
+
+// seminarGroups Model for fetching json data
+// url - https://luna.htwk-leipzig.de/api/studierendengruppen

backend/service/fetch/v3/fetchSeminarGroupService.go

@@ -0,0 +1,229 @@
+package v3
+
+import (
+	"encoding/json"
+	"github.com/pocketbase/pocketbase"
+	"github.com/pocketbase/pocketbase/models"
+	"golang.org/x/net/http2"
+	"htwkalender/model"
+	"net/http"
+
+	"htwkalender/service/db"
+	"log/slog"
+	"strconv"
+)
+
+func FetchSeminarGroups(app *pocketbase.PocketBase) ([]*models.Record, error) {
+	var groups []model.SeminarGroup
+	client := &http.Client{}
+	client.Transport = &http2.Transport{}
+
+	apiUrl := "https://luna.htwk-leipzig.de/api/"
+
+	seminarUrl := apiUrl + "studierendengruppen"
+
+	parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client)
+
+	studyTypeUrl := apiUrl + "studiengaenge"
+
+	parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client)
+
+	if err != nil {
+		slog.Error("Error while fetching study types", err)
+		return nil, err
+	}
+
+	facultyUrl := apiUrl + "fakultaeten"
+	parsedFaculties, err := parseFaculties(facultyUrl, client)
+
+	slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes)))
+	slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems))
+	slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties)))
+
+	// map seminar groups to model seminar groups
+	for _, group := range parsedSeminarGroups.Groups {
+		var newGroup model.SeminarGroup
+		newGroup.University = "HTWK Leipzig"
+		newGroup.Course = group.SeminarGroup
+		newGroup.Faculty = group.Faculty
+		newGroup.Semester = group.Semester
+
+		// find corresponding study type by studiengang in parsedStudyTypes
+		for _, studyTypeItem := range parsedStudyTypes {
+			if studyTypeItem.ID == group.Studiengang {
+				newGroup.GroupShortcut = studyTypeItem.Description
+				newGroup.GroupId = studyTypeItem.GroupID
+				break
+			}
+		}
+
+		for _, facultyItem := range parsedFaculties.Faculties {
+			if facultyItem.ID == group.Faculty {
+				newGroup.FacultyId = facultyItem.ShortCut
+				newGroup.Faculty = facultyItem.Description
+				break
+			}
+		}
+
+		groups = append(groups, newGroup)
+	}
+
+	collection, dbError := db.FindCollection(app, "groups")
+	if dbError != nil {
+		slog.Error("Error while searching collection groups", dbError)
+		return nil, err
+	}
+	var insertedGroups []*models.Record
+
+	insertedGroups, dbError = db.SaveGroups(groups, collection, app)
+	if dbError != nil {
+		slog.Error("Error while saving groups", dbError)
+		return nil, err
+	}
+
+	return insertedGroups, nil
+}
+
+func parseFaculties(url string, client *http.Client) (faculties, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/fakultaeten?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedFaculties []faculty
+	var page = 1
+	var itemsPerPage = 100
+
+	for {
+		requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
+		response, err := requestJSON(requestUrl, client)
+		if err != nil {
+			slog.Error("Error while fetching faculties", err)
+			return faculties{}, err
+		}
+
+		var facs faculties
+		err = json.Unmarshal([]byte(response), &facs)
+		if err != nil {
+			slog.Error("Error while unmarshalling faculties", err)
+			return faculties{}, err
+		}
+
+		fetchedFaculties = append(fetchedFaculties, facs.Faculties...)
+
+		if len(facs.Faculties) == 0 {
+			break
+		}
+		page++
+	}
+
+	return faculties{
+		Faculties: fetchedFaculties,
+	}, nil
+
+}
+
+func parseStudyTypes(url string, client *http.Client) ([]studyType, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedStudyTypes []studyType
+	var page = 1
+	var itemsPerPage = 100
+
+	for {
+		requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
+		response, err := requestJSON(requestUrl, client)
+		if err != nil {
+			slog.Error("Error while fetching study types", err)
+			return nil, err
+		}
+
+		var types studyTypes
+		err = json.Unmarshal([]byte(response), &types)
+		if err != nil {
+			slog.Error("Error while unmarshalling study types", err)
+			return nil, err
+		}
+
+		fetchedStudyTypes = append(fetchedStudyTypes, types.Types...)
+
+		if len(types.Types) == 0 {
+			break
+		}
+		page++
+	}
+
+	return fetchedStudyTypes, nil
+}
+
+func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/studierendengruppen?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedSeminarGroups []seminarGroup
+	var page = 1
+	var itemsPerPage = 100
+	var totalItems = 0
+
+	for {
+		requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
+		response, err := requestJSON(requestUrl, client)
+		if err != nil {
+			slog.Error("Error while fetching seminar groups", err)
+			return seminarGroups{}, err
+		}
+
+		var groups seminarGroups
+		err = json.Unmarshal([]byte(response), &groups)
+		if err != nil {
+			slog.Error("Error while unmarshalling seminar groups", err)
+			return seminarGroups{}, err
+		}
+
+		// cut api iri prefix
+		for i, group := range groups.Groups {
+			//keep the last 32 characters of the string
+			groups.Groups[i].Faculty = group.Faculty[len(group.Faculty)-32:]
+			groups.Groups[i].Studiengang = group.Studiengang[len(group.Studiengang)-32:]
+			groups.Groups[i].Semester = group.Semester[len(group.Semester)-2:]
+		}
+
+		totalItems = groups.TotalItems
+		fetchedSeminarGroups = append(fetchedSeminarGroups, groups.Groups...)
+
+		if len(groups.Groups) == 0 {
+			break
+		}
+		page++
+	}
+
+	return seminarGroups{
+		TotalItems: totalItems,
+		Groups:     fetchedSeminarGroups,
+	}, nil
+
+}
+
+func removeDuplicates(groups []model.SeminarGroup) []model.SeminarGroup {
+	var uniqueGroups []model.SeminarGroup
+	for _, group := range groups {
+		if !contains(uniqueGroups, group) {
+			uniqueGroups = append(uniqueGroups, group)
+		}
+	}
+	return uniqueGroups
+}
+
+func contains(groups []model.SeminarGroup, group model.SeminarGroup) bool {
+	for _, a := range groups {
+		if (a.Course == group.Course) && (a.Semester == group.Semester) {
+			return true
+		}
+	}
+	return false
+}