feat:#39 added fetcher for Events

backend/service/addRoute.go

@@ -69,10 +69,11 @@ func AddRoutes(app *pocketbase.PocketBase) {
 	app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
 		_, err := e.Router.AddRoute(echo.Route{
 			Method: http.MethodGet,
-			Path:   "/api/fetch/v3/groups",
+			Path:   "/api/fetch/v3",
 			Handler: func(c echo.Context) error {
-				groups, err := v3.FetchSeminarGroups(app)
+				groups, err := v3.FetchAllResources(app)
 				if err != nil {
+					slog.Error("Failed to fetch seminar groups: %v", err)
 					return c.JSON(http.StatusBadRequest, "Failed to fetch seminar groups")
 				}
 				return c.JSON(http.StatusOK, groups)

backend/service/fetch/v3/apiModel.go

@@ -42,3 +42,26 @@ type faculty struct {
 	Description string   `json:"bezeichnung"`
 	Internal    string   `json:"internal"`
 }
+
+type Events struct {
+	TotalItems int     `json:"hydra:totalItems"`
+	Events     []Event `json:"hydra:member"`
+}
+
+type Event struct {
+	ID            string   `json:"id"`
+	Faculty       string   `json:"fakultaet"`
+	SeminarGroups []string `json:"seminargruppen"`
+	Flags         []string `json:"flags"`
+	Modul         string   `json:"modul"`
+	EventType     string   `json:"veranstaltungstyp"`
+	Professors    []string `json:"dozierende"`
+	Rooms         []string `json:"raeume"`
+	Courses       []string `json:"studiengaenge"`
+	Description   string   `json:"bezeichnung"`
+	Day           string   `json:"wochentag"`
+	StartTime     string   `json:"beginAt"`
+	EndTime       string   `json:"endAt"`
+	CalendarWeeks []int    `json:"kalenderwochen"`
+	Semester      string   `json:"semester"`
+}

backend/service/fetch/v3/fetchEvents.go

@@ -0,0 +1,38 @@
+package v3
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+)
+
+func parseEvents(url string, client *http.Client) (Events, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/veranstaltungen?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedEvents Events
+	var itemsPerPage = 100
+
+	responses, err := paginatedFetch(url, itemsPerPage, client)
+
+	if err != nil {
+		slog.Error("Error while fetching events", err)
+		return Events{}, err
+	}
+
+	for _, response := range responses {
+		var events Events
+		err = json.Unmarshal([]byte(response), &events)
+		if err != nil {
+			slog.Error("Error while unmarshalling events", err)
+			return Events{}, err
+		}
+
+		fetchedEvents.Events = append(fetchedEvents.Events, events.Events...)
+		fetchedEvents.TotalItems = events.TotalItems
+	}
+
+	return fetchedEvents, nil
+}

backend/service/fetch/v3/fetchFaculty.go

@@ -0,0 +1,39 @@
+package v3
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+)
+
+func parseFaculties(url string, client *http.Client) (faculties, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/fakultaeten?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedFaculties []faculty
+	var itemsPerPage = 100
+
+	responses, err := paginatedFetch(url, itemsPerPage, client)
+
+	if err != nil {
+		slog.Error("Error while fetching faculties", err)
+		return faculties{}, err
+	}
+
+	for _, response := range responses {
+		var facs faculties
+		err = json.Unmarshal([]byte(response), &facs)
+		if err != nil {
+			slog.Error("Error while unmarshalling faculties", err)
+			return faculties{}, err
+		}
+
+		fetchedFaculties = append(fetchedFaculties, facs.Faculties...)
+	}
+
+	return faculties{
+		Faculties: fetchedFaculties,
+	}, nil
+}

backend/service/fetch/v3/fetchLunaApi.go

@@ -0,0 +1,100 @@
+package v3
+
+import (
+	"github.com/pocketbase/pocketbase"
+	"github.com/pocketbase/pocketbase/models"
+	"golang.org/x/net/http2"
+	"htwkalender/model"
+	"htwkalender/service/db"
+	"log/slog"
+	"net/http"
+	"strconv"
+)
+
+func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) {
+	var groups []model.SeminarGroup
+	client := &http.Client{}
+	client.Transport = &http2.Transport{}
+
+	apiUrl := "https://luna.htwk-leipzig.de/api/"
+
+	seminarUrl := apiUrl + "studierendengruppen"
+
+	parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client)
+
+	if err != nil {
+		slog.Error("Error while fetching seminar groups", err)
+		return nil, err
+	}
+
+	studyTypeUrl := apiUrl + "studiengaenge"
+
+	parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client)
+
+	if err != nil {
+		slog.Error("Error while fetching study types", err)
+		return nil, err
+	}
+
+	facultyUrl := apiUrl + "fakultaeten"
+	parsedFaculties, err := parseFaculties(facultyUrl, client)
+
+	if err != nil {
+		slog.Error("Error while fetching faculties", err)
+		return nil, err
+	}
+
+	slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes)))
+	slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems))
+	slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties)))
+
+	// map seminar groups to model seminar groups
+	for _, group := range parsedSeminarGroups.Groups {
+		var newGroup model.SeminarGroup
+		newGroup.University = "HTWK Leipzig"
+		newGroup.Course = group.SeminarGroup
+		newGroup.Faculty = group.Faculty
+		newGroup.Semester = group.Semester
+
+		// find corresponding study type by studiengang in parsedStudyTypes
+		for _, studyTypeItem := range parsedStudyTypes {
+			if studyTypeItem.ID == group.Studiengang {
+				newGroup.GroupShortcut = studyTypeItem.Description
+				newGroup.GroupId = studyTypeItem.GroupID
+				break
+			}
+		}
+
+		for _, facultyItem := range parsedFaculties.Faculties {
+			if facultyItem.ID == group.Faculty {
+				newGroup.FacultyId = facultyItem.ShortCut
+				newGroup.Faculty = facultyItem.Description
+				break
+			}
+		}
+
+		groups = append(groups, newGroup)
+	}
+
+	collection, dbError := db.FindCollection(app, "groups")
+	if dbError != nil {
+		slog.Error("Error while searching collection groups", dbError)
+		return nil, err
+	}
+	var insertedGroups []*models.Record
+
+	insertedGroups, dbError = db.SaveGroups(groups, collection, app)
+	if dbError != nil {
+		slog.Error("Error while saving groups", dbError)
+		return nil, err
+	}
+
+	//Now fetch all events
+
+	eventUrl := apiUrl + "veranstaltungen"
+	parsedEvents, err := parseEvents(eventUrl, client)
+
+	slog.Info("Fetched events: " + strconv.Itoa(len(parsedEvents.Events)) + " of " + strconv.Itoa(parsedEvents.TotalItems))
+
+	return insertedGroups, nil
+}

backend/service/fetch/v3/fetchSeminarGroupLD.go

@@ -1,4 +0,0 @@
-package v3
-
-// seminarGroups Model for fetching json data
-// url - https://luna.htwk-leipzig.de/api/studierendengruppen

backend/service/fetch/v3/fetchSeminarGroupService.go

@@ -2,163 +2,12 @@ package v3
 
 import (
 	"encoding/json"
-	"github.com/pocketbase/pocketbase"
+	"htwkalender/service/functions"
-	"github.com/pocketbase/pocketbase/models"
-	"golang.org/x/net/http2"
-	"htwkalender/model"
 	"net/http"
 
-	"htwkalender/service/db"
 	"log/slog"
-	"strconv"
 )
 
-func FetchSeminarGroups(app *pocketbase.PocketBase) ([]*models.Record, error) {
-	var groups []model.SeminarGroup
-	client := &http.Client{}
-	client.Transport = &http2.Transport{}
-
-	apiUrl := "https://luna.htwk-leipzig.de/api/"
-
-	seminarUrl := apiUrl + "studierendengruppen"
-
-	parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client)
-
-	studyTypeUrl := apiUrl + "studiengaenge"
-
-	parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client)
-
-	if err != nil {
-		slog.Error("Error while fetching study types", err)
-		return nil, err
-	}
-
-	facultyUrl := apiUrl + "fakultaeten"
-	parsedFaculties, err := parseFaculties(facultyUrl, client)
-
-	slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes)))
-	slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems))
-	slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties)))
-
-	// map seminar groups to model seminar groups
-	for _, group := range parsedSeminarGroups.Groups {
-		var newGroup model.SeminarGroup
-		newGroup.University = "HTWK Leipzig"
-		newGroup.Course = group.SeminarGroup
-		newGroup.Faculty = group.Faculty
-		newGroup.Semester = group.Semester
-
-		// find corresponding study type by studiengang in parsedStudyTypes
-		for _, studyTypeItem := range parsedStudyTypes {
-			if studyTypeItem.ID == group.Studiengang {
-				newGroup.GroupShortcut = studyTypeItem.Description
-				newGroup.GroupId = studyTypeItem.GroupID
-				break
-			}
-		}
-
-		for _, facultyItem := range parsedFaculties.Faculties {
-			if facultyItem.ID == group.Faculty {
-				newGroup.FacultyId = facultyItem.ShortCut
-				newGroup.Faculty = facultyItem.Description
-				break
-			}
-		}
-
-		groups = append(groups, newGroup)
-	}
-
-	collection, dbError := db.FindCollection(app, "groups")
-	if dbError != nil {
-		slog.Error("Error while searching collection groups", dbError)
-		return nil, err
-	}
-	var insertedGroups []*models.Record
-
-	insertedGroups, dbError = db.SaveGroups(groups, collection, app)
-	if dbError != nil {
-		slog.Error("Error while saving groups", dbError)
-		return nil, err
-	}
-
-	return insertedGroups, nil
-}
-
-func parseFaculties(url string, client *http.Client) (faculties, error) {
-
-	// the url is paginated, so we need to fetch all pages
-	// example url: https://luna.htwk-leipzig.de/api/fakultaeten?page=1&itemsPerPage=100
-	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
-
-	var fetchedFaculties []faculty
-	var page = 1
-	var itemsPerPage = 100
-
-	for {
-		requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
-		response, err := requestJSON(requestUrl, client)
-		if err != nil {
-			slog.Error("Error while fetching faculties", err)
-			return faculties{}, err
-		}
-
-		var facs faculties
-		err = json.Unmarshal([]byte(response), &facs)
-		if err != nil {
-			slog.Error("Error while unmarshalling faculties", err)
-			return faculties{}, err
-		}
-
-		fetchedFaculties = append(fetchedFaculties, facs.Faculties...)
-
-		if len(facs.Faculties) == 0 {
-			break
-		}
-		page++
-	}
-
-	return faculties{
-		Faculties: fetchedFaculties,
-	}, nil
-
-}
-
-func parseStudyTypes(url string, client *http.Client) ([]studyType, error) {
-
-	// the url is paginated, so we need to fetch all pages
-	// example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100
-	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
-
-	var fetchedStudyTypes []studyType
-	var page = 1
-	var itemsPerPage = 100
-
-	for {
-		requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
-		response, err := requestJSON(requestUrl, client)
-		if err != nil {
-			slog.Error("Error while fetching study types", err)
-			return nil, err
-		}
-
-		var types studyTypes
-		err = json.Unmarshal([]byte(response), &types)
-		if err != nil {
-			slog.Error("Error while unmarshalling study types", err)
-			return nil, err
-		}
-
-		fetchedStudyTypes = append(fetchedStudyTypes, types.Types...)
-
-		if len(types.Types) == 0 {
-			break
-		}
-		page++
-	}
-
-	return fetchedStudyTypes, nil
-}
-
 func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error) {
 
 	// the url is paginated, so we need to fetch all pages
@@ -170,14 +19,14 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error)
 	var itemsPerPage = 100
 	var totalItems = 0
 
-	for {
+	responses, err := paginatedFetch(url, itemsPerPage, client)
-		requestUrl := url + "?page=" + strconv.Itoa(page) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
-		response, err := requestJSON(requestUrl, client)
-		if err != nil {
-			slog.Error("Error while fetching seminar groups", err)
-			return seminarGroups{}, err
-		}
 
+	if err != nil {
+		slog.Error("Error while fetching seminar groups", err)
+		return seminarGroups{}, err
+	}
+
+	for _, response := range responses {
 		var groups seminarGroups
 		err = json.Unmarshal([]byte(response), &groups)
 		if err != nil {
@@ -188,9 +37,9 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error)
 		// cut api iri prefix
 		for i, group := range groups.Groups {
 			//keep the last 32 characters of the string
-			groups.Groups[i].Faculty = group.Faculty[len(group.Faculty)-32:]
+			groups.Groups[i].Faculty = functions.RemoveIriPrefix(group.Faculty, 32)
-			groups.Groups[i].Studiengang = group.Studiengang[len(group.Studiengang)-32:]
+			groups.Groups[i].Studiengang = functions.RemoveIriPrefix(group.Studiengang, 32)
-			groups.Groups[i].Semester = group.Semester[len(group.Semester)-2:]
+			groups.Groups[i].Semester = functions.RemoveIriPrefix(group.Semester, 2)
 		}
 
 		totalItems = groups.TotalItems
@@ -208,22 +57,3 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error)
 	}, nil
 
 }
-
-func removeDuplicates(groups []model.SeminarGroup) []model.SeminarGroup {
-	var uniqueGroups []model.SeminarGroup
-	for _, group := range groups {
-		if !contains(uniqueGroups, group) {
-			uniqueGroups = append(uniqueGroups, group)
-		}
-	}
-	return uniqueGroups
-}
-
-func contains(groups []model.SeminarGroup, group model.SeminarGroup) bool {
-	for _, a := range groups {
-		if (a.Course == group.Course) && (a.Semester == group.Semester) {
-			return true
-		}
-	}
-	return false
-}

backend/service/fetch/v3/fetchStudyTypes.go

@@ -0,0 +1,37 @@
+package v3
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+)
+
+func parseStudyTypes(url string, client *http.Client) ([]studyType, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedStudyTypes []studyType
+	var itemsPerPage = 100
+
+	responses, err := paginatedFetch(url, itemsPerPage, client)
+
+	if err != nil {
+		slog.Error("Error while fetching study types", err)
+		return nil, err
+	}
+
+	for _, response := range responses {
+		var types studyTypes
+		err = json.Unmarshal([]byte(response), &types)
+		if err != nil {
+			slog.Error("Error while unmarshalling study types", err)
+			return nil, err
+		}
+
+		fetchedStudyTypes = append(fetchedStudyTypes, types.Types...)
+	}
+
+	return fetchedStudyTypes, nil
+}

backend/service/fetch/v3/paginatedFetch.go

@@ -0,0 +1,104 @@
+package v3
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type hydraResponse struct {
+	TotalItems int       `json:"totalItems"`
+	View       hydraView `json:"hydra:view"`
+}
+
+type hydraView struct {
+	First string `json:"hydra:first"`
+	Last  string `json:"hydra:last"`
+	Next  string `json:"hydra:next"`
+}
+
+func paginatedFetch(url string, itemsPerPage int, client *http.Client) ([]string, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/studiengangstypen?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var firstPage = 1
+	var responses []string
+	link := url + "?page=" + strconv.Itoa(firstPage) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
+	response, err := requestPage(link, client)
+
+	if err != nil {
+		slog.Error("Error while fetching paginated api", err)
+		return nil, err
+	}
+
+	//extract the first and last page from the response
+	var hydra hydraResponse
+	err = json.Unmarshal([]byte(response), &hydra)
+	if err != nil {
+		slog.Error("Error while unmarshalling hydra response", err, link)
+		return nil, err
+	}
+	var lastPage = extractPageNumber(hydra.View.Last)
+	responses = append(responses, response)
+
+	// prepare the links for the multithreaded requests
+	var links []string
+	for i := firstPage + 1; i <= lastPage; i++ {
+		link := url + "?page=" + strconv.Itoa(i) + "&itemsPerPage=" + strconv.Itoa(itemsPerPage)
+		links = append(links, link)
+	}
+
+	//multithreading webpage requests to speed up the process
+
+	var maxThreads = 20
+	var htmlPageArray = make([]string, len(links))
+
+	var wg sync.WaitGroup
+	wg.Add(maxThreads)
+	for i := 0; i < maxThreads; i++ {
+		go func(i int) {
+			for j := i; j < len(links); j += maxThreads {
+				slog.Info("Fetching page: " + strconv.Itoa(j) + " of " + strconv.Itoa(len(links)))
+				doc, err := requestPage(links[j], client)
+				if err == nil {
+					htmlPageArray[j] = doc
+				}
+			}
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+
+	responses = append(responses, htmlPageArray...)
+
+	return responses, nil
+}
+
+func requestPage(url string, client *http.Client) (string, error) {
+	response, err := requestJSON(url, client)
+	if err != nil {
+		slog.Error("Error while fetching paginated api", err)
+		return "", err
+	}
+	return response, nil
+}
+
+func extractPageNumber(url string) int {
+
+	if url == "" {
+		return 0
+	}
+	split := strings.Split(url, "page=")
+	lastPart := split[len(split)-1]
+	pageNumber, err := strconv.Atoi(lastPart)
+	if err != nil {
+		slog.Error("Error while extracting page number", err)
+		return 0
+	}
+	return pageNumber
+}

backend/service/functions/string.go

@@ -51,3 +51,7 @@ func SeperateRoomString(rooms string) []string {
 		[]rune{',', '\t', '\n', '\r', ';', ' ', '\u00A0'}),
 	)
 }
+
+func RemoveIriPrefix(iri string, idSize int) string {
+	return iri[len(iri)-idSize:]
+}