feat:#39 added module and prof fetch

backend/service/fetch/v3/apiModel.go

@@ -51,9 +51,9 @@ type Events struct {
 type Event struct {
 	ID            string   `json:"id"`
 	Faculty       string   `json:"fakultaet"`
-	SeminarGroups []string `json:"seminargruppen"`
+	SeminarGroups []string `json:"studierendengruppen"`
 	Flags         []string `json:"flags"`
-	Modul         string   `json:"modul"`
+	Module        string   `json:"modul"`
 	EventType     string   `json:"veranstaltungstyp"`
 	Professors    []string `json:"dozierende"`
 	Rooms         []string `json:"raeume"`
@@ -64,4 +64,5 @@ type Event struct {
 	EndTime       string   `json:"endAt"`
 	CalendarWeeks []int    `json:"kalenderwochen"`
 	Semester      string   `json:"semester"`
+	BookedAt      string   `json:"internalUpdatedAt"`
 }

backend/service/fetch/v3/fetchEventType.go

@@ -0,0 +1,50 @@
+package v3
+
+/*
+
+{
+  "hydra:member": [
+    {
+      "@context": "string",
+      "@id": "string",
+      "@type": "string",
+      "id": "string"
+    }
+  ],
+  "hydra:totalItems": 0,
+  "hydra:view": {
+    "@id": "string",
+    "type": "string",
+    "hydra:first": "string",
+    "hydra:last": "string",
+    "hydra:previous": "string",
+    "hydra:next": "string"
+  },
+  "hydra:search": {
+    "@type": "string",
+    "hydra:template": "string",
+    "hydra:variableRepresentation": "string",
+    "hydra:mapping": [
+      {
+        "@type": "string",
+        "variable": "string",
+        "property": "string",
+        "required": true
+      }
+    ]
+  }
+}
+
+*/
+
+// EventType Model for fetching json data
+
+type EventTypes struct {
+	TotalItems int         `json:"hydra:totalItems"`
+	EventTypes []EventType `json:"hydra:member"`
+}
+
+type EventType struct {
+	ID   string `json:"id"`
+	Type string `json:"@type"`
+}

backend/service/fetch/v3/fetchEvents.go

@@ -2,6 +2,7 @@ package v3
 
 import (
 	"encoding/json"
+	"htwkalender/service/functions"
 	"log/slog"
 	"net/http"
 )
@@ -13,7 +14,7 @@ func parseEvents(url string, client *http.Client) (Events, error) {
 	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
 
 	var fetchedEvents Events
-	var itemsPerPage = 100
+	var itemsPerPage = 999
 
 	responses, err := paginatedFetch(url, itemsPerPage, client)
 
@@ -30,6 +31,24 @@ func parseEvents(url string, client *http.Client) (Events, error) {
 			return Events{}, err
 		}
 
+		// cut api iri prefix
+		for i, event := range events.Events {
+
+			events.Events[i].EventType = functions.RemoveIriPrefix(event.EventType, 32)
+			events.Events[i].Faculty = functions.RemoveIriPrefix(event.Faculty, 32)
+			events.Events[i].Module = functions.RemoveIriPrefix(event.Module, 32)
+			events.Events[i].Semester = functions.RemoveIriPrefix(event.Semester, 2)
+			for j, professors := range event.Professors {
+				events.Events[i].Professors[j] = functions.RemoveIriPrefix(professors, 32)
+			}
+			for j, rooms := range event.Rooms {
+				events.Events[i].Rooms[j] = functions.RemoveIriPrefix(rooms, 32)
+			}
+			for j, courses := range event.Courses {
+				events.Events[i].Courses[j] = functions.RemoveIriPrefix(courses, 32)
+			}
+		}
+
 		fetchedEvents.Events = append(fetchedEvents.Events, events.Events...)
 		fetchedEvents.TotalItems = events.TotalItems
 	}

backend/service/fetch/v3/fetchFaculty.go

@@ -13,7 +13,7 @@ func parseFaculties(url string, client *http.Client) (faculties, error) {
 	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
 
 	var fetchedFaculties []faculty
-	var itemsPerPage = 100
+	var itemsPerPage = 999
 
 	responses, err := paginatedFetch(url, itemsPerPage, client)
 

backend/service/fetch/v3/fetchLunaApi.go

@@ -2,16 +2,15 @@ package v3
 
 import (
 	"github.com/pocketbase/pocketbase"
-	"github.com/pocketbase/pocketbase/models"
+	"github.com/pocketbase/pocketbase/tools/types"
 	"golang.org/x/net/http2"
 	"htwkalender/model"
-	"htwkalender/service/db"
 	"log/slog"
 	"net/http"
 	"strconv"
 )
 
-func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) {
+func FetchAllResources(app *pocketbase.PocketBase) (*[]model.Event, error) {
 	var groups []model.SeminarGroup
 	client := &http.Client{}
 	client.Transport = &http2.Transport{}
@@ -21,7 +20,7 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) {
 	seminarUrl := apiUrl + "studierendengruppen"
 
 	parsedSeminarGroups, err := parseSeminarGroups(seminarUrl, client)
-
+	slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems))
 	if err != nil {
 		slog.Error("Error while fetching seminar groups", err)
 		return nil, err
@@ -30,7 +29,7 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) {
 	studyTypeUrl := apiUrl + "studiengaenge"
 
 	parsedStudyTypes, err := parseStudyTypes(studyTypeUrl, client)
-
+	slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes)))
 	if err != nil {
 		slog.Error("Error while fetching study types", err)
 		return nil, err
@@ -38,15 +37,31 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) {
 
 	facultyUrl := apiUrl + "fakultaeten"
 	parsedFaculties, err := parseFaculties(facultyUrl, client)
-
+	slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties)))
 	if err != nil {
 		slog.Error("Error while fetching faculties", err)
 		return nil, err
 	}
 
-	slog.Info("Fetched study types: " + strconv.Itoa(len(parsedStudyTypes)))
-	slog.Info("Fetched seminar groups: " + strconv.Itoa(len(parsedSeminarGroups.Groups)) + " of " + strconv.Itoa(parsedSeminarGroups.TotalItems))
-	slog.Info("Fetched faculties: " + strconv.Itoa(len(parsedFaculties.Faculties)))
+	moduleUrl := apiUrl + "module"
+	parsedModules, err := parseModules(moduleUrl, client)
+	slog.Info("Fetched modules: " + strconv.Itoa(len(parsedModules.Modules)))
+	if err != nil {
+		slog.Error("Error while fetching modules", err)
+		return nil, err
+	}
+
+	roomUrl := apiUrl + "raeume"
+	parsedRooms, err := parseRooms(roomUrl, client)
+	slog.Info("Fetched rooms: " + strconv.Itoa(len(parsedRooms.Rooms)))
+	if err != nil {
+		slog.Error("Error while fetching rooms", err)
+		return nil, err
+	}
+
+	profUrl := apiUrl + "dozierende"
+	parsedProfs, err := parseProfs(profUrl, client)
+	slog.Info("Fetched profs: " + strconv.Itoa(len(parsedProfs.Professors)))
 
 	// map seminar groups to model seminar groups
 	for _, group := range parsedSeminarGroups.Groups {
@@ -76,30 +91,106 @@ func FetchAllResources(app *pocketbase.PocketBase) ([]*models.Record, error) {
 		groups = append(groups, newGroup)
 	}
 
-	collection, dbError := db.FindCollection(app, "groups")
-	if dbError != nil {
-		slog.Error("Error while searching collection groups", dbError)
-		return nil, err
-	}
-	var insertedGroups []*models.Record
-
-	insertedGroups, dbError = db.SaveGroups(groups, collection, app)
-	if dbError != nil {
-		slog.Error("Error while saving groups", dbError)
-		return nil, err
-	}
-
 	//Now fetch all events
-
 	eventUrl := apiUrl + "veranstaltungen"
 	parsedEvents, err := parseEvents(eventUrl, client)
+	slog.Info("Fetched events: " + strconv.Itoa(len(parsedEvents.Events)) + " of " + strconv.Itoa(parsedEvents.TotalItems))
 
 	if err != nil {
 		slog.Error("Error while fetching events", err)
 		return nil, err
 	}
 
-	slog.Info("Fetched events: " + strconv.Itoa(len(parsedEvents.Events)) + " of " + strconv.Itoa(parsedEvents.TotalItems))
+	//resolve course id in parsedEvents to course name
+	for i, event := range parsedEvents.Events {
+		for j, course := range event.Courses {
+			for _, group := range parsedStudyTypes {
+				if group.ID == course {
+					parsedEvents.Events[i].Courses[j] = group.GroupID
+				}
+			}
+		}
 
-	return insertedGroups, nil
+		for _, group := range parsedSeminarGroups.Groups {
+			for courseItemId, seminarGroupItem := range event.SeminarGroups {
+				if group.ID == seminarGroupItem {
+					parsedEvents.Events[i].SeminarGroups[courseItemId] = group.SeminarGroup
+				}
+			}
+		}
+
+		for _, facultyItem := range parsedFaculties.Faculties {
+			if facultyItem.ID == event.Faculty {
+				parsedEvents.Events[i].Faculty = facultyItem.ShortCut
+			}
+		}
+
+		for _, module := range parsedModules.Modules {
+			if module.ID == event.Module {
+				parsedEvents.Events[i].Module = module.Name
+			}
+		}
+
+		for _, room := range parsedRooms.Rooms {
+			for roomPlace, roomID := range event.Rooms {
+				if room.ID == roomID {
+					parsedEvents.Events[i].Rooms[roomPlace] = room.ShortCut
+				}
+			}
+		}
+
+		for _, prof := range parsedProfs.Professors {
+			for profPlace, profID := range event.Professors {
+				if prof.ID == profID {
+					parsedEvents.Events[i].Professors[profPlace] = prof.Description1
+				}
+			}
+		}
+	}
+
+	// map events to model events
+	var events []model.Event
+
+	for _, event := range parsedEvents.Events {
+		for _, week := range event.CalendarWeeks {
+			for _, course := range event.Courses {
+				var newEvent model.Event
+				newEvent.UUID = ""
+				newEvent.Day = event.Day
+				newEvent.Week = strconv.Itoa(week)
+				newEvent.Start, _ = types.ParseDateTime(event.StartTime)
+				newEvent.End, _ = types.ParseDateTime(event.EndTime)
+				newEvent.Name = event.Module
+				newEvent.EventType = event.EventType
+				newEvent.Compulsory = ""
+				newEvent.Prof = concatStringArray(event.Professors, "|")
+				newEvent.Rooms = concatStringArray(event.Rooms, " ")
+				newEvent.Notes = event.Description
+				newEvent.BookedAt = event.BookedAt
+				newEvent.Course = course
+				newEvent.Semester = event.Semester
+				events = append(events, newEvent)
+			}
+		}
+	}
+
+	// return events
+	return &events, nil
+}
+
+func concatStringArray(array []string, concatChar string) string {
+	var result string
+
+	if len(array) == 0 {
+		return result
+	}
+
+	if len(array) > 1 {
+		for _, item := range array[:len(array)-1] {
+			result += item + concatChar
+		}
+	}
+	result += array[len(array)-1]
+
+	return result
 }

backend/service/fetch/v3/fetchModule.go

@@ -0,0 +1,114 @@
+package v3
+
+import (
+	"encoding/json"
+	"htwkalender/service/functions"
+	"log/slog"
+	"net/http"
+)
+
+/*
+
+{
+  "@context": "/api/contexts/Modul",
+  "@id": "/api/module",
+  "@type": "hydra:Collection",
+  "hydra:totalItems": 1678,
+  "hydra:member": [
+    {
+      "@id": "/api/module/0055CE2545A3411A00C32F30A645463B",
+      "@type": "Modul",
+      "id": "0055CE2545A3411A00C32F30A645463B",
+      "semester": "/api/semester/ws",
+      "fakultaet": "/api/fakultaeten/9B89016985E5156B1C033BB0FD3AF9B4",
+      "kuerzel": "PPKMSBM&SMM&STM3",
+      "bezeichnung": "W833 Produkt- und Prozesskostenmanagement  SBM & SMM & STM 3. FS (wpf)",
+      "hinweisOne": "",
+      "internalUpdatedAt": "2023-03-07T07:35:34+01:00"
+    }
+  ],
+  "hydra:view": {
+    "@id": "/api/module?itemsPerPage=100&page=1",
+    "@type": "hydra:PartialCollectionView",
+    "hydra:first": "/api/module?itemsPerPage=100&page=1",
+    "hydra:last": "/api/module?itemsPerPage=100&page=17",
+    "hydra:next": "/api/module?itemsPerPage=100&page=2"
+  },
+  "hydra:search": {
+    "@type": "hydra:IriTemplate",
+    "hydra:template": "/api/module{?veranstaltungen,veranstaltungen[]}",
+    "hydra:variableRepresentation": "BasicRepresentation",
+    "hydra:mapping": [
+      {
+        "@type": "IriTemplateMapping",
+        "variable": "veranstaltungen",
+        "property": "veranstaltungen",
+        "required": false
+      },
+      {
+        "@type": "IriTemplateMapping",
+        "variable": "veranstaltungen[]",
+        "property": "veranstaltungen",
+        "required": false
+      }
+    ]
+  }
+}
+
+
+*/
+
+// Module Model for fetching json data
+
+type Modules struct {
+	TotalItems int      `json:"hydra:totalItems"`
+	Modules    []Module `json:"hydra:member"`
+}
+
+type Module struct {
+	ID       string `json:"id"`
+	Semester string `json:"semester"`
+	Faculty  string `json:"fakultaet"`
+	ShortCut string `json:"kuerzel"`
+	Name     string `json:"bezeichnung"`
+	Note     string `json:"hinweisOne"`
+	Internal string `json:"internalUpdatedAt"`
+}
+
+func parseModules(url string, client *http.Client) (Modules, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/module?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedModules Modules
+	var itemsPerPage = 999
+
+	responses, err := paginatedFetch(url, itemsPerPage, client)
+
+	if err != nil {
+		slog.Error("Error while fetching modules", err)
+		return Modules{}, err
+	}
+
+	for _, response := range responses {
+		var modules Modules
+		err = json.Unmarshal([]byte(response), &modules)
+		if err != nil {
+			slog.Error("Error while unmarshalling modules", err)
+			return Modules{}, err
+		}
+
+		// cut api iri prefix
+		for i, module := range modules.Modules {
+
+			modules.Modules[i].Faculty = functions.RemoveIriPrefix(module.Faculty, 32)
+			modules.Modules[i].Semester = functions.RemoveIriPrefix(module.Semester, 2)
+		}
+
+		fetchedModules.Modules = append(fetchedModules.Modules, modules.Modules...)
+		fetchedModules.TotalItems = modules.TotalItems
+	}
+
+	return fetchedModules, nil
+}

backend/service/fetch/v3/fetchProfs.go

@@ -0,0 +1,102 @@
+package v3
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+)
+
+/*
+
+{
+  "hydra:member": [
+    {
+      "@context": "string",
+      "@id": "string",
+      "@type": "string",
+      "id": "string",
+      "kuerzel": "string",
+      "nachname": "string",
+      "fakultaet": "https://example.com/",
+      "bezeichnung": "string",
+      "bezeichnungOne": "string"
+    }
+  ],
+  "hydra:totalItems": 0,
+  "hydra:view": {
+    "@id": "string",
+    "type": "string",
+    "hydra:first": "string",
+    "hydra:last": "string",
+    "hydra:previous": "string",
+    "hydra:next": "string"
+  },
+  "hydra:search": {
+    "@type": "string",
+    "hydra:template": "string",
+    "hydra:variableRepresentation": "string",
+    "hydra:mapping": [
+      {
+        "@type": "string",
+        "variable": "string",
+        "property": "string",
+        "required": true
+      }
+    ]
+  }
+}
+
+*/
+
+// professor Model for fetching json data
+
+type Professors struct {
+	TotalItems int         `json:"hydra:totalItems"`
+	Professors []Professor `json:"hydra:member"`
+}
+
+type Professor struct {
+	ID           string `json:"id"`
+	ShortCut     string `json:"kuerzel"`
+	LastName     string `json:"nachname"`
+	Faculty      string `json:"fakultaet"`
+	Description  string `json:"bezeichnung"`
+	Description1 string `json:"bezeichnungOne"`
+}
+
+func parseProfs(url string, client *http.Client) (Professors, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/dozierende?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedProfs Professors
+	var itemsPerPage = 999
+
+	responses, err := paginatedFetch(url, itemsPerPage, client)
+
+	if err != nil {
+		slog.Error("Error while fetching paginated api", err)
+		return fetchedProfs, err
+	}
+
+	for _, response := range responses {
+		var hydra hydraResponse
+		err = json.Unmarshal([]byte(response), &hydra)
+		if err != nil {
+			slog.Error("Error while unmarshalling hydra response", err, url)
+			return fetchedProfs, err
+		}
+
+		var professors Professors
+		err = json.Unmarshal([]byte(response), &professors)
+		if err != nil {
+			slog.Error("Error while unmarshalling professors", err)
+			return fetchedProfs, err
+		}
+		fetchedProfs.TotalItems = professors.TotalItems
+		fetchedProfs.Professors = append(fetchedProfs.Professors, professors.Professors...)
+	}
+
+	return fetchedProfs, nil
+}

backend/service/fetch/v3/fetchRooms.go

@@ -0,0 +1,93 @@
+package v3
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+)
+
+/*
+
+{
+  "hydra:member": [
+    {
+      "@context": "string",
+      "@id": "string",
+      "@type": "string",
+      "id": "string",
+      "kuerzel": "string",
+      "beschreibung": "string",
+      "hinweisOne": "string",
+      "hinweisTwo": "string",
+      "plaetze": 0
+    }
+  ],
+  "hydra:totalItems": 0,
+  "hydra:view": {
+    "@id": "string",
+    "type": "string",
+    "hydra:first": "string",
+    "hydra:last": "string",
+    "hydra:previous": "string",
+    "hydra:next": "string"
+  },
+  "hydra:search": {
+    "@type": "string",
+    "hydra:template": "string",
+    "hydra:variableRepresentation": "string",
+    "hydra:mapping": [
+      {
+        "@type": "string",
+        "variable": "string",
+        "property": "string",
+        "required": true
+      }
+    ]
+  }
+}
+
+*/
+
+// room Model for fetching json data
+
+type Rooms struct {
+	TotalItems int    `json:"hydra:totalItems"`
+	Rooms      []Room `json:"hydra:member"`
+}
+
+type Room struct {
+	ID          string `json:"id"`
+	ShortCut    string `json:"kuerzel"`
+	Description string `json:"beschreibung"`
+	Seats       int    `json:"plaetze"`
+	Note1       string `json:"hinweisOne"`
+	Note2       string `json:"hinweisTwo"`
+}
+
+func parseRooms(url string, client *http.Client) (Rooms, error) {
+
+	// the url is paginated, so we need to fetch all pages
+	// example url: https://luna.htwk-leipzig.de/api/raum?page=1&itemsPerPage=100
+	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
+
+	var fetchedRooms Rooms
+	var itemsPerPage = 999
+
+	responses, err := paginatedFetch(url, itemsPerPage, client)
+
+	if err != nil {
+		slog.Error("Error while fetching paginated api", err)
+		return fetchedRooms, err
+	}
+
+	for _, response := range responses {
+		var rooms Rooms
+		err = json.Unmarshal([]byte(response), &rooms)
+		if err != nil {
+			slog.Error("Error while unmarshalling rooms", err)
+			return fetchedRooms, err
+		}
+		fetchedRooms.Rooms = append(fetchedRooms.Rooms, rooms.Rooms...)
+	}
+	return fetchedRooms, nil
+}

backend/service/fetch/v3/fetchSeminarGroupService.go

@@ -16,7 +16,7 @@ func parseSeminarGroups(url string, client *http.Client) (seminarGroups, error)
 
 	var fetchedSeminarGroups []seminarGroup
 	var page = 1
-	var itemsPerPage = 100
+	var itemsPerPage = 999
 	var totalItems = 0
 
 	responses, err := paginatedFetch(url, itemsPerPage, client)

backend/service/fetch/v3/fetchStudyTypes.go

@@ -13,7 +13,7 @@ func parseStudyTypes(url string, client *http.Client) ([]studyType, error) {
 	// the itemsPerPage is set to 100, so we need to fetch all pages until we get an empty response
 
 	var fetchedStudyTypes []studyType
-	var itemsPerPage = 100
+	var itemsPerPage = 999
 
 	responses, err := paginatedFetch(url, itemsPerPage, client)
 

backend/service/fetch/v3/paginatedFetch.go

@@ -63,7 +63,6 @@ func paginatedFetch(url string, itemsPerPage int, client *http.Client) ([]string
 	for i := 0; i < maxThreads; i++ {
 		go func(i int) {
 			for j := i; j < len(links); j += maxThreads {
-				slog.Info("Fetching page: " + strconv.Itoa(j) + " of " + strconv.Itoa(len(links)))
 				doc, err := requestPage(links[j], client)
 				if err == nil {
 					htmlPageArray[j] = doc

backend/service/functions/string.go

@@ -46,5 +46,8 @@ func SeperateRoomString(rooms string) []string {
 }
 
 func RemoveIriPrefix(iri string, idSize int) string {
+	if len(iri) < idSize {
+		return ""
+	}
 	return iri[len(iri)-idSize:]
 }