package fetch import ( "fmt" "github.com/labstack/echo/v5" "github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase/apis" "github.com/pocketbase/pocketbase/models" "golang.org/x/net/html" "htwk-planner/model" "htwk-planner/service/date" "htwk-planner/service/db" "io" "net/http" "regexp" "strconv" "strings" "time" ) func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error { var seminarGroups []model.SeminarGroup //seminarGroupsLabel := []string{ // "22EIM-MET", //} seminarGroupsLabel := db.GetAllCourses(app) for _, seminarGroupLabel := range seminarGroupsLabel { result, getError := getPlanHTML("ws", seminarGroupLabel) if getError == nil { seminarGroup := parseSeminarGroup(result) seminarGroups = append(seminarGroups, seminarGroup) } } //Save SeminarGroups to DB print("Saving SeminarGroups to DB...\n") collection, dbError := findCollection(app, "events") if dbError != nil { return apis.NewNotFoundError("Collection not found", dbError) } dbError = db.SaveEvents(seminarGroups, collection, app) if dbError != nil { return apis.NewApiError(400, "Could not save Event into database", dbError) } return c.JSON(http.StatusOK, seminarGroups) } func findCollection(app *pocketbase.PocketBase, collectionName string) (*models.Collection, error) { collection, dbError := app.Dao().FindCollectionByNameOrId(collectionName) return collection, dbError } func parseSeminarGroup(result string) model.SeminarGroup { doc, err := html.Parse(strings.NewReader(result)) if err != nil { fmt.Printf("Error occurred while parsing the HTML document: %s\n", err.Error()) return model.SeminarGroup{} } table := findFirstTable(doc) eventTables := getEventTables(doc) allDayLabels := getAllDayLabels(doc) if eventTables == nil || allDayLabels == nil { return model.SeminarGroup{} } eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels) splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks) events := splitEventsBySingleWeek(splitEventsByWeekVal) semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data semester, year := extractSemesterAndYear(semesterString) events = convertWeeksToDates(events, semester, year) var seminarGroup = model.SeminarGroup{ University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data, Course: findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data, Events: events, } return seminarGroup } func convertWeeksToDates(events []model.Event, semester string, year string) []model.Event { var newEvents []model.Event eventYear, _ := strconv.Atoi(year) // for each event we need to calculate the start and end date based on the week and the year for _, event := range events { eventWeek, _ := strconv.Atoi(event.Week) eventDay, _ := date.GetDateFromWeekNumber(eventYear, eventWeek, event.Day) start := addTimeToDate(eventDay, event.Start) end := addTimeToDate(eventDay, event.End) newEvent := event newEvent.Start = start.String() newEvent.End = end.String() newEvent.Semester = semester newEvents = append(newEvents, newEvent) } return newEvents } func addTimeToDate(date time.Time, timeString string) time.Time { //convert time string to time timeParts := strings.Split(timeString, ":") hour, _ := strconv.Atoi(timeParts[0]) minute, _ := strconv.Atoi(timeParts[1]) return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, time.UTC) } func extractSemesterAndYear(semesterString string) (string, string) { winterPattern := "Wintersemester" summerPattern := "Sommersemester" winterMatch := strings.Contains(semesterString, winterPattern) summerMatch := strings.Contains(semesterString, summerPattern) semester := "" semesterShortcut := "" if winterMatch { semester = "Wintersemester" semesterShortcut = "ws" } else if summerMatch { semester = "Sommersemester" semesterShortcut = "ss" } else { return "", "" } yearPattern := `\d{4}` combinedPattern := semester + `\s` + yearPattern re := regexp.MustCompile(combinedPattern) match := re.FindString(semesterString) year := "" if match != "" { reYear := regexp.MustCompile(yearPattern) year = reYear.FindString(match) } return semesterShortcut, year } func toEvents(tables [][]*html.Node, days []string) []model.Event { var events []model.Event for table := range tables { for row := range tables[table] { tableData := findTableData(tables[table][row]) if len(tableData) > 0 { events = append(events, model.Event{ Day: days[table], Week: getTextContent(tableData[0]), Start: getTextContent(tableData[1]), End: getTextContent(tableData[2]), Name: getTextContent(tableData[3]), EventType: getTextContent(tableData[4]), Prof: getTextContent(tableData[5]), Rooms: getTextContent(tableData[6]), Notes: getTextContent(tableData[7]), BookedAt: getTextContent(tableData[8]), }) } } } return events } func splitEventsByWeek(events []model.Event) []model.Event { var newEvents []model.Event for _, event := range events { weeks := strings.Split(event.Week, ",") for _, week := range weeks { newEvent := event newEvent.Week = strings.TrimSpace(week) newEvents = append(newEvents, newEvent) } } return newEvents } func splitEventsBySingleWeek(events []model.Event) []model.Event { var newEvents []model.Event for _, event := range events { if strings.Contains(event.Week, "-") { weeks := splitWeekRange(event.Week) for _, week := range weeks { newEvent := event newEvent.Week = week newEvents = append(newEvents, newEvent) } } else { newEvents = append(newEvents, event) } } return newEvents } func splitWeekRange(weekRange string) []string { parts := strings.Split(weekRange, "-") if len(parts) != 2 { return nil // Invalid format } start, errStart := strconv.Atoi(strings.TrimSpace(parts[0])) end, errEnd := strconv.Atoi(strings.TrimSpace(parts[1])) if errStart != nil || errEnd != nil { return nil // Error converting to integers } var weeks []string for i := start; i <= end; i++ { weeks = append(weeks, strconv.Itoa(i)) } return weeks } func toUtf8(iso88591Buf []byte) string { buf := make([]rune, len(iso88591Buf)) for i, b := range iso88591Buf { buf[i] = rune(b) } return string(buf) } func getPlanHTML(semester string, matrikel string) (string, error) { url := "https://stundenplan.htwk-leipzig.de/" + string(semester) + "/Berichte/Text-Listen;Studenten-Sets;name;" + matrikel + "?template=sws_semgrp&weeks=1-65" // Send GET request response, err := http.Get(url) if err != nil { fmt.Printf("Error occurred while making the request: %s\n", err.Error()) return "", err } defer func(Body io.ReadCloser) { err := Body.Close() if err != nil { return } }(response.Body) // Read the response body body, err := io.ReadAll(response.Body) if err != nil { fmt.Printf("Error occurred while reading the response: %s\n", err.Error()) return "", err } return toUtf8(body), err } // Find the first element in the HTML document func findFirstTable(node *html.Node) *html.Node { if node.Type == html.ElementNode && node.Data == "table" { return node } // Traverse child nodes recursively for child := node.FirstChild; child != nil; child = child.NextSibling { found := findFirstTable(child) if found != nil { return found } } return nil } // Find the first element with the specified class attribute value func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node { if node.Type == html.ElementNode && node.Data == "span" { if hasClassAttribute(node, classValue) { return node } } // Traverse child nodes recursively for child := node.FirstChild; child != nil; child = child.NextSibling { found := findFirstSpanWithClass(child, classValue) if found != nil { return found } } return nil } // Check if the specified element has the specified class attribute value func hasClassAttribute(node *html.Node, classValue string) bool { for _, attr := range node.Attr { if attr.Key == "class" && strings.Contains(attr.Val, classValue) { return true } } return false } // Get Tables with days func getEventTables(node *html.Node) [][]*html.Node { var eventTables [][]*html.Node tables := findTables(node) // get all tables with events for events := range tables { rows := findTableRows(tables[events]) // check that a first row exists if len(rows) > 0 { rows = rows[1:] eventTables = append(eventTables, rows) } } return eventTables } // Get Tables with days func getAllDayLabels(node *html.Node) []string { paragraphs := findParagraphs(node) var dayArray []string for _, p := range paragraphs { label := getDayLabel(p) if label != "" { dayArray = append(dayArray, label) } } return dayArray } // Find all

elements in the HTML document func findParagraphs(node *html.Node) []*html.Node { var paragraphs []*html.Node if node.Type == html.ElementNode && node.Data == "p" { paragraphs = append(paragraphs, node) } for child := node.FirstChild; child != nil; child = child.NextSibling { paragraphs = append(paragraphs, findParagraphs(child)...) } return paragraphs } // Find all

elements in , excluding the first one func findTableRows(node *html.Node) []*html.Node { var tableRows []*html.Node if node.Type == html.ElementNode && node.Data == "tbody" { child := node.FirstChild for child != nil { if child.Type == html.ElementNode && child.Data == "tr" { tableRows = append(tableRows, child) } child = child.NextSibling } } // Traverse child nodes recursively for child := node.FirstChild; child != nil; child = child.NextSibling { var tableRowElement = findTableRows(child) if tableRowElement != nil { tableRows = append(tableRows, tableRowElement...) } } if tableRows == nil { return []*html.Node{} } else { return tableRows } } // Find all

elements in the HTML document func findTables(node *html.Node) []*html.Node { var tables []*html.Node if node.Type == html.ElementNode && node.Data == "table" { tables = append(tables, node) } for child := node.FirstChild; child != nil; child = child.NextSibling { tables = append(tables, findDayTables(child)...) } return tables } // Find all

elements in the HTML document func findDayTables(node *html.Node) []*html.Node { var tables []*html.Node for child := node.FirstChild; child != nil; child = child.NextSibling { tables = append(tables, findDayTables(child)...) } if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") { tables = append(tables, node) } return tables } // Get the text content of the specified node and its descendants func getDayLabel(node *html.Node) string { child := node.FirstChild if child != nil { if child.Type == html.ElementNode && child.Data == "span" { if child.FirstChild != nil { return child.FirstChild.Data } } } return "" } // Find all

func findTableData(node *html.Node) []*html.Node { var tableData []*html.Node if node.Type == html.ElementNode && node.Data == "tr" { child := node.FirstChild for child != nil { if child.Type == html.ElementNode && child.Data == "td" { tableData = append(tableData, child) } child = child.NextSibling } } return tableData } func getTextContent(node *html.Node) string { var textContent string if node.Type == html.TextNode { textContent = node.Data } for child := node.FirstChild; child != nil; child = child.NextSibling { textContent += getTextContent(child) } return textContent }
elements in the current