package service import ( "fmt" "github.com/labstack/echo/v5" "github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase/apis" "github.com/pocketbase/pocketbase/models" "golang.org/x/net/html" "htwk-planner/model" "io" "net/http" "strings" ) func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error { var seminarGroups []model.SeminarGroup seminarGroupsLabel := []string{ "22INB-3", "22INB-2", "22INB-1", "21INB-2", "21INB-1", "22INM", //"21INM (Masterarbeit)", "22MIB-BIN", "22MIB-2", "22MIB-1", "21MIB-BIN", "21MIB-1", "21MIB-2", //"20MIB (Praxis/Bachelorarbeit)", "22MIM", //"21MIM (Masterarbeit)" } for _, seminarGroupLabel := range seminarGroupsLabel { result, err := getPlanHTML("ss", seminarGroupLabel) seminarGroup := parseSeminarGroup(result) if err != nil { return apis.NewNotFoundError("The Data could not be fetched", err) } collection, err := app.Dao().FindCollectionByNameOrId("events") if err != nil { return err } for _, event := range seminarGroup.Events { record := models.NewRecord(collection) record.Set("Day", event.Day) record.Set("Week", event.Week) record.Set("Start", event.Start) record.Set("End", event.End) record.Set("Name", event.Name) record.Set("EventType", event.EventType) record.Set("Prof", event.Prof) record.Set("Rooms", event.Rooms) record.Set("Notes", event.Notes) record.Set("BookedAt", event.BookedAt) if err := app.Dao().SaveRecord(record); err != nil { return err } } seminarGroups = append(seminarGroups, seminarGroup) } return c.JSON(http.StatusOK, seminarGroups) } func parseSeminarGroup(result string) model.SeminarGroup { doc, _ := html.Parse(strings.NewReader(result)) table := findFirstTable(doc) events := toEvents(getEventTables(doc), getAllDayLabels(doc)) var seminarGroup = model.SeminarGroup{ University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data, GroupShortcut: findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data, Events: events, } return seminarGroup } func toEvents(tables [][]*html.Node, days []string) []model.Events { var events []model.Events for table := range tables { for row := range tables[table] { tableData := findTableData(tables[table][row]) if len(tableData) > 0 { events = append(events, model.Events{ Day: days[table], Week: getTextContent(tableData[0]), Start: getTextContent(tableData[1]), End: getTextContent(tableData[2]), Name: getTextContent(tableData[3]), EventType: getTextContent(tableData[4]), Prof: getTextContent(tableData[5]), Rooms: getTextContent(tableData[6]), Notes: getTextContent(tableData[7]), BookedAt: getTextContent(tableData[8]), }) } } } return events } func toUtf8(iso8859_1_buf []byte) string { buf := make([]rune, len(iso8859_1_buf)) for i, b := range iso8859_1_buf { buf[i] = rune(b) } return string(buf) } func getPlanHTML(semester string, matrikel string) (string, error) { url := "https://stundenplan.htwk-leipzig.de/" + string(semester) + "/Berichte/Text-Listen;Studenten-Sets;name;" + matrikel + "?template=sws_semgrp&weeks=1-65" // Send GET request response, err := http.Get(url) if err != nil { fmt.Printf("Error occurred while making the request: %s\n", err.Error()) return "", err } defer func(Body io.ReadCloser) { err := Body.Close() if err != nil { return } }(response.Body) // Read the response body body, err := io.ReadAll(response.Body) if err != nil { fmt.Printf("Error occurred while reading the response: %s\n", err.Error()) return "", err } return toUtf8(body), err } // Find the first element in the HTML document func findFirstTable(node *html.Node) *html.Node { if node.Type == html.ElementNode && node.Data == "table" { return node } // Traverse child nodes recursively for child := node.FirstChild; child != nil; child = child.NextSibling { found := findFirstTable(child) if found != nil { return found } } return nil } // Find the first element with the specified class attribute value func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node { if node.Type == html.ElementNode && node.Data == "span" { if hasClassAttribute(node, classValue) { return node } } // Traverse child nodes recursively for child := node.FirstChild; child != nil; child = child.NextSibling { found := findFirstSpanWithClass(child, classValue) if found != nil { return found } } return nil } // Check if the specified element has the specified class attribute value func hasClassAttribute(node *html.Node, classValue string) bool { for _, attr := range node.Attr { if attr.Key == "class" && strings.Contains(attr.Val, classValue) { return true } } return false } // Get Tables with days func getEventTables(node *html.Node) [][]*html.Node { var eventTables [][]*html.Node tables := findTables(node) for events := range tables { rows := findTableRows(tables[events]) rows = rows[1:] eventTables = append(eventTables, rows) } return eventTables } // Get Tables with days func getAllDayLabels(node *html.Node) []string { paragraphs := findParagraphs(node) var dayArray []string for _, p := range paragraphs { label := getDayLabel(p) if label != "" { dayArray = append(dayArray, label) } } return dayArray } // Find all

elements in the HTML document func findParagraphs(node *html.Node) []*html.Node { var paragraphs []*html.Node if node.Type == html.ElementNode && node.Data == "p" { paragraphs = append(paragraphs, node) } for child := node.FirstChild; child != nil; child = child.NextSibling { paragraphs = append(paragraphs, findParagraphs(child)...) } return paragraphs } // Find all

elements in , excluding the first one func findTableRows(node *html.Node) []*html.Node { var tableRows []*html.Node if node.Type == html.ElementNode && node.Data == "tbody" { child := node.FirstChild for child != nil { if child.Type == html.ElementNode && child.Data == "tr" { tableRows = append(tableRows, child) } child = child.NextSibling } } for child := node.FirstChild; child != nil; child = child.NextSibling { tableRows = append(tableRows, findTableRows(child)...) } return tableRows } // Find all

elements in the HTML document func findTables(node *html.Node) []*html.Node { var tables []*html.Node if node.Type == html.ElementNode && node.Data == "table" { tables = append(tables, node) } for child := node.FirstChild; child != nil; child = child.NextSibling { tables = append(tables, findDayTables(child)...) } return tables } // Find all

elements in the HTML document func findDayTables(node *html.Node) []*html.Node { var tables []*html.Node for child := node.FirstChild; child != nil; child = child.NextSibling { tables = append(tables, findDayTables(child)...) } if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") { tables = append(tables, node) } return tables } // Get the text content of the specified node and its descendants func getDayLabel(node *html.Node) string { child := node.FirstChild if child != nil { if child.Type == html.ElementNode && child.Data == "span" { if child.FirstChild != nil { return child.FirstChild.Data } } } return "" } // Find all

func findTableData(node *html.Node) []*html.Node { var tableData []*html.Node if node.Type == html.ElementNode && node.Data == "tr" { child := node.FirstChild for child != nil { if child.Type == html.ElementNode && child.Data == "td" { tableData = append(tableData, child) } child = child.NextSibling } } return tableData } func getTextContent(node *html.Node) string { var textContent string if node.Type == html.TextNode { textContent = node.Data } for child := node.FirstChild; child != nil; child = child.NextSibling { textContent += getTextContent(child) } return textContent }
elements in the current