htwkalender/service/fetchService.go

package service

import (
	"fmt"
	"github.com/labstack/echo/v5"
	"github.com/pocketbase/pocketbase"
	"github.com/pocketbase/pocketbase/apis"
	"github.com/pocketbase/pocketbase/models"
	"golang.org/x/net/html"
	"htwk-planner/model"
	"io"
	"net/http"
	"strings"
)

func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error {

	var seminarGroups []model.SeminarGroup

	seminarGroupsLabel := []string{
		"22INB-3",
		"22INB-2",
		"22INB-1",
		"21INB-2",
		"21INB-1",
		"22INM",
		//"21INM (Masterarbeit)",
		"22MIB-BIN",
		"22MIB-2",
		"22MIB-1",
		"21MIB-BIN",
		"21MIB-1",
		"21MIB-2",
		//"20MIB (Praxis/Bachelorarbeit)",
		"22MIM",
		//"21MIM (Masterarbeit)"
	}

	for _, seminarGroupLabel := range seminarGroupsLabel {

		result, err := getPlanHTML("ss", seminarGroupLabel)

		seminarGroup := parseSeminarGroup(result)

		if err != nil {
			return apis.NewNotFoundError("The Data could not be fetched", err)
		}

		collection, err := app.Dao().FindCollectionByNameOrId("events")
		if err != nil {
			return err
		}

		for _, event := range seminarGroup.Events {
			record := models.NewRecord(collection)
			record.Set("Day", event.Day)
			record.Set("Week", event.Week)
			record.Set("Start", event.Start)
			record.Set("End", event.End)
			record.Set("Name", event.Name)
			record.Set("EventType", event.EventType)
			record.Set("Prof", event.Prof)
			record.Set("Rooms", event.Rooms)
			record.Set("Notes", event.Notes)
			record.Set("BookedAt", event.BookedAt)
			if err := app.Dao().SaveRecord(record); err != nil {
				return err
			}
		}

		seminarGroups = append(seminarGroups, seminarGroup)

	}

	return c.JSON(http.StatusOK, seminarGroups)
}

func parseSeminarGroup(result string) model.SeminarGroup {
	doc, _ := html.Parse(strings.NewReader(result))
	table := findFirstTable(doc)
	events := toEvents(getEventTables(doc), getAllDayLabels(doc))

	var seminarGroup = model.SeminarGroup{
		University:    findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data,
		GroupShortcut: findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data,
		Events:        events,
	}

	return seminarGroup
}

func toEvents(tables [][]*html.Node, days []string) []model.Events {
	var events []model.Events

	for table := range tables {
		for row := range tables[table] {

			tableData := findTableData(tables[table][row])
			if len(tableData) > 0 {
				events = append(events, model.Events{
					Day:       days[table],
					Week:      getTextContent(tableData[0]),
					Start:     getTextContent(tableData[1]),
					End:       getTextContent(tableData[2]),
					Name:      getTextContent(tableData[3]),
					EventType: getTextContent(tableData[4]),
					Prof:      getTextContent(tableData[5]),
					Rooms:     getTextContent(tableData[6]),
					Notes:     getTextContent(tableData[7]),
					BookedAt:  getTextContent(tableData[8]),
				})
			}
		}

	}

	return events
}

func toUtf8(iso8859_1_buf []byte) string {
	buf := make([]rune, len(iso8859_1_buf))
	for i, b := range iso8859_1_buf {
		buf[i] = rune(b)
	}
	return string(buf)
}

func getPlanHTML(semester string, matrikel string) (string, error) {
	url := "https://stundenplan.htwk-leipzig.de/" + string(semester) + "/Berichte/Text-Listen;Studenten-Sets;name;" + matrikel + "?template=sws_semgrp&weeks=1-65"

	// Send GET request
	response, err := http.Get(url)
	if err != nil {
		fmt.Printf("Error occurred while making the request: %s\n", err.Error())
		return "", err
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			return
		}
	}(response.Body)

	// Read the response body
	body, err := io.ReadAll(response.Body)

	if err != nil {
		fmt.Printf("Error occurred while reading the response: %s\n", err.Error())
		return "", err
	}

	return toUtf8(body), err

}

// Find the first <table> element in the HTML document
func findFirstTable(node *html.Node) *html.Node {
	if node.Type == html.ElementNode && node.Data == "table" {
		return node
	}
	// Traverse child nodes recursively
	for child := node.FirstChild; child != nil; child = child.NextSibling {
		found := findFirstTable(child)
		if found != nil {
			return found
		}
	}
	return nil
}

// Find the first <span> element with the specified class attribute value
func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node {
	if node.Type == html.ElementNode && node.Data == "span" {
		if hasClassAttribute(node, classValue) {
			return node
		}
	}

	// Traverse child nodes recursively
	for child := node.FirstChild; child != nil; child = child.NextSibling {
		found := findFirstSpanWithClass(child, classValue)
		if found != nil {
			return found
		}
	}

	return nil
}

// Check if the specified element has the specified class attribute value
func hasClassAttribute(node *html.Node, classValue string) bool {
	for _, attr := range node.Attr {
		if attr.Key == "class" && strings.Contains(attr.Val, classValue) {
			return true
		}
	}
	return false
}

// Get Tables with days
func getEventTables(node *html.Node) [][]*html.Node {
	var eventTables [][]*html.Node

	tables := findTables(node)

	for events := range tables {
		rows := findTableRows(tables[events])
		rows = rows[1:]
		eventTables = append(eventTables, rows)
	}

	return eventTables

}

// Get Tables with days
func getAllDayLabels(node *html.Node) []string {
	paragraphs := findParagraphs(node)
	var dayArray []string

	for _, p := range paragraphs {
		label := getDayLabel(p)
		if label != "" {
			dayArray = append(dayArray, label)
		}
	}
	return dayArray
}

// Find all <p> elements in the HTML document
func findParagraphs(node *html.Node) []*html.Node {
	var paragraphs []*html.Node

	if node.Type == html.ElementNode && node.Data == "p" {
		paragraphs = append(paragraphs, node)
	}

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		paragraphs = append(paragraphs, findParagraphs(child)...)
	}

	return paragraphs
}

// Find all <tr> elements in <tbody>, excluding the first one
func findTableRows(node *html.Node) []*html.Node {
	var tableRows []*html.Node

	if node.Type == html.ElementNode && node.Data == "tbody" {
		child := node.FirstChild
		for child != nil {
			if child.Type == html.ElementNode && child.Data == "tr" {
				tableRows = append(tableRows, child)
			}
			child = child.NextSibling
		}
	}

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		tableRows = append(tableRows, findTableRows(child)...)
	}

	return tableRows
}

// Find all <p> elements in the HTML document
func findTables(node *html.Node) []*html.Node {
	var tables []*html.Node

	if node.Type == html.ElementNode && node.Data == "table" {
		tables = append(tables, node)
	}

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		tables = append(tables, findDayTables(child)...)
	}

	return tables
}

// Find all <p> elements in the HTML document
func findDayTables(node *html.Node) []*html.Node {
	var tables []*html.Node

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		tables = append(tables, findDayTables(child)...)
	}

	if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") {
		tables = append(tables, node)
	}

	return tables
}

// Get the text content of the specified node and its descendants
func getDayLabel(node *html.Node) string {

	child := node.FirstChild
	if child != nil {
		if child.Type == html.ElementNode && child.Data == "span" {
			if child.FirstChild != nil {
				return child.FirstChild.Data
			}
		}
	}
	return ""
}

// Find all <td> elements in the current <tr>
func findTableData(node *html.Node) []*html.Node {
	var tableData []*html.Node

	if node.Type == html.ElementNode && node.Data == "tr" {
		child := node.FirstChild
		for child != nil {
			if child.Type == html.ElementNode && child.Data == "td" {
				tableData = append(tableData, child)
			}
			child = child.NextSibling
		}
	}

	return tableData
}

func getTextContent(node *html.Node) string {
	var textContent string

	if node.Type == html.TextNode {
		textContent = node.Data
	}

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		textContent += getTextContent(child)
	}

	return textContent
}