mirror of
https://gitlab.dit.htwk-leipzig.de/htwk-software/htwkalender.git
synced 2025-08-06 11:49:14 +02:00
Initial Commit
This commit is contained in:
Elmar Kresse
344
fetchService.go
Normal file
344
fetchService.go
Normal file
@@ -0,0 +1,344 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/pocketbase/pocketbase"
|
||||
"github.com/pocketbase/pocketbase/apis"
|
||||
"github.com/pocketbase/pocketbase/models"
|
||||
"golang.org/x/net/html"
|
||||
"htwk-planner/model"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func fetchHTWK(c echo.Context, app *pocketbase.PocketBase) error {
|
||||
|
||||
var seminarGroups []model.SeminarGroup
|
||||
|
||||
seminarGroupsLabel := []string{
|
||||
"22INB-3",
|
||||
"22INB-2",
|
||||
"22INB-1",
|
||||
"21INB-2",
|
||||
"21INB-1",
|
||||
"22INM",
|
||||
//"21INM (Masterarbeit)",
|
||||
"22MIB-BIN",
|
||||
"22MIB-2",
|
||||
"22MIB-1",
|
||||
"21MIB-BIN",
|
||||
"21MIB-1",
|
||||
"21MIB-2",
|
||||
//"20MIB (Praxis/Bachelorarbeit)",
|
||||
"22MIM",
|
||||
//"21MIM (Masterarbeit)"
|
||||
}
|
||||
|
||||
for _, seminarGroupLabel := range seminarGroupsLabel {
|
||||
|
||||
result, err := getPlanHTML("ss", seminarGroupLabel)
|
||||
|
||||
seminarGroup := parseSeminarGroup(result)
|
||||
|
||||
if err != nil {
|
||||
return apis.NewNotFoundError("The Data could not be fetched", err)
|
||||
}
|
||||
|
||||
collection, err := app.Dao().FindCollectionByNameOrId("events")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, event := range seminarGroup.Events {
|
||||
record := models.NewRecord(collection)
|
||||
record.Set("Day", event.Day)
|
||||
record.Set("Week", event.Week)
|
||||
record.Set("Start", event.Start)
|
||||
record.Set("End", event.End)
|
||||
record.Set("Name", event.Name)
|
||||
record.Set("EventType", event.EventType)
|
||||
record.Set("Prof", event.Prof)
|
||||
record.Set("Rooms", event.Rooms)
|
||||
record.Set("Notes", event.Notes)
|
||||
record.Set("BookedAt", event.BookedAt)
|
||||
if err := app.Dao().SaveRecord(record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
seminarGroups = append(seminarGroups, seminarGroup)
|
||||
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, seminarGroups)
|
||||
}
|
||||
|
||||
func parseSeminarGroup(result string) model.SeminarGroup {
|
||||
doc, _ := html.Parse(strings.NewReader(result))
|
||||
table := findFirstTable(doc)
|
||||
events := toEvents(getEventTables(doc), getAllDayLabels(doc))
|
||||
|
||||
var seminarGroup = model.SeminarGroup{
|
||||
University: findFirstSpanWithClass(table, "header-1-0-0").FirstChild.Data,
|
||||
GroupShortcut: findFirstSpanWithClass(table, "header-2-0-1").FirstChild.Data,
|
||||
Events: events,
|
||||
}
|
||||
|
||||
return seminarGroup
|
||||
}
|
||||
|
||||
func toEvents(tables [][]*html.Node, days []string) []model.Events {
|
||||
var events []model.Events
|
||||
|
||||
for table := range tables {
|
||||
for row := range tables[table] {
|
||||
|
||||
tableData := findTableData(tables[table][row])
|
||||
if len(tableData) > 0 {
|
||||
events = append(events, model.Events{
|
||||
Day: days[table],
|
||||
Week: getTextContent(tableData[0]),
|
||||
Start: getTextContent(tableData[1]),
|
||||
End: getTextContent(tableData[2]),
|
||||
Name: getTextContent(tableData[3]),
|
||||
EventType: getTextContent(tableData[4]),
|
||||
Prof: getTextContent(tableData[5]),
|
||||
Rooms: getTextContent(tableData[6]),
|
||||
Notes: getTextContent(tableData[7]),
|
||||
BookedAt: getTextContent(tableData[8]),
|
||||
})
|
||||
// Print the content of each <td> element
|
||||
//for _, td := range tableData {
|
||||
// content := getTextContent(td)
|
||||
// println(content)
|
||||
//}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return events
|
||||
}
|
||||
|
||||
func toUtf8(iso8859_1_buf []byte) string {
|
||||
buf := make([]rune, len(iso8859_1_buf))
|
||||
for i, b := range iso8859_1_buf {
|
||||
buf[i] = rune(b)
|
||||
}
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
func getPlanHTML(semester string, matrikel string) (string, error) {
|
||||
url := "https://stundenplan.htwk-leipzig.de/" + string(semester) + "/Berichte/Text-Listen;Studenten-Sets;name;" + matrikel + "?template=sws_semgrp&weeks=1-65"
|
||||
|
||||
// Send GET request
|
||||
response, err := http.Get(url)
|
||||
if err != nil {
|
||||
fmt.Printf("Error occurred while making the request: %s\n", err.Error())
|
||||
return "", err
|
||||
}
|
||||
defer func(Body io.ReadCloser) {
|
||||
err := Body.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}(response.Body)
|
||||
|
||||
// Read the response body
|
||||
body, err := io.ReadAll(response.Body)
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("Error occurred while reading the response: %s\n", err.Error())
|
||||
return "", err
|
||||
}
|
||||
|
||||
return toUtf8(body), err
|
||||
|
||||
}
|
||||
|
||||
// Find the first <table> element in the HTML document
|
||||
func findFirstTable(node *html.Node) *html.Node {
|
||||
if node.Type == html.ElementNode && node.Data == "table" {
|
||||
return node
|
||||
}
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
found := findFirstTable(child)
|
||||
if found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the first <span> element with the specified class attribute value
|
||||
func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node {
|
||||
if node.Type == html.ElementNode && node.Data == "span" {
|
||||
if hasClassAttribute(node, classValue) {
|
||||
return node
|
||||
}
|
||||
}
|
||||
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
found := findFirstSpanWithClass(child, classValue)
|
||||
if found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if the specified element has the specified class attribute value
|
||||
func hasClassAttribute(node *html.Node, classValue string) bool {
|
||||
for _, attr := range node.Attr {
|
||||
if attr.Key == "class" && strings.Contains(attr.Val, classValue) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Get Tables with days
|
||||
func getEventTables(node *html.Node) [][]*html.Node {
|
||||
var eventTables [][]*html.Node
|
||||
|
||||
tables := findTables(node)
|
||||
|
||||
for events := range tables {
|
||||
rows := findTableRows(tables[events])
|
||||
rows = rows[1:]
|
||||
eventTables = append(eventTables, rows)
|
||||
}
|
||||
|
||||
return eventTables
|
||||
|
||||
}
|
||||
|
||||
// Get Tables with days
|
||||
func getAllDayLabels(node *html.Node) []string {
|
||||
paragraphs := findParagraphs(node)
|
||||
var dayArray []string
|
||||
|
||||
for _, p := range paragraphs {
|
||||
label := getDayLabel(p)
|
||||
if label != "" {
|
||||
dayArray = append(dayArray, label)
|
||||
}
|
||||
}
|
||||
return dayArray
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findParagraphs(node *html.Node) []*html.Node {
|
||||
var paragraphs []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "p" {
|
||||
paragraphs = append(paragraphs, node)
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
paragraphs = append(paragraphs, findParagraphs(child)...)
|
||||
}
|
||||
|
||||
return paragraphs
|
||||
}
|
||||
|
||||
// Find all <tr> elements in <tbody>, excluding the first one
|
||||
func findTableRows(node *html.Node) []*html.Node {
|
||||
var tableRows []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "tbody" {
|
||||
child := node.FirstChild
|
||||
for child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "tr" {
|
||||
tableRows = append(tableRows, child)
|
||||
}
|
||||
child = child.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
tableRows = append(tableRows, findTableRows(child)...)
|
||||
}
|
||||
|
||||
return tableRows
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findTables(node *html.Node) []*html.Node {
|
||||
var tables []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "table" {
|
||||
tables = append(tables, node)
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
tables = append(tables, findDayTables(child)...)
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findDayTables(node *html.Node) []*html.Node {
|
||||
var tables []*html.Node
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
tables = append(tables, findDayTables(child)...)
|
||||
}
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") {
|
||||
tables = append(tables, node)
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
// Get the text content of the specified node and its descendants
|
||||
func getDayLabel(node *html.Node) string {
|
||||
|
||||
child := node.FirstChild
|
||||
if child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "span" {
|
||||
if child.FirstChild != nil {
|
||||
return child.FirstChild.Data
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Find all <td> elements in the current <tr>
|
||||
func findTableData(node *html.Node) []*html.Node {
|
||||
var tableData []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "tr" {
|
||||
child := node.FirstChild
|
||||
for child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "td" {
|
||||
tableData = append(tableData, child)
|
||||
}
|
||||
child = child.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
return tableData
|
||||
}
|
||||
|
||||
func getTextContent(node *html.Node) string {
|
||||
var textContent string
|
||||
|
||||
if node.Type == html.TextNode {
|
||||
textContent = node.Data
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
textContent += getTextContent(child)
|
||||
}
|
||||
|
||||
return textContent
|
||||
}
|
||||
Reference in New Issue
Block a user