mirror of
https://gitlab.dit.htwk-leipzig.de/htwk-software/htwkalender.git
synced 2025-07-16 09:38:49 +02:00
refactoring and documentation
This commit is contained in:
Elmar Kresse
@ -24,7 +24,7 @@ func addRoutes(app *pocketbase.PocketBase) {
|
||||
Method: http.MethodGet,
|
||||
Path: "/api/fetchPlans",
|
||||
Handler: func(c echo.Context) error {
|
||||
return fetch.FetchHTWK(c, app)
|
||||
return fetch.GetSeminarEvents(c, app)
|
||||
},
|
||||
Middlewares: []echo.MiddlewareFunc{
|
||||
apis.ActivityLogger(app),
|
||||
|
||||
35
openapi.yml
Normal file
35
openapi.yml
Normal file
@ -0,0 +1,35 @@
|
||||
openapi: 3.0.0
|
||||
info:
|
||||
title: Your API
|
||||
version: 1.0.0
|
||||
paths:
|
||||
/api/fetchPlans:
|
||||
get:
|
||||
summary: Fetch Seminar Plans
|
||||
responses:
|
||||
'200':
|
||||
description: Successful response
|
||||
/api/fetchGroups:
|
||||
get:
|
||||
summary: Fetch Seminar Groups
|
||||
responses:
|
||||
'200':
|
||||
description: Successful response
|
||||
/api/rooms:
|
||||
get:
|
||||
summary: Get Rooms
|
||||
responses:
|
||||
'200':
|
||||
description: Successful response
|
||||
/api/feedURL:
|
||||
get:
|
||||
summary: Get iCal Feed URL
|
||||
responses:
|
||||
'200':
|
||||
description: Successful response
|
||||
/api/feed:
|
||||
get:
|
||||
summary: Get iCal Feed
|
||||
responses:
|
||||
'200':
|
||||
description: Successful response
|
||||
@ -18,27 +18,11 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error {
|
||||
|
||||
var seminarGroups []model.SeminarGroup
|
||||
|
||||
//seminarGroupsLabel := []string{
|
||||
// "22EIM-MET",
|
||||
//}
|
||||
func GetSeminarEvents(c echo.Context, app *pocketbase.PocketBase) error {
|
||||
|
||||
seminarGroupsLabel := db.GetAllCourses(app)
|
||||
for _, seminarGroupLabel := range seminarGroupsLabel {
|
||||
|
||||
result, getError := getPlanHTML("ws", seminarGroupLabel)
|
||||
|
||||
if getError == nil {
|
||||
seminarGroup := parseSeminarGroup(result)
|
||||
seminarGroups = append(seminarGroups, seminarGroup)
|
||||
}
|
||||
}
|
||||
|
||||
//Save SeminarGroups to DB
|
||||
print("Saving SeminarGroups to DB...\n")
|
||||
seminarGroups := GetSeminarGroupsEventsFromHTML(seminarGroupsLabel)
|
||||
|
||||
collection, dbError := findCollection(app, "events")
|
||||
if dbError != nil {
|
||||
@ -53,6 +37,25 @@ func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error {
|
||||
return c.JSON(http.StatusOK, seminarGroups)
|
||||
}
|
||||
|
||||
func GetSeminarGroupsEventsFromHTML(seminarGroupsLabel []string) []model.SeminarGroup {
|
||||
var seminarGroups []model.SeminarGroup
|
||||
for _, seminarGroupLabel := range seminarGroupsLabel {
|
||||
|
||||
result, getError := getPlanHTML("ss", seminarGroupLabel)
|
||||
if getError == nil {
|
||||
seminarGroup := parseSeminarGroup(result)
|
||||
seminarGroups = append(seminarGroups, seminarGroup)
|
||||
}
|
||||
|
||||
result, getError = getPlanHTML("ws", seminarGroupLabel)
|
||||
if getError == nil {
|
||||
seminarGroup := parseSeminarGroup(result)
|
||||
seminarGroups = append(seminarGroups, seminarGroup)
|
||||
}
|
||||
}
|
||||
return seminarGroups
|
||||
}
|
||||
|
||||
func findCollection(app *pocketbase.PocketBase, collectionName string) (*models.Collection, error) {
|
||||
collection, dbError := app.Dao().FindCollectionByNameOrId(collectionName)
|
||||
return collection, dbError
|
||||
@ -237,6 +240,7 @@ func toUtf8(iso88591Buf []byte) string {
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
// getPlanHTML Get the HTML document from the specified URL
|
||||
func getPlanHTML(semester string, matrikel string) (string, error) {
|
||||
url := "https://stundenplan.htwk-leipzig.de/" + string(semester) + "/Berichte/Text-Listen;Studenten-Sets;name;" + matrikel + "?template=sws_semgrp&weeks=1-65"
|
||||
|
||||
@ -264,199 +268,3 @@ func getPlanHTML(semester string, matrikel string) (string, error) {
|
||||
return toUtf8(body), err
|
||||
|
||||
}
|
||||
|
||||
// Find the first <table> element in the HTML document
|
||||
func findFirstTable(node *html.Node) *html.Node {
|
||||
if node.Type == html.ElementNode && node.Data == "table" {
|
||||
return node
|
||||
}
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
found := findFirstTable(child)
|
||||
if found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the first <span> element with the specified class attribute value
|
||||
func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node {
|
||||
if node.Type == html.ElementNode && node.Data == "span" {
|
||||
if hasClassAttribute(node, classValue) {
|
||||
return node
|
||||
}
|
||||
}
|
||||
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
found := findFirstSpanWithClass(child, classValue)
|
||||
if found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if the specified element has the specified class attribute value
|
||||
func hasClassAttribute(node *html.Node, classValue string) bool {
|
||||
for _, attr := range node.Attr {
|
||||
if attr.Key == "class" && strings.Contains(attr.Val, classValue) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Get Tables with days
|
||||
func getEventTables(node *html.Node) [][]*html.Node {
|
||||
var eventTables [][]*html.Node
|
||||
|
||||
tables := findTables(node)
|
||||
// get all tables with events
|
||||
for events := range tables {
|
||||
rows := findTableRows(tables[events])
|
||||
// check that a first row exists
|
||||
if len(rows) > 0 {
|
||||
rows = rows[1:]
|
||||
eventTables = append(eventTables, rows)
|
||||
}
|
||||
}
|
||||
|
||||
return eventTables
|
||||
|
||||
}
|
||||
|
||||
// Get Tables with days
|
||||
func getAllDayLabels(node *html.Node) []string {
|
||||
paragraphs := findParagraphs(node)
|
||||
var dayArray []string
|
||||
|
||||
for _, p := range paragraphs {
|
||||
label := getDayLabel(p)
|
||||
if label != "" {
|
||||
dayArray = append(dayArray, label)
|
||||
}
|
||||
}
|
||||
return dayArray
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findParagraphs(node *html.Node) []*html.Node {
|
||||
var paragraphs []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "p" {
|
||||
paragraphs = append(paragraphs, node)
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
paragraphs = append(paragraphs, findParagraphs(child)...)
|
||||
}
|
||||
|
||||
return paragraphs
|
||||
}
|
||||
|
||||
// Find all <tr> elements in <tbody>, excluding the first one
|
||||
func findTableRows(node *html.Node) []*html.Node {
|
||||
var tableRows []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "tbody" {
|
||||
child := node.FirstChild
|
||||
for child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "tr" {
|
||||
tableRows = append(tableRows, child)
|
||||
}
|
||||
child = child.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
var tableRowElement = findTableRows(child)
|
||||
if tableRowElement != nil {
|
||||
tableRows = append(tableRows, tableRowElement...)
|
||||
}
|
||||
}
|
||||
|
||||
if tableRows == nil {
|
||||
return []*html.Node{}
|
||||
} else {
|
||||
return tableRows
|
||||
}
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findTables(node *html.Node) []*html.Node {
|
||||
var tables []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "table" {
|
||||
tables = append(tables, node)
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
tables = append(tables, findDayTables(child)...)
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findDayTables(node *html.Node) []*html.Node {
|
||||
var tables []*html.Node
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
tables = append(tables, findDayTables(child)...)
|
||||
}
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") {
|
||||
tables = append(tables, node)
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
// Get the text content of the specified node and its descendants
|
||||
func getDayLabel(node *html.Node) string {
|
||||
|
||||
child := node.FirstChild
|
||||
if child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "span" {
|
||||
if child.FirstChild != nil {
|
||||
return child.FirstChild.Data
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Find all <td> elements in the current <tr>
|
||||
func findTableData(node *html.Node) []*html.Node {
|
||||
var tableData []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "tr" {
|
||||
child := node.FirstChild
|
||||
for child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "td" {
|
||||
tableData = append(tableData, child)
|
||||
}
|
||||
child = child.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
return tableData
|
||||
}
|
||||
|
||||
func getTextContent(node *html.Node) string {
|
||||
var textContent string
|
||||
|
||||
if node.Type == html.TextNode {
|
||||
textContent = node.Data
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
textContent += getTextContent(child)
|
||||
}
|
||||
|
||||
return textContent
|
||||
}
|
||||
@ -85,6 +85,5 @@ func parseSeminarGroups(result string) []model.SeminarGroup {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return seminarGroups
|
||||
}
|
||||
|
||||
202
service/fetch/htmlParsingFunctions.go
Normal file
202
service/fetch/htmlParsingFunctions.go
Normal file
@ -0,0 +1,202 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"golang.org/x/net/html"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Find the first <table> element in the HTML document
|
||||
func findFirstTable(node *html.Node) *html.Node {
|
||||
if node.Type == html.ElementNode && node.Data == "table" {
|
||||
return node
|
||||
}
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
found := findFirstTable(child)
|
||||
if found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the first <span> element with the specified class attribute value
|
||||
func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node {
|
||||
|
||||
// Check if the current node is a <span> element with the specified class attribute value
|
||||
if node.Type == html.ElementNode && node.Data == "span" {
|
||||
if hasClassAttribute(node, classValue) {
|
||||
return node
|
||||
}
|
||||
}
|
||||
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
found := findFirstSpanWithClass(child, classValue)
|
||||
if found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if the specified element has the specified class attribute value
|
||||
func hasClassAttribute(node *html.Node, classValue string) bool {
|
||||
for _, attr := range node.Attr {
|
||||
if attr.Key == "class" && strings.Contains(attr.Val, classValue) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Get Tables with days
|
||||
func getEventTables(node *html.Node) [][]*html.Node {
|
||||
var eventTables [][]*html.Node
|
||||
tables := findTables(node)
|
||||
// get all tables with events
|
||||
for events := range tables {
|
||||
rows := findTableRows(tables[events])
|
||||
// check that a first row exists
|
||||
if len(rows) > 0 {
|
||||
rows = rows[1:]
|
||||
eventTables = append(eventTables, rows)
|
||||
}
|
||||
}
|
||||
return eventTables
|
||||
}
|
||||
|
||||
// Get Tables with days
|
||||
func getAllDayLabels(node *html.Node) []string {
|
||||
paragraphs := findParagraphs(node)
|
||||
var dayArray []string
|
||||
|
||||
for _, p := range paragraphs {
|
||||
label := getDayLabel(p)
|
||||
if label != "" {
|
||||
dayArray = append(dayArray, label)
|
||||
}
|
||||
}
|
||||
return dayArray
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findParagraphs(node *html.Node) []*html.Node {
|
||||
var paragraphs []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "p" {
|
||||
paragraphs = append(paragraphs, node)
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
paragraphs = append(paragraphs, findParagraphs(child)...)
|
||||
}
|
||||
|
||||
return paragraphs
|
||||
}
|
||||
|
||||
// Find all <tr> elements in <tbody>, excluding the first one
|
||||
func findTableRows(node *html.Node) []*html.Node {
|
||||
var tableRows []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "tbody" {
|
||||
child := node.FirstChild
|
||||
for child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "tr" {
|
||||
tableRows = append(tableRows, child)
|
||||
}
|
||||
child = child.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
// Traverse child nodes recursively
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
var tableRowElement = findTableRows(child)
|
||||
if tableRowElement != nil {
|
||||
tableRows = append(tableRows, tableRowElement...)
|
||||
}
|
||||
}
|
||||
|
||||
// check if tableRows is nil
|
||||
if tableRows == nil {
|
||||
return []*html.Node{}
|
||||
} else {
|
||||
return tableRows
|
||||
}
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findTables(node *html.Node) []*html.Node {
|
||||
var tables []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "table" {
|
||||
tables = append(tables, node)
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
tables = append(tables, findDayTables(child)...)
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
// Find all <p> elements in the HTML document
|
||||
func findDayTables(node *html.Node) []*html.Node {
|
||||
var tables []*html.Node
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
tables = append(tables, findDayTables(child)...)
|
||||
}
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") {
|
||||
tables = append(tables, node)
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
// Get the text content of the specified node and its descendants
|
||||
func getDayLabel(node *html.Node) string {
|
||||
|
||||
child := node.FirstChild
|
||||
if child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "span" {
|
||||
if child.FirstChild != nil {
|
||||
return child.FirstChild.Data
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Find all <td> elements in the current <tr>
|
||||
func findTableData(node *html.Node) []*html.Node {
|
||||
var tableData []*html.Node
|
||||
|
||||
if node.Type == html.ElementNode && node.Data == "tr" {
|
||||
child := node.FirstChild
|
||||
for child != nil {
|
||||
if child.Type == html.ElementNode && child.Data == "td" {
|
||||
tableData = append(tableData, child)
|
||||
}
|
||||
child = child.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
return tableData
|
||||
}
|
||||
|
||||
// Get the text content of the specified node and its descendants
|
||||
func getTextContent(node *html.Node) string {
|
||||
var textContent string
|
||||
|
||||
if node.Type == html.TextNode {
|
||||
textContent = node.Data
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
textContent += getTextContent(child)
|
||||
}
|
||||
|
||||
return textContent
|
||||
}
|
||||
Reference in New Issue
Block a user