diff --git a/addRoute.go b/addRoute.go index f217fa9..f623702 100644 --- a/addRoute.go +++ b/addRoute.go @@ -24,7 +24,7 @@ func addRoutes(app *pocketbase.PocketBase) { Method: http.MethodGet, Path: "/api/fetchPlans", Handler: func(c echo.Context) error { - return fetch.FetchHTWK(c, app) + return fetch.GetSeminarEvents(c, app) }, Middlewares: []echo.MiddlewareFunc{ apis.ActivityLogger(app), diff --git a/openapi.yml b/openapi.yml new file mode 100644 index 0000000..1220dcd --- /dev/null +++ b/openapi.yml @@ -0,0 +1,35 @@ +openapi: 3.0.0 +info: + title: Your API + version: 1.0.0 +paths: + /api/fetchPlans: + get: + summary: Fetch Seminar Plans + responses: + '200': + description: Successful response + /api/fetchGroups: + get: + summary: Fetch Seminar Groups + responses: + '200': + description: Successful response + /api/rooms: + get: + summary: Get Rooms + responses: + '200': + description: Successful response + /api/feedURL: + get: + summary: Get iCal Feed URL + responses: + '200': + description: Successful response + /api/feed: + get: + summary: Get iCal Feed + responses: + '200': + description: Successful response diff --git a/service/fetch/fetchService.go b/service/fetch/fetchSeminarEventService.go similarity index 58% rename from service/fetch/fetchService.go rename to service/fetch/fetchSeminarEventService.go index fa5b625..6df31e7 100644 --- a/service/fetch/fetchService.go +++ b/service/fetch/fetchSeminarEventService.go @@ -18,27 +18,11 @@ import ( "time" ) -func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error { - - var seminarGroups []model.SeminarGroup - - //seminarGroupsLabel := []string{ - // "22EIM-MET", - //} +func GetSeminarEvents(c echo.Context, app *pocketbase.PocketBase) error { seminarGroupsLabel := db.GetAllCourses(app) - for _, seminarGroupLabel := range seminarGroupsLabel { - result, getError := getPlanHTML("ws", seminarGroupLabel) - - if getError == nil { - seminarGroup := parseSeminarGroup(result) - seminarGroups = append(seminarGroups, seminarGroup) - } - } - - //Save SeminarGroups to DB - print("Saving SeminarGroups to DB...\n") + seminarGroups := GetSeminarGroupsEventsFromHTML(seminarGroupsLabel) collection, dbError := findCollection(app, "events") if dbError != nil { @@ -53,6 +37,25 @@ func FetchHTWK(c echo.Context, app *pocketbase.PocketBase) error { return c.JSON(http.StatusOK, seminarGroups) } +func GetSeminarGroupsEventsFromHTML(seminarGroupsLabel []string) []model.SeminarGroup { + var seminarGroups []model.SeminarGroup + for _, seminarGroupLabel := range seminarGroupsLabel { + + result, getError := getPlanHTML("ss", seminarGroupLabel) + if getError == nil { + seminarGroup := parseSeminarGroup(result) + seminarGroups = append(seminarGroups, seminarGroup) + } + + result, getError = getPlanHTML("ws", seminarGroupLabel) + if getError == nil { + seminarGroup := parseSeminarGroup(result) + seminarGroups = append(seminarGroups, seminarGroup) + } + } + return seminarGroups +} + func findCollection(app *pocketbase.PocketBase, collectionName string) (*models.Collection, error) { collection, dbError := app.Dao().FindCollectionByNameOrId(collectionName) return collection, dbError @@ -237,6 +240,7 @@ func toUtf8(iso88591Buf []byte) string { return string(buf) } +// getPlanHTML Get the HTML document from the specified URL func getPlanHTML(semester string, matrikel string) (string, error) { url := "https://stundenplan.htwk-leipzig.de/" + string(semester) + "/Berichte/Text-Listen;Studenten-Sets;name;" + matrikel + "?template=sws_semgrp&weeks=1-65" @@ -264,199 +268,3 @@ func getPlanHTML(semester string, matrikel string) (string, error) { return toUtf8(body), err } - -// Find the first element in the HTML document -func findFirstTable(node *html.Node) *html.Node { - if node.Type == html.ElementNode && node.Data == "table" { - return node - } - // Traverse child nodes recursively - for child := node.FirstChild; child != nil; child = child.NextSibling { - found := findFirstTable(child) - if found != nil { - return found - } - } - return nil -} - -// Find the first element with the specified class attribute value -func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node { - if node.Type == html.ElementNode && node.Data == "span" { - if hasClassAttribute(node, classValue) { - return node - } - } - - // Traverse child nodes recursively - for child := node.FirstChild; child != nil; child = child.NextSibling { - found := findFirstSpanWithClass(child, classValue) - if found != nil { - return found - } - } - - return nil -} - -// Check if the specified element has the specified class attribute value -func hasClassAttribute(node *html.Node, classValue string) bool { - for _, attr := range node.Attr { - if attr.Key == "class" && strings.Contains(attr.Val, classValue) { - return true - } - } - return false -} - -// Get Tables with days -func getEventTables(node *html.Node) [][]*html.Node { - var eventTables [][]*html.Node - - tables := findTables(node) - // get all tables with events - for events := range tables { - rows := findTableRows(tables[events]) - // check that a first row exists - if len(rows) > 0 { - rows = rows[1:] - eventTables = append(eventTables, rows) - } - } - - return eventTables - -} - -// Get Tables with days -func getAllDayLabels(node *html.Node) []string { - paragraphs := findParagraphs(node) - var dayArray []string - - for _, p := range paragraphs { - label := getDayLabel(p) - if label != "" { - dayArray = append(dayArray, label) - } - } - return dayArray -} - -// Find all

elements in the HTML document -func findParagraphs(node *html.Node) []*html.Node { - var paragraphs []*html.Node - - if node.Type == html.ElementNode && node.Data == "p" { - paragraphs = append(paragraphs, node) - } - - for child := node.FirstChild; child != nil; child = child.NextSibling { - paragraphs = append(paragraphs, findParagraphs(child)...) - } - - return paragraphs -} - -// Find all

elements in , excluding the first one -func findTableRows(node *html.Node) []*html.Node { - var tableRows []*html.Node - - if node.Type == html.ElementNode && node.Data == "tbody" { - child := node.FirstChild - for child != nil { - if child.Type == html.ElementNode && child.Data == "tr" { - tableRows = append(tableRows, child) - } - child = child.NextSibling - } - } - - // Traverse child nodes recursively - for child := node.FirstChild; child != nil; child = child.NextSibling { - var tableRowElement = findTableRows(child) - if tableRowElement != nil { - tableRows = append(tableRows, tableRowElement...) - } - } - - if tableRows == nil { - return []*html.Node{} - } else { - return tableRows - } -} - -// Find all

elements in the HTML document -func findTables(node *html.Node) []*html.Node { - var tables []*html.Node - - if node.Type == html.ElementNode && node.Data == "table" { - tables = append(tables, node) - } - - for child := node.FirstChild; child != nil; child = child.NextSibling { - tables = append(tables, findDayTables(child)...) - } - - return tables -} - -// Find all

elements in the HTML document -func findDayTables(node *html.Node) []*html.Node { - var tables []*html.Node - - for child := node.FirstChild; child != nil; child = child.NextSibling { - tables = append(tables, findDayTables(child)...) - } - - if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") { - tables = append(tables, node) - } - - return tables -} - -// Get the text content of the specified node and its descendants -func getDayLabel(node *html.Node) string { - - child := node.FirstChild - if child != nil { - if child.Type == html.ElementNode && child.Data == "span" { - if child.FirstChild != nil { - return child.FirstChild.Data - } - } - } - return "" -} - -// Find all

-func findTableData(node *html.Node) []*html.Node { - var tableData []*html.Node - - if node.Type == html.ElementNode && node.Data == "tr" { - child := node.FirstChild - for child != nil { - if child.Type == html.ElementNode && child.Data == "td" { - tableData = append(tableData, child) - } - child = child.NextSibling - } - } - - return tableData -} - -func getTextContent(node *html.Node) string { - var textContent string - - if node.Type == html.TextNode { - textContent = node.Data - } - - for child := node.FirstChild; child != nil; child = child.NextSibling { - textContent += getTextContent(child) - } - - return textContent -} diff --git a/service/fetch/fetchService_test.go b/service/fetch/fetchSeminarEventService_test.go similarity index 100% rename from service/fetch/fetchService_test.go rename to service/fetch/fetchSeminarEventService_test.go diff --git a/service/fetch/fetchSeminarGroupService.go b/service/fetch/fetchSeminarGroupService.go index ec5c281..41b85c2 100644 --- a/service/fetch/fetchSeminarGroupService.go +++ b/service/fetch/fetchSeminarGroupService.go @@ -85,6 +85,5 @@ func parseSeminarGroups(result string) []model.SeminarGroup { } } } - return seminarGroups } diff --git a/service/fetch/htmlParsingFunctions.go b/service/fetch/htmlParsingFunctions.go new file mode 100644 index 0000000..dd3b8b2 --- /dev/null +++ b/service/fetch/htmlParsingFunctions.go @@ -0,0 +1,202 @@ +package fetch + +import ( + "golang.org/x/net/html" + "strings" +) + +// Find the first
elements in the current
element in the HTML document +func findFirstTable(node *html.Node) *html.Node { + if node.Type == html.ElementNode && node.Data == "table" { + return node + } + // Traverse child nodes recursively + for child := node.FirstChild; child != nil; child = child.NextSibling { + found := findFirstTable(child) + if found != nil { + return found + } + } + return nil +} + +// Find the first element with the specified class attribute value +func findFirstSpanWithClass(node *html.Node, classValue string) *html.Node { + + // Check if the current node is a element with the specified class attribute value + if node.Type == html.ElementNode && node.Data == "span" { + if hasClassAttribute(node, classValue) { + return node + } + } + + // Traverse child nodes recursively + for child := node.FirstChild; child != nil; child = child.NextSibling { + found := findFirstSpanWithClass(child, classValue) + if found != nil { + return found + } + } + return nil +} + +// Check if the specified element has the specified class attribute value +func hasClassAttribute(node *html.Node, classValue string) bool { + for _, attr := range node.Attr { + if attr.Key == "class" && strings.Contains(attr.Val, classValue) { + return true + } + } + return false +} + +// Get Tables with days +func getEventTables(node *html.Node) [][]*html.Node { + var eventTables [][]*html.Node + tables := findTables(node) + // get all tables with events + for events := range tables { + rows := findTableRows(tables[events]) + // check that a first row exists + if len(rows) > 0 { + rows = rows[1:] + eventTables = append(eventTables, rows) + } + } + return eventTables +} + +// Get Tables with days +func getAllDayLabels(node *html.Node) []string { + paragraphs := findParagraphs(node) + var dayArray []string + + for _, p := range paragraphs { + label := getDayLabel(p) + if label != "" { + dayArray = append(dayArray, label) + } + } + return dayArray +} + +// Find all

elements in the HTML document +func findParagraphs(node *html.Node) []*html.Node { + var paragraphs []*html.Node + + if node.Type == html.ElementNode && node.Data == "p" { + paragraphs = append(paragraphs, node) + } + + for child := node.FirstChild; child != nil; child = child.NextSibling { + paragraphs = append(paragraphs, findParagraphs(child)...) + } + + return paragraphs +} + +// Find all

elements in , excluding the first one +func findTableRows(node *html.Node) []*html.Node { + var tableRows []*html.Node + + if node.Type == html.ElementNode && node.Data == "tbody" { + child := node.FirstChild + for child != nil { + if child.Type == html.ElementNode && child.Data == "tr" { + tableRows = append(tableRows, child) + } + child = child.NextSibling + } + } + + // Traverse child nodes recursively + for child := node.FirstChild; child != nil; child = child.NextSibling { + var tableRowElement = findTableRows(child) + if tableRowElement != nil { + tableRows = append(tableRows, tableRowElement...) + } + } + + // check if tableRows is nil + if tableRows == nil { + return []*html.Node{} + } else { + return tableRows + } +} + +// Find all

elements in the HTML document +func findTables(node *html.Node) []*html.Node { + var tables []*html.Node + + if node.Type == html.ElementNode && node.Data == "table" { + tables = append(tables, node) + } + + for child := node.FirstChild; child != nil; child = child.NextSibling { + tables = append(tables, findDayTables(child)...) + } + + return tables +} + +// Find all

elements in the HTML document +func findDayTables(node *html.Node) []*html.Node { + var tables []*html.Node + + for child := node.FirstChild; child != nil; child = child.NextSibling { + tables = append(tables, findDayTables(child)...) + } + + if node.Type == html.ElementNode && node.Data == "table" && hasClassAttribute(node, "spreadsheet") { + tables = append(tables, node) + } + + return tables +} + +// Get the text content of the specified node and its descendants +func getDayLabel(node *html.Node) string { + + child := node.FirstChild + if child != nil { + if child.Type == html.ElementNode && child.Data == "span" { + if child.FirstChild != nil { + return child.FirstChild.Data + } + } + } + return "" +} + +// Find all

+func findTableData(node *html.Node) []*html.Node { + var tableData []*html.Node + + if node.Type == html.ElementNode && node.Data == "tr" { + child := node.FirstChild + for child != nil { + if child.Type == html.ElementNode && child.Data == "td" { + tableData = append(tableData, child) + } + child = child.NextSibling + } + } + + return tableData +} + +// Get the text content of the specified node and its descendants +func getTextContent(node *html.Node) string { + var textContent string + + if node.Type == html.TextNode { + textContent = node.Data + } + + for child := node.FirstChild; child != nil; child = child.NextSibling { + textContent += getTextContent(child) + } + + return textContent +}
elements in the current