From 57e3e41a9ac5c492a6216f8d2fadf578700fabb7 Mon Sep 17 00:00:00 2001 From: joossm Date: Fri, 8 Dec 2023 17:38:37 +0100 Subject: [PATCH 01/18] Since im not that deep into the project i seperated it from the rest, its in "backend/sport/main.go". I added the task which are still needed to do. Added Package "github.com/PuerkitoBio/goquery" --- backend/go.mod | 2 + backend/go.sum | 11 +++ backend/sport/main.go | 185 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 backend/sport/main.go diff --git a/backend/go.mod b/backend/go.mod index 3a2ca8b..a9ed30a 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,6 +3,7 @@ module htwkalender go 1.21 require ( + github.com/PuerkitoBio/goquery v1.8.1 github.com/google/uuid v1.3.1 github.com/jordic/goics v0.0.0-20210404174824-5a0337b716a0 github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 @@ -13,6 +14,7 @@ require ( require ( github.com/AlecAivazis/survey/v2 v2.3.7 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.46.3 // indirect github.com/aws/aws-sdk-go-v2 v1.21.2 // indirect diff --git a/backend/go.sum b/backend/go.sum index cdbecb5..70f7586 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -14,6 +14,10 @@ github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1r github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -239,8 +243,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -258,25 +264,30 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= diff --git a/backend/sport/main.go b/backend/sport/main.go new file mode 100644 index 0000000..b62e2c2 --- /dev/null +++ b/backend/sport/main.go @@ -0,0 +1,185 @@ +package main + +import ( + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" +) + +// just to test the code +// @TODO: remove this +// @TODO: add tests +// @TODO: add it to the service +// @TODO: make it like a cron job to fetch the sport courses once a week +func main() { + events := fetchAllHtwkSportCourses() + for _, event := range events { + print(event.Title) + } +} + +// fetchAllHtwkSportCourses fetches all sport courses from the htwk sport website. +// It iterates over all ids from 0 to 9999 and tries to fetch the sport course. +// If the sport course does not exist, it will continue with the next id. +// If the sport course exists, it will be added to the events slice. +// Since the ids are not consecutive, it will take a while to fetch all sport courses. +// @TODO: find the highest id and iterate over all ids from 0 to highest id +func fetchAllHtwkSportCourses() []Event { + var events []Event + for i := 0; i <= 9999; i++ { + newEvent, err := fetchHtwkSportCourse("https://sport.htwk-leipzig.de/sportangebote/detail/sport/", i) + if err != nil { + continue + } + events = append(events, newEvent...) + + } + return events +} + +// fetchHtwkSportCourse fetches the sport course from the given url and id. +// If the sport course does not exist, it will return an error. +// If the sport course exists, it will return the sport course. +// goquery is used to parse the html. The html structure is not very consistent, so it is hard to parse. +// May be improved in the future. +func fetchHtwkSportCourse(url string, id int) ([]Event, error) { + var events []Event + + resp, err := http.Get(url + strconv.Itoa(id)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, err + } + + if doc.Find("h1").Text() == "Aktuelle Sportangebote" { + return nil, errors.New("not a sport course page") + } + + doc.Find(".eventHead").Each(func(i int, s *goquery.Selection) { + var event Event + var details EventDetails + + fullTitle := strings.TrimSpace(s.Find("h3").Text()) + titleParts := strings.Split(fullTitle, "-") + if len(titleParts) > 0 { + event.Title = strings.TrimSpace(titleParts[0]) + } + s.NextFiltered("table.eventDetails").Find("tr").Each(func(i int, s *goquery.Selection) { + key := strings.TrimSpace(s.Find("td").First().Text()) + value := strings.TrimSpace(s.Find("td").Last().Text()) + + switch key { + case "Zeitraum": + dates := strings.Split(value, "-") + if len(dates) == 2 { + startDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[0])) + endDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[1])) + details.DateRange = DateRange{Start: startDate, End: endDate} + } + case "Zyklus": + details.Cycle = value + case "Geschlecht": + details.Gender = value + case "Leiter": + leaderName := strings.TrimSpace(s.Find("td a").Text()) + leadersSlice := strings.Split(leaderName, "\n") + for i, leader := range leadersSlice { + leadersSlice[i] = strings.TrimSpace(leader) + } + formattedLeaders := strings.Join(leadersSlice, ", ") + leaderLink, _ := s.Find("td a").Attr("href") + details.CourseLead = CourseLead{Name: formattedLeaders, Link: leaderLink} + case "Ort": + locationDetails := strings.Split(value, "(") + if len(locationDetails) == 2 { + details.Location = Location{ + Name: strings.TrimSpace(locationDetails[0]), + Address: strings.TrimRight(strings.TrimSpace(locationDetails[1]), ")"), + } + } + case "Teilnehmer": + parts := strings.Split(value, "/") + if len(parts) >= 3 { + bookings, _ := strconv.Atoi(strings.TrimSpace(parts[0])) + totalPlaces, _ := strconv.Atoi(strings.TrimSpace(parts[1])) + waitList, _ := strconv.Atoi(strings.TrimSpace(parts[2])) + details.Participants = Participants{Bookings: bookings, TotalPlaces: totalPlaces, WaitList: waitList} + } + case "Kosten": + details.Cost = value // makes no sense since you need to be logged in to see the price + case "Hinweis": + var allNotes []string + + s.Find("td").Last().Contents().Each(func(i int, s *goquery.Selection) { + if s.Is("h4.eventAdvice") || goquery.NodeName(s) == "#text" { + note := strings.TrimSpace(s.Text()) + if note != "" { + allNotes = append(allNotes, note) + } + } + }) + + event.AdditionalNote = strings.Join(allNotes, " ") + } + }) + + event.Details = details + events = append(events, event) + }) + + return events, nil +} + +// MODELS + +// Event represents the overall event details. +type Event struct { + Title string + Details EventDetails + AdditionalNote string +} + +// EventDetails represents detailed information about the event. +type EventDetails struct { + DateRange DateRange + Cycle string + Gender string + CourseLead CourseLead + Location Location + Participants Participants + Cost string +} + +// DateRange represents a start and end date. +type DateRange struct { + Start time.Time + End time.Time +} + +// CourseLead represents a person with a name and a contact link. +type CourseLead struct { + Name string + Link string +} + +// Location represents the location of the event. +type Location struct { + Name string + Address string +} + +// Participants represents the participants' details. +type Participants struct { + Bookings int + TotalPlaces int + WaitList int +} From 8b9d4e175083eab0bbf46dd5513acf47500cafbe Mon Sep 17 00:00:00 2001 From: Niclas Jost <35239311+niclas-j@users.noreply.github.com> Date: Sun, 10 Dec 2023 12:21:51 +0100 Subject: [PATCH 02/18] 108 - lazy load routes --- frontend/src/router/index.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts index 191a3eb..3f1ff6f 100644 --- a/frontend/src/router/index.ts +++ b/frontend/src/router/index.ts @@ -1,15 +1,17 @@ import { createRouter, createWebHistory } from "vue-router"; -import Faq from "../components/FaqPage.vue"; -import AdditionalModules from "../view/AdditionalModules.vue"; -import CalendarLink from "../components/CalendarLink.vue"; -import Imprint from "../view/ImprintPage.vue"; -import PrivacyPolicy from "../view/PrivacyPolicy.vue"; -import RenameModules from "../components/RenameModules.vue"; -import RoomFinder from "../view/RoomFinder.vue"; -import EditCalendarView from "../view/EditCalendarView.vue"; -import EditAdditionalModules from "../components/editCalendar/EditAdditionalModules.vue"; -import EditModules from "../components/editCalendar/EditModules.vue"; -import CourseSelection from "../view/CourseSelection.vue"; + +const Faq = () => import("../components/FaqPage.vue"); +const AdditionalModules = () => import("../view/AdditionalModules.vue"); +const CalendarLink = () => import("../components/CalendarLink.vue"); +const Imprint = () => import("../view/ImprintPage.vue"); +const PrivacyPolicy = () => import("../view/PrivacyPolicy.vue"); +const RenameModules = () => import("../components/RenameModules.vue"); +const RoomFinder = () => import("../view/RoomFinder.vue"); +const EditCalendarView = () => import("../view/EditCalendarView.vue"); +const EditAdditionalModules = () => import("../components/editCalendar/EditAdditionalModules.vue"); +const EditModules = () => import("../components/editCalendar/EditModules.vue"); +const CourseSelection = () => import("../view/CourseSelection.vue"); + import i18n from "../i18n"; const router = createRouter({ From 184dc70be483e4fe2cbf11acd1c8523467eb8ec9 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:48:44 +0100 Subject: [PATCH 03/18] fix:#82 added multithreaded fetching --- backend/sport/main.go | 65 +++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/backend/sport/main.go b/backend/sport/main.go index b62e2c2..31164b2 100644 --- a/backend/sport/main.go +++ b/backend/sport/main.go @@ -5,6 +5,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/PuerkitoBio/goquery" @@ -29,27 +30,43 @@ func main() { // Since the ids are not consecutive, it will take a while to fetch all sport courses. // @TODO: find the highest id and iterate over all ids from 0 to highest id func fetchAllHtwkSportCourses() []Event { - var events []Event - for i := 0; i <= 9999; i++ { - newEvent, err := fetchHtwkSportCourse("https://sport.htwk-leipzig.de/sportangebote/detail/sport/", i) - if err != nil { - continue - } - events = append(events, newEvent...) + //multithreaded webpage requests to speed up the process + + var maxPageID = 9999 + var maxThreads = 300 + var htmlPageArray = make([]*goquery.Document, maxPageID) + var url = "https://sport.htwk-leipzig.de/sportangebote/detail/sport/" + + var wg sync.WaitGroup + wg.Add(maxThreads) + + for i := 0; i < maxThreads; i++ { + go func(i int) { + defer wg.Done() + for j := i; j < maxPageID; j += maxThreads { + doc, err := htmlRequest(url + strconv.Itoa(j)) + if err == nil { + htmlPageArray[j] = doc + } + } + }(i) } + + wg.Wait() + + println("finished fetching all pages") + + //print count of all pages + + var events []Event return events } -// fetchHtwkSportCourse fetches the sport course from the given url and id. -// If the sport course does not exist, it will return an error. -// If the sport course exists, it will return the sport course. -// goquery is used to parse the html. The html structure is not very consistent, so it is hard to parse. -// May be improved in the future. -func fetchHtwkSportCourse(url string, id int) ([]Event, error) { - var events []Event +func htmlRequest(url string) (*goquery.Document, error) { + println("fetching " + url) - resp, err := http.Get(url + strconv.Itoa(id)) + resp, err := http.Get(url) if err != nil { return nil, err } @@ -60,6 +77,18 @@ func fetchHtwkSportCourse(url string, id int) ([]Event, error) { return nil, err } + println("finished fetching " + url) + return doc, nil +} + +// fetchHtwkSportCourse fetches the sport course from the given url and id. +// If the sport course does not exist, it will return an error. +// If the sport course exists, it will return the sport course. +// goquery is used to parse the html. The html structure is not very consistent, so it is hard to parse. +// May be improved in the future. +func fetchHtwkSportCourse(doc *goquery.Document) ([]Event, error) { + var events []Event + if doc.Find("h1").Text() == "Aktuelle Sportangebote" { return nil, errors.New("not a sport course page") } @@ -73,6 +102,11 @@ func fetchHtwkSportCourse(url string, id int) ([]Event, error) { if len(titleParts) > 0 { event.Title = strings.TrimSpace(titleParts[0]) } + + if len(titleParts) > 2 { + details.Type = strings.TrimSpace(titleParts[len(titleParts)-1]) + } + s.NextFiltered("table.eventDetails").Find("tr").Each(func(i int, s *goquery.Selection) { key := strings.TrimSpace(s.Find("td").First().Text()) value := strings.TrimSpace(s.Find("td").Last().Text()) @@ -157,6 +191,7 @@ type EventDetails struct { Location Location Participants Participants Cost string + Type string } // DateRange represents a start and end date. From 31c27635d395f928659c839481e5b0afd8f1e97e Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:16:10 +0100 Subject: [PATCH 04/18] fix:#82 fixed fetching with main page fetch for sport course link list --- backend/sport/main.go | 119 ++++++++++++----------------- backend/sport/sportFetcherModel.go | 49 ++++++++++++ 2 files changed, 98 insertions(+), 70 deletions(-) create mode 100644 backend/sport/sportFetcherModel.go diff --git a/backend/sport/main.go b/backend/sport/main.go index 31164b2..99657ce 100644 --- a/backend/sport/main.go +++ b/backend/sport/main.go @@ -11,55 +11,80 @@ import ( "github.com/PuerkitoBio/goquery" ) -// just to test the code -// @TODO: remove this +// @TODO: reformat the extracted data to the event model that there are events with real start and end dates // @TODO: add tests // @TODO: add it to the service // @TODO: make it like a cron job to fetch the sport courses once a week func main() { - events := fetchAllHtwkSportCourses() + + var sportCourseLinks = fetchAllAvailableSportCourses() + events := fetchHTWKSportCourses(sportCourseLinks) + for _, event := range events { - print(event.Title) + println(event.Title) } } -// fetchAllHtwkSportCourses fetches all sport courses from the htwk sport website. -// It iterates over all ids from 0 to 9999 and tries to fetch the sport course. -// If the sport course does not exist, it will continue with the next id. -// If the sport course exists, it will be added to the events slice. -// Since the ids are not consecutive, it will take a while to fetch all sport courses. -// @TODO: find the highest id and iterate over all ids from 0 to highest id -func fetchAllHtwkSportCourses() []Event { +// fetch the main page where all sport courses are listed and extract all links to the sport courses +func fetchAllAvailableSportCourses() []string { + var url = "https://sport.htwk-leipzig.de/sportangebote" + + var doc, err = htmlRequest(url) + + if err != nil { + return nil + } + + // link list of all sport courses + var links []string + + // find all links to sport courses with regex https://sport.htwk-leipzig.de/sportangebote/detail/sport/ + [0-9]{1,4} + doc.Find("a[href]").Each(func(i int, s *goquery.Selection) { + link, _ := s.Attr("href") + if strings.HasPrefix(link, "/sportangebote/detail/sport/") { + links = append(links, link) + } + }) + + return links +} + +// fetchAllHTWKSportCourses fetches all sport courses from the given links. +// to speed up the process, it uses multithreading. + +func fetchHTWKSportCourses(links []string) []Event { //multithreaded webpage requests to speed up the process - var maxPageID = 9999 - var maxThreads = 300 - var htmlPageArray = make([]*goquery.Document, maxPageID) - var url = "https://sport.htwk-leipzig.de/sportangebote/detail/sport/" + var maxThreads = 10 + var htmlPageArray = make([]*goquery.Document, len(links)) + var hostUrl = "https://sport.htwk-leipzig.de" var wg sync.WaitGroup wg.Add(maxThreads) - for i := 0; i < maxThreads; i++ { go func(i int) { - defer wg.Done() - for j := i; j < maxPageID; j += maxThreads { - doc, err := htmlRequest(url + strconv.Itoa(j)) + for j := i; j < len(links); j += maxThreads { + doc, err := htmlRequest(hostUrl + links[j]) if err == nil { htmlPageArray[j] = doc } } + wg.Done() }(i) } - wg.Wait() - println("finished fetching all pages") - - //print count of all pages - var events []Event + + for _, doc := range htmlPageArray { + if doc != nil { + event, err := fetchHtwkSportCourse(doc) + if err == nil { + events = append(events, event...) + } + } + } return events } @@ -172,49 +197,3 @@ func fetchHtwkSportCourse(doc *goquery.Document) ([]Event, error) { return events, nil } - -// MODELS - -// Event represents the overall event details. -type Event struct { - Title string - Details EventDetails - AdditionalNote string -} - -// EventDetails represents detailed information about the event. -type EventDetails struct { - DateRange DateRange - Cycle string - Gender string - CourseLead CourseLead - Location Location - Participants Participants - Cost string - Type string -} - -// DateRange represents a start and end date. -type DateRange struct { - Start time.Time - End time.Time -} - -// CourseLead represents a person with a name and a contact link. -type CourseLead struct { - Name string - Link string -} - -// Location represents the location of the event. -type Location struct { - Name string - Address string -} - -// Participants represents the participants' details. -type Participants struct { - Bookings int - TotalPlaces int - WaitList int -} diff --git a/backend/sport/sportFetcherModel.go b/backend/sport/sportFetcherModel.go new file mode 100644 index 0000000..785e582 --- /dev/null +++ b/backend/sport/sportFetcherModel.go @@ -0,0 +1,49 @@ +package main + +import "time" + +// MODELS + +// Event represents the overall event details. +type Event struct { + Title string + Details EventDetails + AdditionalNote string +} + +// EventDetails represents detailed information about the event. +type EventDetails struct { + DateRange DateRange + Cycle string + Gender string + CourseLead CourseLead + Location Location + Participants Participants + Cost string + Type string +} + +// DateRange represents a start and end date. +type DateRange struct { + Start time.Time + End time.Time +} + +// CourseLead represents a person with a name and a contact link. +type CourseLead struct { + Name string + Link string +} + +// Location represents the location of the event. +type Location struct { + Name string + Address string +} + +// Participants represents the participants' details. +type Participants struct { + Bookings int + TotalPlaces int + WaitList int +} From a714e92cd9c96988a6993000729565bddab8ef75 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:12:46 +0100 Subject: [PATCH 05/18] fix:#82 added event parser --- backend/sport/main.go | 168 +++++++++++++++++++++++++++-- backend/sport/sportFetcherModel.go | 4 +- 2 files changed, 163 insertions(+), 9 deletions(-) diff --git a/backend/sport/main.go b/backend/sport/main.go index 99657ce..64ffe81 100644 --- a/backend/sport/main.go +++ b/backend/sport/main.go @@ -2,7 +2,11 @@ package main import ( "errors" + "github.com/google/uuid" + "github.com/pocketbase/pocketbase/tools/types" + "htwkalender/model" "net/http" + "regexp" "strconv" "strings" "sync" @@ -18,10 +22,160 @@ import ( func main() { var sportCourseLinks = fetchAllAvailableSportCourses() - events := fetchHTWKSportCourses(sportCourseLinks) + sportEntries := fetchHTWKSportCourses(sportCourseLinks) + + for _, event := range sportEntries { + println(event.Title) + } + + events := formatEntriesToEvents(sportEntries) for _, event := range events { - println(event.Title) + println(event.Name) + } +} + +func formatEntriesToEvents(entries []SportEntry) []model.Event { + + var events []model.Event + + for i, entry := range entries { + + eventStarts, eventEnds := calculateEventStarts(entry.Details.DateRange.Start, entry.Details.DateRange.End, entry.Details.Cycle) + + for j := range eventStarts { + + start, _ := types.ParseDateTime(eventStarts[j].In(time.UTC)) + end, _ := types.ParseDateTime(eventEnds[j].In(time.UTC)) + + var event = model.Event{ + UUID: uuid.NewSHA1(uuid.NameSpaceDNS, []byte(entry.Title+strconv.FormatInt(int64(i), 10))).String(), + Day: entry.Details.DateRange.Start.Weekday().String(), + Week: strconv.Itoa(23), + Start: start, + End: end, + Name: entry.Title, + EventType: entry.Details.Type, + Prof: entry.Details.CourseLead.Name, + Rooms: entry.Details.Location.Name, + Notes: entry.AdditionalNote, + BookedAt: "", + Course: "Sport", + Semester: checkSemester(entry.Details.DateRange.Start), + } + events = append(events, event) + } + + } + return events +} + +func calculateEventStarts(start time.Time, end time.Time, cycle string) ([]time.Time, []time.Time) { + + // start is the begin of the cycle e.g. 01.04.2020 + // end is the end of the cycle e.g. 30.09.2020 + // cycle is the day and timespan (e.g. "Mo 18:00-20:00") + + // check if start is before end + if start.After(end) { + return nil, nil + } + + // check if cycle is valid + if !checkCycle(cycle) { + return nil, nil + } + + var weekDay = cycle[0:2] + // match weekday to time.Weekday (e.g. "Mo" -> time.Monday) + var weekDayInt int + + switch weekDay { + case "Mo": + weekDayInt = 1 + case "Di": + weekDayInt = 2 + case "Mi": + weekDayInt = 3 + case "Do": + weekDayInt = 4 + case "Fr": + weekDayInt = 5 + case "Sa": + weekDayInt = 6 + case "So": + weekDayInt = 0 + } + + // get every date matching the weekday in the cycle between start and end + var eventDates []time.Time + for d := start; d.Before(end); d = d.AddDate(0, 0, 1) { + if d.Weekday() == time.Weekday(weekDayInt) { + eventDates = append(eventDates, d) + } + } + + // add hours and minutes to the dates in eventDates + // array of tuple of start and end times + var eventStartsWithTime []time.Time + var eventEndWithTime []time.Time + + for _, eventStart := range eventDates { + timeRegExp, _ := regexp.Compile("[0-9]{2}:[0-9]{2}") + times := timeRegExp.FindAllString(cycle, 2) + startHour, _ := strconv.Atoi(times[0][0:2]) + startMinute, _ := strconv.Atoi(times[0][3:5]) + + endHour, _ := strconv.Atoi(times[1][0:2]) + endMinute, _ := strconv.Atoi(times[1][3:5]) + eventStartsWithTime = append(eventStartsWithTime, time.Date(eventStart.Year(), eventStart.Month(), eventStart.Day(), startHour, startMinute, 0, 0, eventStart.Location())) + eventEndWithTime = append(eventEndWithTime, time.Date(eventStart.Year(), eventStart.Month(), eventStart.Day(), endHour, endMinute, 0, 0, eventStart.Location())) + } + + return eventStartsWithTime, eventEndWithTime +} + +func checkCycle(cycle string) bool { + + // check if cycle is valid + if len(cycle) < 12 { + return false + } + + // check if cycle has a weekday + weekDay := cycle[0:2] + if weekDay != "Mo" && weekDay != "Di" && weekDay != "Mi" && weekDay != "Do" && weekDay != "Fr" && weekDay != "Sa" && weekDay != "So" { + return false + } + + // check if cycle has a timespan + timeSpan := cycle[3:12] + if len(timeSpan) != 9 { + return false + } + + // check if timespan has a start and end time + startTime := timeSpan[0:5] + endTime := timeSpan[6:9] + if len(startTime) != 5 || len(endTime) != 3 { + return false + } + + // check if start time is before end time + if startTime > endTime { + return false + } + + return true + +} + +// check if ws or ss +func checkSemester(date time.Time) string { + if date.Month() >= 4 && date.Month() <= 9 { + return "ss" + } else { + return "ws" } } @@ -52,7 +206,7 @@ func fetchAllAvailableSportCourses() []string { // fetchAllHTWKSportCourses fetches all sport courses from the given links. // to speed up the process, it uses multithreading. -func fetchHTWKSportCourses(links []string) []Event { +func fetchHTWKSportCourses(links []string) []SportEntry { //multithreaded webpage requests to speed up the process @@ -75,7 +229,7 @@ func fetchHTWKSportCourses(links []string) []Event { } wg.Wait() - var events []Event + var events []SportEntry for _, doc := range htmlPageArray { if doc != nil { @@ -111,15 +265,15 @@ func htmlRequest(url string) (*goquery.Document, error) { // If the sport course exists, it will return the sport course. // goquery is used to parse the html. The html structure is not very consistent, so it is hard to parse. // May be improved in the future. -func fetchHtwkSportCourse(doc *goquery.Document) ([]Event, error) { - var events []Event +func fetchHtwkSportCourse(doc *goquery.Document) ([]SportEntry, error) { + var events []SportEntry if doc.Find("h1").Text() == "Aktuelle Sportangebote" { return nil, errors.New("not a sport course page") } doc.Find(".eventHead").Each(func(i int, s *goquery.Selection) { - var event Event + var event SportEntry var details EventDetails fullTitle := strings.TrimSpace(s.Find("h3").Text()) diff --git a/backend/sport/sportFetcherModel.go b/backend/sport/sportFetcherModel.go index 785e582..d7b82ce 100644 --- a/backend/sport/sportFetcherModel.go +++ b/backend/sport/sportFetcherModel.go @@ -4,8 +4,8 @@ import "time" // MODELS -// Event represents the overall event details. -type Event struct { +// SportEntry represents the overall event details. +type SportEntry struct { Title string Details EventDetails AdditionalNote string From e8432974a0605a1c237af5e35113ce99e1b4a6e5 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:13:32 +0100 Subject: [PATCH 06/18] fix:#82 update todo --- backend/sport/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/sport/main.go b/backend/sport/main.go index 64ffe81..5efed87 100644 --- a/backend/sport/main.go +++ b/backend/sport/main.go @@ -15,7 +15,7 @@ import ( "github.com/PuerkitoBio/goquery" ) -// @TODO: reformat the extracted data to the event model that there are events with real start and end dates +// @TODO: fix bug where cycle contains multiple days (e.g. "Mo + Mi 18:00-20:00") // @TODO: add tests // @TODO: add it to the service // @TODO: make it like a cron job to fetch the sport courses once a week From 9956261eabbc0ecdd192025b796cb96a0225c8eb Mon Sep 17 00:00:00 2001 From: survellow <59056368+survellow@users.noreply.github.com> Date: Tue, 12 Dec 2023 23:40:43 +0100 Subject: [PATCH 07/18] 98 implement anonymized DTO events --- backend/model/eventModel.go | 50 ++++++--- backend/model/eventModel_test.go | 71 ++++++++++++ backend/service/room/roomService.go | 19 +++- backend/service/room/roomService_test.go | 125 +++++++++++++++++++++ frontend/src/api/fetchRoom.ts | 8 +- frontend/src/components/RoomOccupation.vue | 2 +- frontend/src/model/event.ts | 11 ++ 7 files changed, 263 insertions(+), 23 deletions(-) create mode 100644 backend/service/room/roomService_test.go diff --git a/backend/model/eventModel.go b/backend/model/eventModel.go index 355e594..098c88a 100644 --- a/backend/model/eventModel.go +++ b/backend/model/eventModel.go @@ -2,6 +2,7 @@ package model import ( "slices" + "strings" "github.com/pocketbase/pocketbase/models" "github.com/pocketbase/pocketbase/tools/types" @@ -13,21 +14,30 @@ func (m Events) Contains(event Event) bool { return slices.Contains(m, event) } +type AnonymizedEventDTO struct { + Day string `db:"Day" json:"day"` + Week string `db:"Week" json:"week"` + Start types.DateTime `db:"start" json:"start"` + End types.DateTime `db:"end" json:"end"` + Rooms string `db:"Rooms" json:"rooms"` + Free bool `json:"free"` +} + type Event struct { - UUID string `db:"uuid" json:"uuid"` - Day string `db:"Day" json:"day"` - Week string `db:"Week" json:"week"` - Start types.DateTime `db:"start" json:"start"` - End types.DateTime `db:"end" json:"end"` - Name string `db:"Name" json:"name"` - EventType string `db:"EventType" json:"eventType"` - Compulsory string `db:"Compulsory" json:"compulsory"` - Prof string `db:"Prof" json:"prof"` - Rooms string `db:"Rooms" json:"rooms"` - Notes string `db:"Notes" json:"notes"` - BookedAt string `db:"BookedAt" json:"bookedAt"` - Course string `db:"course" json:"course"` - Semester string `db:"semester" json:"semester"` + UUID string `db:"uuid" json:"uuid"` + Day string `db:"Day" json:"day"` + Week string `db:"Week" json:"week"` + Start types.DateTime `db:"start" json:"start"` + End types.DateTime `db:"end" json:"end"` + Name string `db:"Name" json:"name"` + EventType string `db:"EventType" json:"eventType"` + Compulsory string `db:"Compulsory" json:"compulsory"` + Prof string `db:"Prof" json:"prof"` + Rooms string `db:"Rooms" json:"rooms"` + Notes string `db:"Notes" json:"notes"` + BookedAt string `db:"BookedAt" json:"bookedAt"` + Course string `db:"course" json:"course"` + Semester string `db:"semester" json:"semester"` models.BaseModel } @@ -52,3 +62,15 @@ func (m *Event) SetCourse(course string) Event { m.Course = course return *m } + +// Creates an AnonymizedEventDTO from an Event hiding all sensitive data +func (m *Event) AnonymizeEvent() AnonymizedEventDTO { + return AnonymizedEventDTO{ + Day: m.Day, + Week: m.Week, + Start: m.Start, + End: m.End, + Rooms: m.Rooms, + Free: strings.Contains(strings.ToLower(m.Name), "zur freien verfügung"), + } +} diff --git a/backend/model/eventModel_test.go b/backend/model/eventModel_test.go index 4462f4e..9608ba5 100644 --- a/backend/model/eventModel_test.go +++ b/backend/model/eventModel_test.go @@ -1,6 +1,7 @@ package model import ( + "reflect" "testing" "github.com/pocketbase/pocketbase/models" @@ -126,3 +127,73 @@ func TestEvent_Equals(t *testing.T) { }) } } + +func TestEvent_AnonymizeEvent(t *testing.T) { + type fields struct { + UUID string + Day string + Week string + Start types.DateTime + End types.DateTime + Name string + EventType string + Compulsory string + Prof string + Rooms string + Notes string + BookedAt string + Course string + Semester string + BaseModel models.BaseModel + } + tests := []struct { + name string + fields fields + want AnonymizedEventDTO + }{ + { + name: "empty event", + fields: fields{}, + want: AnonymizedEventDTO{Day: "", Week: "", Start: types.DateTime{}, End: types.DateTime{}, Rooms: "", Free: false}, + }, + { + name: "one event", + fields: fields{Name: "Event", Day: "test", Week: "test", Rooms: "test"}, + want: AnonymizedEventDTO{Day: "test", Week: "test", Start: types.DateTime{}, End: types.DateTime{}, Rooms: "test", Free: false}, + }, + { + name: "one event with free", + fields: fields{Name: "Räume zur freien Verfügung", Day: "test", Week: "test", Rooms: "test", Course: "test"}, + want: AnonymizedEventDTO{Day: "test", Week: "test", Start: types.DateTime{}, End: types.DateTime{}, Rooms: "test", Free: true}, + }, + { + name: "another free event", + fields: fields{Name: "Zur freien Verfügung", Day: "Montag", Week: "5", Start: types.DateTime{}, End: types.DateTime{}, Rooms: "TR_A1.28-S", Course: "42INM-3"}, + want: AnonymizedEventDTO{Day: "Montag", Week: "5", Start: types.DateTime{}, End: types.DateTime{}, Rooms: "TR_A1.28-S", Free: true}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &Event{ + UUID: tt.fields.UUID, + Day: tt.fields.Day, + Week: tt.fields.Week, + Start: tt.fields.Start, + End: tt.fields.End, + Name: tt.fields.Name, + EventType: tt.fields.EventType, + Compulsory: tt.fields.Compulsory, + Prof: tt.fields.Prof, + Rooms: tt.fields.Rooms, + Notes: tt.fields.Notes, + BookedAt: tt.fields.BookedAt, + Course: tt.fields.Course, + Semester: tt.fields.Semester, + BaseModel: tt.fields.BaseModel, + } + if got := m.AnonymizeEvent(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Event.AnonymizeEvent() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/service/room/roomService.go b/backend/service/room/roomService.go index 198b2dd..320bfea 100644 --- a/backend/service/room/roomService.go +++ b/backend/service/room/roomService.go @@ -1,10 +1,12 @@ package room import ( - "github.com/labstack/echo/v5" - "github.com/pocketbase/pocketbase" + "htwkalender/model" "htwkalender/service/db" "net/http" + + "github.com/labstack/echo/v5" + "github.com/pocketbase/pocketbase" ) func GetRooms(c echo.Context, app *pocketbase.PocketBase) error { @@ -14,10 +16,19 @@ func GetRooms(c echo.Context, app *pocketbase.PocketBase) error { func GetRoomScheduleForDay(c echo.Context, app *pocketbase.PocketBase, room string, date string) error { events := db.GetRoomScheduleForDay(app, room, date) - return c.JSON(http.StatusOK, events) + return c.JSON(http.StatusOK, anonymizeRooms(events)) } func GetRoomSchedule(c echo.Context, app *pocketbase.PocketBase, room string, from string, to string) error { events := db.GetRoomSchedule(app, room, from, to) - return c.JSON(http.StatusOK, events) + return c.JSON(http.StatusOK, anonymizeRooms(events)) +} + +// Transform the events to anonymized events throwing away all unnecessary information +func anonymizeRooms(events []model.Event) []model.AnonymizedEventDTO { + var anonymizedEvents = []model.AnonymizedEventDTO{} + for _, event := range events { + anonymizedEvents = append(anonymizedEvents, event.AnonymizeEvent()) + } + return anonymizedEvents } diff --git a/backend/service/room/roomService_test.go b/backend/service/room/roomService_test.go new file mode 100644 index 0000000..34345b8 --- /dev/null +++ b/backend/service/room/roomService_test.go @@ -0,0 +1,125 @@ +package room + +import ( + "htwkalender/model" + "reflect" + "testing" + + "github.com/pocketbase/pocketbase/tools/types" +) + +func Test_anonymizeRooms(t *testing.T) { + type args struct { + events []model.Event + } + tests := []struct { + name string + args args + want []model.AnonymizedEventDTO + }{ + { + name: "anonymize single event", + args: args{ + events: []model.Event{ + { + UUID: "testUUID", + Day: "Montag", + Week: "52", + Start: types.DateTime{}, + End: types.DateTime{}, + Name: "Secret", + EventType: "V", + Prof: "Prof. Dr. Secret", + Rooms: "Room", + Notes: "Secret", + BookedAt: "Secret", + Course: "42INM-3", + Semester: "ws", + Compulsory: "p", + }, + }, + }, + want: []model.AnonymizedEventDTO{ + { + Day: "Montag", + Week: "52", + Start: types.DateTime{}, + End: types.DateTime{}, + Rooms: "Room", + Free: false, + }, + }, + }, + { + name: "anonymize empty list", + args: args{ + events: []model.Event{}, + }, + want: []model.AnonymizedEventDTO{}, + }, + { + name: "anonymize multiple events", + args: args{ + events: []model.Event{ + { + UUID: "testUUID1", + Day: "Montag", + Week: "51", + Start: types.DateTime{}, + End: types.DateTime{}, + Name: "Incognito", + EventType: "V", + Prof: "Prof. Dr. Incognito", + Rooms: "Room", + Notes: "Incognito", + BookedAt: "Incognito", + Course: "69INM-2", + Semester: "sose", + Compulsory: "p", + }, + { + UUID: "testUUID2", + Day: "Dienstag", + Week: "52", + Start: types.DateTime{}, + End: types.DateTime{}, + Name: "Private", + EventType: "S", + Prof: "Prof.In. Dr.-Ing. Private", + Rooms: "Room", + Notes: "Private", + BookedAt: "Private", + Course: "42MIM-3", + Semester: "ws", + Compulsory: "w", + }, + }, + }, + want: []model.AnonymizedEventDTO{ + { + Day: "Montag", + Week: "51", + Start: types.DateTime{}, + End: types.DateTime{}, + Rooms: "Room", + Free: false, + }, + { + Day: "Dienstag", + Week: "52", + Start: types.DateTime{}, + End: types.DateTime{}, + Rooms: "Room", + Free: false, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := anonymizeRooms(tt.args.events); !reflect.DeepEqual(got, tt.want) { + t.Errorf("anonymizeRooms() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/frontend/src/api/fetchRoom.ts b/frontend/src/api/fetchRoom.ts index 1ca7caa..9163d6a 100644 --- a/frontend/src/api/fetchRoom.ts +++ b/frontend/src/api/fetchRoom.ts @@ -1,4 +1,4 @@ -import { Event } from "../model/event.ts"; +import { AnonymizedEventDTO } from "../model/event.ts"; export async function fetchRoom(): Promise { const rooms: string[] = []; @@ -16,8 +16,8 @@ export async function fetchEventsByRoomAndDuration( room: string, from_date: string, to_date: string, -): Promise { - const events: Event[] = []; +): Promise { + const events: AnonymizedEventDTO[] = []; await fetch( "/api/schedule?room=" + room + "&from=" + from_date + "&to=" + to_date, ) @@ -27,7 +27,7 @@ export async function fetchEventsByRoomAndDuration( }) .then((eventsResponse) => { console.log("Response:", eventsResponse); - eventsResponse.forEach((event: Event) => events.push(event)); + eventsResponse.forEach((event: AnonymizedEventDTO) => events.push(event)); }) .catch((error) => { console.log("Error fetching events: ", error); diff --git a/frontend/src/components/RoomOccupation.vue b/frontend/src/components/RoomOccupation.vue index 96fb947..ed294a9 100644 --- a/frontend/src/components/RoomOccupation.vue +++ b/frontend/src/components/RoomOccupation.vue @@ -52,7 +52,7 @@ async function getOccupation() { id: index, start: event.start.replace(/\s\+\d{4}\s\w+$/, "").replace(" ", "T"), end: event.end.replace(/\s\+\d{4}\s\w+$/, "").replace(" ", "T"), - showFree: event.name.toLowerCase().includes("zur freien verfügung"), + showFree: event.free }; }); diff --git a/frontend/src/model/event.ts b/frontend/src/model/event.ts index b9164d3..e4df3e9 100644 --- a/frontend/src/model/event.ts +++ b/frontend/src/model/event.ts @@ -14,3 +14,14 @@ export class Event { public week: string, ) {} } + +export class AnonymizedEventDTO { + constructor( + public day: string, + public week: string, + public start: string, + public end: string, + public rooms: string, + public free: boolean + ) {} +} From 2f73c99c5190591b1b46e65babe7c6ac44eb5bb6 Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 00:46:06 +0100 Subject: [PATCH 08/18] feat:#82 integrated sport into fetch service --- backend/{sport => model}/sportFetcherModel.go | 8 +- backend/service/db/dbEvents.go | 13 + backend/service/fetch/sport/sportFetcher.go | 467 ++++++++++++++++++ .../service/fetch/sport/sportFetcher_test.go | 40 ++ backend/sport/main.go | 353 ------------- 5 files changed, 527 insertions(+), 354 deletions(-) rename backend/{sport => model}/sportFetcherModel.go (90%) create mode 100644 backend/service/fetch/sport/sportFetcher.go create mode 100644 backend/service/fetch/sport/sportFetcher_test.go delete mode 100644 backend/sport/main.go diff --git a/backend/sport/sportFetcherModel.go b/backend/model/sportFetcherModel.go similarity index 90% rename from backend/sport/sportFetcherModel.go rename to backend/model/sportFetcherModel.go index d7b82ce..600f6ba 100644 --- a/backend/sport/sportFetcherModel.go +++ b/backend/model/sportFetcherModel.go @@ -1,4 +1,4 @@ -package main +package model import "time" @@ -47,3 +47,9 @@ type Participants struct { TotalPlaces int WaitList int } + +type SportDayStartEnd struct { + Start time.Time + End time.Time + Day time.Weekday +} diff --git a/backend/service/db/dbEvents.go b/backend/service/db/dbEvents.go index 3253525..e301f00 100644 --- a/backend/service/db/dbEvents.go +++ b/backend/service/db/dbEvents.go @@ -2,6 +2,7 @@ package db import ( "htwkalender/model" + "time" "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase" @@ -251,3 +252,15 @@ func FindAllEventsByModule(app *pocketbase.PocketBase, module model.Module) (mod return events, nil } + +func GetAllModulesByNameAndDateRange(app *pocketbase.PocketBase, name string, startDate time.Time, endDate time.Time) (model.Events, error) { + var events model.Events + + err := app.Dao().DB().Select("*").From("events").Where(dbx.NewExp("Name = {:name} AND Start >= {:startDate} AND End <= {:endDate}", dbx.Params{"name": name, "startDate": startDate, "endDate": endDate})).All(&events) + if err != nil { + print("Error while getting events from database: ", err) + return nil, err + } + + return events, nil +} diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go new file mode 100644 index 0000000..e71c285 --- /dev/null +++ b/backend/service/fetch/sport/sportFetcher.go @@ -0,0 +1,467 @@ +package sport + +import ( + "errors" + "github.com/google/uuid" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/tools/types" + "htwkalender/model" + "htwkalender/service/db" + "net/http" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/PuerkitoBio/goquery" +) + +// @TODO: add tests +// @TODO: make it like a cron job to fetch the sport courses once a week +func RetrieveAndFetchAllSportCourses(app *pocketbase.PocketBase) []model.Event { + + var sportCourseLinks = fetchAllAvailableSportCourses() + sportEntries := fetchHTWKSportCourses(sportCourseLinks) + events := formatEntriesToEvents(sportEntries) + + var earliestDate time.Time + var latestDate time.Time + + // find earliest and latest date in events + for _, event := range events { + if event.Start.Time().Before(earliestDate) { + earliestDate = event.Start.Time() + } + if event.End.Time().After(latestDate) { + latestDate = event.End.Time() + } + } + + // get all events from database where name = Feiertage und lehrveranstaltungsfreie Tage + holidays, err := db.GetAllModulesByNameAndDateRange(app, "Feiertage und lehrveranstaltungsfreie Tage", earliestDate, latestDate) + if err != nil { + return nil + } + + // remove all events that have same year, month and day as items in holidays + for _, holiday := range holidays { + for i, event := range events { + if event.Start.Time().Year() == holiday.Start.Time().Year() && + event.Start.Time().Month() == holiday.Start.Time().Month() && + event.Start.Time().Day() == holiday.Start.Time().Day() { + events = append(events[:i], events[i+1:]...) + } + } + } + + // save events to database + savedEvents, err := db.SaveEvents(events, app) + + if err != nil { + return nil + } + + return savedEvents + +} + +func formatEntriesToEvents(entries []model.SportEntry) []model.Event { + + var events []model.Event + + for i, entry := range entries { + eventStarts, eventEnds := getWeekEvents(entry.Details.DateRange.Start, entry.Details.DateRange.End, entry.Details.Cycle) + for j := range eventStarts { + + start, _ := types.ParseDateTime(eventStarts[j].In(time.UTC)) + end, _ := types.ParseDateTime(eventEnds[j].In(time.UTC)) + + var event = model.Event{ + UUID: uuid.NewSHA1(uuid.NameSpaceDNS, []byte(entry.Title+strconv.FormatInt(int64(i), 10))).String(), + Day: toGermanWeekdayString(entry.Details.DateRange.Start.Weekday()), + Week: strconv.Itoa(23), + Start: start, + End: end, + Name: entry.Title, + EventType: entry.Details.Type, + Prof: entry.Details.CourseLead.Name, + Rooms: entry.Details.Location.Name, + Notes: entry.AdditionalNote, + BookedAt: "", + Course: "Sport", + Semester: checkSemester(entry.Details.DateRange.Start), + } + events = append(events, event) + } + } + return events +} + +func getDayInt(weekDay string) int { + var weekDayInt int + switch weekDay { + case "Mo": + weekDayInt = 1 + case "Di": + weekDayInt = 2 + case "Mi": + weekDayInt = 3 + case "Do": + weekDayInt = 4 + case "Fr": + weekDayInt = 5 + case "Sa": + weekDayInt = 6 + case "So": + weekDayInt = 0 + } + return weekDayInt +} + +func toGermanWeekdayString(weekday time.Weekday) string { + switch weekday { + case time.Monday: + return "Montag" + case time.Tuesday: + return "Dienstag" + case time.Wednesday: + return "Mittwoch" + case time.Thursday: + return "Donnerstag" + case time.Friday: + return "Freitag" + case time.Saturday: + return "Samstag" + case time.Sunday: + return "Sonntag" + default: + return "" + } +} + +func extractStartAndEndTime(cycle string) (int, int, int, int) { + timeRegExp, _ := regexp.Compile("[0-9]{2}:[0-9]{2}") + times := timeRegExp.FindAllString(cycle, 2) + startHour, _ := strconv.Atoi(times[0][0:2]) + startMinute, _ := strconv.Atoi(times[0][3:5]) + + endHour, _ := strconv.Atoi(times[1][0:2]) + endMinute, _ := strconv.Atoi(times[1][3:5]) + return startHour, startMinute, endHour, endMinute +} + +func getWeekEvents(start time.Time, end time.Time, cycle string) ([]time.Time, []time.Time) { + + var weekEvents []model.SportDayStartEnd + + // split by regexp to get the cycle parts + var cycleParts []string + cycleParts = splitByCommaWithTime(cycle) + + for _, cyclePart := range cycleParts { + + //cut string at the first integer/number + cyclePartWithDaysOnly := cyclePart[0:strings.IndexFunc(cyclePart, func(r rune) bool { return r >= '0' && r <= '9' })] + + // check if cycle has multiple days by checking if it has a plus sign + if strings.Contains(cyclePartWithDaysOnly, "+") { + // find all days in cycle part by regexp + dayRegExp, _ := regexp.Compile("[A-Z][a-z]") + days := dayRegExp.FindAllString(cyclePart, -1) + startHour, startMinute, endHour, endMinute := extractStartAndEndTime(cyclePart) + + // creating a SportDayStartEnd for each day in the cycle + for _, day := range days { + weekEvents = append(weekEvents, model.SportDayStartEnd{ + Start: time.Date(start.Year(), start.Month(), start.Day(), startHour, startMinute, 0, 0, start.Location()), + End: time.Date(end.Year(), end.Month(), end.Day(), endHour, endMinute, 0, 0, end.Location()), + Day: time.Weekday(getDayInt(day)), + }) + } + + } + + // check if cycle has multiple days by checking if it has a minus sign + if strings.Contains(cyclePartWithDaysOnly, "-") { + // find all days in cycle part by regexp + dayRegExp, _ := regexp.Compile("[A-Z][a-z]") + days := dayRegExp.FindAllString(cyclePart, 2) + startHour, startMinute, endHour, endMinute := extractStartAndEndTime(cyclePart) + + //create a int array with all days from start to end day + var daysBetween []int + for i := getDayInt(days[0]); i <= getDayInt(days[1]); i++ { + daysBetween = append(daysBetween, i) + } + + // creating a SportDayStartEnd for each day in the cycle + for _, day := range daysBetween { + weekEvents = append(weekEvents, model.SportDayStartEnd{ + Start: time.Date(start.Year(), start.Month(), start.Day(), startHour, startMinute, 0, 0, start.Location()), + End: time.Date(end.Year(), end.Month(), end.Day(), endHour, endMinute, 0, 0, end.Location()), + Day: time.Weekday(day), + }) + } + } + + // check if cycle has only one day + if !strings.Contains(cyclePartWithDaysOnly, "-") && !strings.Contains(cyclePartWithDaysOnly, "+") { + // find all days in cycle part by regexp + dayRegExp, _ := regexp.Compile("[A-Z][a-z]") + days := dayRegExp.FindAllString(cyclePart, -1) + startHour, startMinute, endHour, endMinute := extractStartAndEndTime(cyclePart) + + // creating a SportDayStartEnd for each day in the cycle + for _, day := range days { + weekEvents = append(weekEvents, model.SportDayStartEnd{ + Start: time.Date(start.Year(), start.Month(), start.Day(), startHour, startMinute, 0, 0, start.Location()), + End: time.Date(end.Year(), end.Month(), end.Day(), endHour, endMinute, 0, 0, end.Location()), + Day: time.Weekday(getDayInt(day)), + }) + } + } + } + + var startDatesList []time.Time + var endDatesList []time.Time + + for _, weekEvent := range weekEvents { + startDates, endDates := createEventListFromStartToEndMatchingDay(weekEvent) + startDatesList = append(startDatesList, startDates...) + endDatesList = append(endDatesList, endDates...) + } + + return startDatesList, endDatesList +} + +func createEventListFromStartToEndMatchingDay(weekEvent model.SportDayStartEnd) ([]time.Time, []time.Time) { + var startDates []time.Time + var endDates []time.Time + for d := weekEvent.Start; d.Before(weekEvent.End); d = d.AddDate(0, 0, 1) { + if d.Weekday() == weekEvent.Day { + startDates = append(startDates, time.Date(d.Year(), d.Month(), d.Day(), weekEvent.Start.Hour(), weekEvent.Start.Minute(), 0, 0, d.Location())) + endDates = append(endDates, time.Date(d.Year(), d.Month(), d.Day(), weekEvent.End.Hour(), weekEvent.End.Minute(), 0, 0, d.Location())) + } + } + return startDates, endDates +} + +func splitByCommaWithTime(input string) []string { + var result []string + + // Split by comma + parts := strings.Split(input, ", ") + + // Regular expression to match a day with time + regex := regexp.MustCompile(`([A-Za-z]{2,}(\+[A-Za-z]{2,})* \d{2}:\d{2}-\d{2}:\d{2})`) + + // Iterate over parts and combine when necessary + var currentPart string + for _, part := range parts { + if regex.MatchString(part) { + if currentPart != "" { + currentPart += ", " + part + result = append(result, currentPart) + currentPart = "" + } else { + result = append(result, part) + } + // If the part contains a day with time, start a new currentPart + + } else { + // If there's no currentPart, start a new one + if currentPart != "" { + currentPart += ", " + part + } else { + currentPart = part + } + } + } + + // Add the last currentPart to the result + if currentPart != "" { + result = append(result, currentPart) + } + + return result +} + +// check if ws or ss +func checkSemester(date time.Time) string { + if date.Month() >= 4 && date.Month() <= 9 { + return "ss" + } else { + return "ws" + } +} + +// fetch the main page where all sport courses are listed and extract all links to the sport courses +func fetchAllAvailableSportCourses() []string { + var url = "https://sport.htwk-leipzig.de/sportangebote" + + var doc, err = htmlRequest(url) + + if err != nil { + return nil + } + + // link list of all sport courses + var links []string + + // find all links to sport courses with regex https://sport.htwk-leipzig.de/sportangebote/detail/sport/ + [0-9]{1,4} + doc.Find("a[href]").Each(func(i int, s *goquery.Selection) { + link, _ := s.Attr("href") + if strings.HasPrefix(link, "/sportangebote/detail/sport/") { + links = append(links, link) + } + }) + + return links +} + +// fetchAllHTWKSportCourses fetches all sport courses from the given links. +// to speed up the process, it uses multithreading. + +func fetchHTWKSportCourses(links []string) []model.SportEntry { + + //multithreaded webpage requests to speed up the process + + var maxThreads = 10 + var htmlPageArray = make([]*goquery.Document, len(links)) + var hostUrl = "https://sport.htwk-leipzig.de" + + var wg sync.WaitGroup + wg.Add(maxThreads) + for i := 0; i < maxThreads; i++ { + go func(i int) { + for j := i; j < len(links); j += maxThreads { + doc, err := htmlRequest(hostUrl + links[j]) + if err == nil { + htmlPageArray[j] = doc + } + } + wg.Done() + }(i) + } + wg.Wait() + + var events []model.SportEntry + + for _, doc := range htmlPageArray { + if doc != nil { + event, err := fetchHtwkSportCourse(doc) + if err == nil { + events = append(events, event...) + } + } + } + return events +} + +func htmlRequest(url string) (*goquery.Document, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, err + } + + return doc, nil +} + +// fetchHtwkSportCourse fetches the sport course from the given url and id. +// If the sport course does not exist, it will return an error. +// If the sport course exists, it will return the sport course. +// goquery is used to parse the html. The html structure is not very consistent, so it is hard to parse. +// May be improved in the future. +func fetchHtwkSportCourse(doc *goquery.Document) ([]model.SportEntry, error) { + var events []model.SportEntry + + if doc.Find("h1").Text() == "Aktuelle Sportangebote" { + return nil, errors.New("not a sport course page") + } + + doc.Find(".eventHead").Each(func(i int, s *goquery.Selection) { + var event model.SportEntry + var details model.EventDetails + + fullTitle := strings.TrimSpace(s.Find("h3").Text()) + titleParts := strings.Split(fullTitle, "-") + if len(titleParts) > 0 { + event.Title = "Sport: " + strings.TrimSpace(titleParts[0]) + } + + if len(titleParts) > 2 { + details.Type = strings.TrimSpace(titleParts[len(titleParts)-1]) + } + + s.NextFiltered("table.eventDetails").Find("tr").Each(func(i int, s *goquery.Selection) { + key := strings.TrimSpace(s.Find("td").First().Text()) + value := strings.TrimSpace(s.Find("td").Last().Text()) + + switch key { + case "Zeitraum": + dates := strings.Split(value, "-") + if len(dates) == 2 { + startDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[0])) + endDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[1])) + details.DateRange = model.DateRange{Start: startDate, End: endDate} + } + case "Zyklus": + details.Cycle = value + case "Geschlecht": + details.Gender = value + case "Leiter": + leaderName := strings.TrimSpace(s.Find("td a").Text()) + leadersSlice := strings.Split(leaderName, "\n") + for i, leader := range leadersSlice { + leadersSlice[i] = strings.TrimSpace(leader) + } + formattedLeaders := strings.Join(leadersSlice, ", ") + leaderLink, _ := s.Find("td a").Attr("href") + details.CourseLead = model.CourseLead{Name: formattedLeaders, Link: leaderLink} + case "Ort": + locationDetails := strings.Split(value, "(") + if len(locationDetails) == 2 { + details.Location = model.Location{ + Name: strings.TrimSpace(locationDetails[0]), + Address: strings.TrimRight(strings.TrimSpace(locationDetails[1]), ")"), + } + } + case "Teilnehmer": + parts := strings.Split(value, "/") + if len(parts) >= 3 { + bookings, _ := strconv.Atoi(strings.TrimSpace(parts[0])) + totalPlaces, _ := strconv.Atoi(strings.TrimSpace(parts[1])) + waitList, _ := strconv.Atoi(strings.TrimSpace(parts[2])) + details.Participants = model.Participants{Bookings: bookings, TotalPlaces: totalPlaces, WaitList: waitList} + } + case "Kosten": + details.Cost = value // makes no sense since you need to be logged in to see the price + case "Hinweis": + var allNotes []string + + s.Find("td").Last().Contents().Each(func(i int, s *goquery.Selection) { + if s.Is("h4.eventAdvice") || goquery.NodeName(s) == "#text" { + note := strings.TrimSpace(s.Text()) + if note != "" { + allNotes = append(allNotes, note) + } + } + }) + + event.AdditionalNote = strings.Join(allNotes, " ") + } + }) + + event.Details = details + events = append(events, event) + }) + + return events, nil +} diff --git a/backend/service/fetch/sport/sportFetcher_test.go b/backend/service/fetch/sport/sportFetcher_test.go new file mode 100644 index 0000000..c630e88 --- /dev/null +++ b/backend/service/fetch/sport/sportFetcher_test.go @@ -0,0 +1,40 @@ +package sport + +import ( + "reflect" + "testing" +) + +func Test_splitByCommaWithTime(t *testing.T) { + type args struct { + input string + } + tests := []struct { + name string + args args + want []string + }{ + {"one string", args{"one"}, []string{"one"}}, + {"two strings", args{"one,two"}, []string{"one,two"}}, + {"three strings", args{"one,two,three"}, []string{"one,two,three"}}, + // e.g. "Mo 18:00-20:00, Di 18:00-20:00" -> ["Mo 18:00-20:00", "Di 18:00-20:00"] + // e.g. "Mo 18:00-20:00, Di 18:00-20:00, Mi 18:00-20:00" -> ["Mo 18:00-20:00", "Di 18:00-20:00", "Mi 18:00-20:00"] + // e.g. "Mo, Mi, Fr 18:00-20:00, Sa 20:00-21:00" -> ["Mo, Mi, Fr 18:00-20:00", "Sa 20:00-21:00"] + // e.g. "Mo, Mi, Fr 18:00-20:00, Sa 20:00-21:00, So 20:00-21:00" -> ["Mo, Mi, Fr 18:00-20:00", "Sa 20:00-21:00", "So 20:00-21:00"] + // e.g. "Mo+Mi+Fr 18:00-20:00, Sa 20:00-21:00" -> ["Mo+Mi+Fr 18:00-20:00", "Sa 20:00-21:00"] + // e.g. "Mo+Mi 18:00-20:00, Sa 20:00-21:00, So 20:00-21:00" -> ["Mo+Mi 18:00-20:00", "Sa 20:00-21:00", "So 20:00-21:00"] + {"Mo 18:00-20:00, Di 18:00-20:00", args{"Mo 18:00-20:00, Di 18:00-20:00"}, []string{"Mo 18:00-20:00", "Di 18:00-20:00"}}, + {"Mo 18:00-20:00, Di 18:00-20:00, Mi 18:00-20:00", args{"Mo 18:00-20:00, Di 18:00-20:00, Mi 18:00-20:00"}, []string{"Mo 18:00-20:00", "Di 18:00-20:00", "Mi 18:00-20:00"}}, + {"Mo, Mi, Fr 18:00-20:00, Sa 20:00-21:00", args{"Mo, Mi, Fr 18:00-20:00, Sa 20:00-21:00"}, []string{"Mo, Mi, Fr 18:00-20:00", "Sa 20:00-21:00"}}, + {"Mo, Mi, Fr 18:00-20:00, Sa 20:00-21:00, So 20:00-21:00", args{"Mo, Mi, Fr 18:00-20:00, Sa 20:00-21:00, So 20:00-21:00"}, []string{"Mo, Mi, Fr 18:00-20:00", "Sa 20:00-21:00", "So 20:00-21:00"}}, + {"Mo+Mi+Fr 18:00-20:00, Sa 20:00-21:00", args{"Mo+Mi+Fr 18:00-20:00, Sa 20:00-21:00"}, []string{"Mo+Mi+Fr 18:00-20:00", "Sa 20:00-21:00"}}, + {"Mo+Mi 18:00-20:00, Sa 20:00-21:00, So 20:00-21:00", args{"Mo+Mi 18:00-20:00, Sa 20:00-21:00, So 20:00-21:00"}, []string{"Mo+Mi 18:00-20:00", "Sa 20:00-21:00", "So 20:00-21:00"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := splitByCommaWithTime(tt.args.input); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitByCommaWithTime() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/sport/main.go b/backend/sport/main.go deleted file mode 100644 index 5efed87..0000000 --- a/backend/sport/main.go +++ /dev/null @@ -1,353 +0,0 @@ -package main - -import ( - "errors" - "github.com/google/uuid" - "github.com/pocketbase/pocketbase/tools/types" - "htwkalender/model" - "net/http" - "regexp" - "strconv" - "strings" - "sync" - "time" - - "github.com/PuerkitoBio/goquery" -) - -// @TODO: fix bug where cycle contains multiple days (e.g. "Mo + Mi 18:00-20:00") -// @TODO: add tests -// @TODO: add it to the service -// @TODO: make it like a cron job to fetch the sport courses once a week -func main() { - - var sportCourseLinks = fetchAllAvailableSportCourses() - sportEntries := fetchHTWKSportCourses(sportCourseLinks) - - for _, event := range sportEntries { - println(event.Title) - } - - events := formatEntriesToEvents(sportEntries) - - for _, event := range events { - println(event.Name) - } -} - -func formatEntriesToEvents(entries []SportEntry) []model.Event { - - var events []model.Event - - for i, entry := range entries { - - eventStarts, eventEnds := calculateEventStarts(entry.Details.DateRange.Start, entry.Details.DateRange.End, entry.Details.Cycle) - - for j := range eventStarts { - - start, _ := types.ParseDateTime(eventStarts[j].In(time.UTC)) - end, _ := types.ParseDateTime(eventEnds[j].In(time.UTC)) - - var event = model.Event{ - UUID: uuid.NewSHA1(uuid.NameSpaceDNS, []byte(entry.Title+strconv.FormatInt(int64(i), 10))).String(), - Day: entry.Details.DateRange.Start.Weekday().String(), - Week: strconv.Itoa(23), - Start: start, - End: end, - Name: entry.Title, - EventType: entry.Details.Type, - Prof: entry.Details.CourseLead.Name, - Rooms: entry.Details.Location.Name, - Notes: entry.AdditionalNote, - BookedAt: "", - Course: "Sport", - Semester: checkSemester(entry.Details.DateRange.Start), - } - events = append(events, event) - } - - } - return events -} - -func calculateEventStarts(start time.Time, end time.Time, cycle string) ([]time.Time, []time.Time) { - - // start is the begin of the cycle e.g. 01.04.2020 - // end is the end of the cycle e.g. 30.09.2020 - // cycle is the day and timespan (e.g. "Mo 18:00-20:00") - - // check if start is before end - if start.After(end) { - return nil, nil - } - - // check if cycle is valid - if !checkCycle(cycle) { - return nil, nil - } - - var weekDay = cycle[0:2] - // match weekday to time.Weekday (e.g. "Mo" -> time.Monday) - var weekDayInt int - - switch weekDay { - case "Mo": - weekDayInt = 1 - case "Di": - weekDayInt = 2 - case "Mi": - weekDayInt = 3 - case "Do": - weekDayInt = 4 - case "Fr": - weekDayInt = 5 - case "Sa": - weekDayInt = 6 - case "So": - weekDayInt = 0 - } - - // get every date matching the weekday in the cycle between start and end - var eventDates []time.Time - for d := start; d.Before(end); d = d.AddDate(0, 0, 1) { - if d.Weekday() == time.Weekday(weekDayInt) { - eventDates = append(eventDates, d) - } - } - - // add hours and minutes to the dates in eventDates - // array of tuple of start and end times - var eventStartsWithTime []time.Time - var eventEndWithTime []time.Time - - for _, eventStart := range eventDates { - timeRegExp, _ := regexp.Compile("[0-9]{2}:[0-9]{2}") - times := timeRegExp.FindAllString(cycle, 2) - startHour, _ := strconv.Atoi(times[0][0:2]) - startMinute, _ := strconv.Atoi(times[0][3:5]) - - endHour, _ := strconv.Atoi(times[1][0:2]) - endMinute, _ := strconv.Atoi(times[1][3:5]) - eventStartsWithTime = append(eventStartsWithTime, time.Date(eventStart.Year(), eventStart.Month(), eventStart.Day(), startHour, startMinute, 0, 0, eventStart.Location())) - eventEndWithTime = append(eventEndWithTime, time.Date(eventStart.Year(), eventStart.Month(), eventStart.Day(), endHour, endMinute, 0, 0, eventStart.Location())) - } - - return eventStartsWithTime, eventEndWithTime -} - -func checkCycle(cycle string) bool { - - // check if cycle is valid - if len(cycle) < 12 { - return false - } - - // check if cycle has a weekday - weekDay := cycle[0:2] - if weekDay != "Mo" && weekDay != "Di" && weekDay != "Mi" && weekDay != "Do" && weekDay != "Fr" && weekDay != "Sa" && weekDay != "So" { - return false - } - - // check if cycle has a timespan - timeSpan := cycle[3:12] - if len(timeSpan) != 9 { - return false - } - - // check if timespan has a start and end time - startTime := timeSpan[0:5] - endTime := timeSpan[6:9] - if len(startTime) != 5 || len(endTime) != 3 { - return false - } - - // check if start time is before end time - if startTime > endTime { - return false - } - - return true - -} - -// check if ws or ss -func checkSemester(date time.Time) string { - if date.Month() >= 4 && date.Month() <= 9 { - return "ss" - } else { - return "ws" - } -} - -// fetch the main page where all sport courses are listed and extract all links to the sport courses -func fetchAllAvailableSportCourses() []string { - var url = "https://sport.htwk-leipzig.de/sportangebote" - - var doc, err = htmlRequest(url) - - if err != nil { - return nil - } - - // link list of all sport courses - var links []string - - // find all links to sport courses with regex https://sport.htwk-leipzig.de/sportangebote/detail/sport/ + [0-9]{1,4} - doc.Find("a[href]").Each(func(i int, s *goquery.Selection) { - link, _ := s.Attr("href") - if strings.HasPrefix(link, "/sportangebote/detail/sport/") { - links = append(links, link) - } - }) - - return links -} - -// fetchAllHTWKSportCourses fetches all sport courses from the given links. -// to speed up the process, it uses multithreading. - -func fetchHTWKSportCourses(links []string) []SportEntry { - - //multithreaded webpage requests to speed up the process - - var maxThreads = 10 - var htmlPageArray = make([]*goquery.Document, len(links)) - var hostUrl = "https://sport.htwk-leipzig.de" - - var wg sync.WaitGroup - wg.Add(maxThreads) - for i := 0; i < maxThreads; i++ { - go func(i int) { - for j := i; j < len(links); j += maxThreads { - doc, err := htmlRequest(hostUrl + links[j]) - if err == nil { - htmlPageArray[j] = doc - } - } - wg.Done() - }(i) - } - wg.Wait() - - var events []SportEntry - - for _, doc := range htmlPageArray { - if doc != nil { - event, err := fetchHtwkSportCourse(doc) - if err == nil { - events = append(events, event...) - } - } - } - return events -} - -func htmlRequest(url string) (*goquery.Document, error) { - println("fetching " + url) - - resp, err := http.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - doc, err := goquery.NewDocumentFromReader(resp.Body) - if err != nil { - return nil, err - } - - println("finished fetching " + url) - return doc, nil -} - -// fetchHtwkSportCourse fetches the sport course from the given url and id. -// If the sport course does not exist, it will return an error. -// If the sport course exists, it will return the sport course. -// goquery is used to parse the html. The html structure is not very consistent, so it is hard to parse. -// May be improved in the future. -func fetchHtwkSportCourse(doc *goquery.Document) ([]SportEntry, error) { - var events []SportEntry - - if doc.Find("h1").Text() == "Aktuelle Sportangebote" { - return nil, errors.New("not a sport course page") - } - - doc.Find(".eventHead").Each(func(i int, s *goquery.Selection) { - var event SportEntry - var details EventDetails - - fullTitle := strings.TrimSpace(s.Find("h3").Text()) - titleParts := strings.Split(fullTitle, "-") - if len(titleParts) > 0 { - event.Title = strings.TrimSpace(titleParts[0]) - } - - if len(titleParts) > 2 { - details.Type = strings.TrimSpace(titleParts[len(titleParts)-1]) - } - - s.NextFiltered("table.eventDetails").Find("tr").Each(func(i int, s *goquery.Selection) { - key := strings.TrimSpace(s.Find("td").First().Text()) - value := strings.TrimSpace(s.Find("td").Last().Text()) - - switch key { - case "Zeitraum": - dates := strings.Split(value, "-") - if len(dates) == 2 { - startDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[0])) - endDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[1])) - details.DateRange = DateRange{Start: startDate, End: endDate} - } - case "Zyklus": - details.Cycle = value - case "Geschlecht": - details.Gender = value - case "Leiter": - leaderName := strings.TrimSpace(s.Find("td a").Text()) - leadersSlice := strings.Split(leaderName, "\n") - for i, leader := range leadersSlice { - leadersSlice[i] = strings.TrimSpace(leader) - } - formattedLeaders := strings.Join(leadersSlice, ", ") - leaderLink, _ := s.Find("td a").Attr("href") - details.CourseLead = CourseLead{Name: formattedLeaders, Link: leaderLink} - case "Ort": - locationDetails := strings.Split(value, "(") - if len(locationDetails) == 2 { - details.Location = Location{ - Name: strings.TrimSpace(locationDetails[0]), - Address: strings.TrimRight(strings.TrimSpace(locationDetails[1]), ")"), - } - } - case "Teilnehmer": - parts := strings.Split(value, "/") - if len(parts) >= 3 { - bookings, _ := strconv.Atoi(strings.TrimSpace(parts[0])) - totalPlaces, _ := strconv.Atoi(strings.TrimSpace(parts[1])) - waitList, _ := strconv.Atoi(strings.TrimSpace(parts[2])) - details.Participants = Participants{Bookings: bookings, TotalPlaces: totalPlaces, WaitList: waitList} - } - case "Kosten": - details.Cost = value // makes no sense since you need to be logged in to see the price - case "Hinweis": - var allNotes []string - - s.Find("td").Last().Contents().Each(func(i int, s *goquery.Selection) { - if s.Is("h4.eventAdvice") || goquery.NodeName(s) == "#text" { - note := strings.TrimSpace(s.Text()) - if note != "" { - allNotes = append(allNotes, note) - } - } - }) - - event.AdditionalNote = strings.Join(allNotes, " ") - } - }) - - event.Details = details - events = append(events, event) - }) - - return events, nil -} From 4572541d6530a3a460985907f37bdbd72782bd6f Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 00:49:39 +0100 Subject: [PATCH 09/18] feat:#82 added fetch sports route --- backend/service/addRoute.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index 79877fd..1029a28 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -3,6 +3,7 @@ package service import ( "htwkalender/service/db" "htwkalender/service/events" + "htwkalender/service/fetch/sport" v1 "htwkalender/service/fetch/v1" v2 "htwkalender/service/fetch/v2" "htwkalender/service/ical" @@ -54,6 +55,26 @@ func AddRoutes(app *pocketbase.PocketBase) { return nil }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + _, err := e.Router.AddRoute(echo.Route{ + Method: http.MethodGet, + Path: "/api/fetch/sports", + Handler: func(c echo.Context) error { + + sportEvents := sport.RetrieveAndFetchAllSportCourses(app) + return c.JSON(200, sportEvents) + }, + Middlewares: []echo.MiddlewareFunc{ + apis.ActivityLogger(app), + apis.RequireAdminAuth(), + }, + }) + if err != nil { + return err + } + return nil + }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { _, err := e.Router.AddRoute(echo.Route{ Method: http.MethodDelete, From cdbbe4bbf9c8a0e4dc1cb7111315e8eb0d23e13e Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:03:37 +0100 Subject: [PATCH 10/18] update:#82 added schedule for sport update --- backend/service/addRoute.go | 2 +- backend/service/addSchedule.go | 7 +++++++ backend/service/fetch/sport/sportFetcher.go | 8 +++++++- backend/service/functions/semester.go | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 backend/service/functions/semester.go diff --git a/backend/service/addRoute.go b/backend/service/addRoute.go index 1029a28..422021e 100644 --- a/backend/service/addRoute.go +++ b/backend/service/addRoute.go @@ -61,7 +61,7 @@ func AddRoutes(app *pocketbase.PocketBase) { Path: "/api/fetch/sports", Handler: func(c echo.Context) error { - sportEvents := sport.RetrieveAndFetchAllSportCourses(app) + sportEvents := sport.FetchAndUpdateSportEvents(app) return c.JSON(200, sportEvents) }, Middlewares: []echo.MiddlewareFunc{ diff --git a/backend/service/addSchedule.go b/backend/service/addSchedule.go index bd0cd96..0b62f39 100644 --- a/backend/service/addSchedule.go +++ b/backend/service/addSchedule.go @@ -6,6 +6,7 @@ import ( "github.com/pocketbase/pocketbase/tools/cron" "htwkalender/service/course" "htwkalender/service/feed" + "htwkalender/service/fetch/sport" "htwkalender/service/functions/time" ) @@ -26,6 +27,12 @@ func AddSchedules(app *pocketbase.PocketBase) { // clean feeds older than 6 months feed.ClearFeeds(app.Dao(), 6, time.RealClock{}) }) + + // Every day at 2am update all Sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * *" + scheduler.MustAdd("fetchEvents", "0 2 * * *", func() { + sport.FetchAndUpdateSportEvents(app) + }) + scheduler.Start() return nil }) diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go index e71c285..ddc7377 100644 --- a/backend/service/fetch/sport/sportFetcher.go +++ b/backend/service/fetch/sport/sportFetcher.go @@ -7,6 +7,7 @@ import ( "github.com/pocketbase/pocketbase/tools/types" "htwkalender/model" "htwkalender/service/db" + "htwkalender/service/functions" "net/http" "regexp" "strconv" @@ -19,7 +20,7 @@ import ( // @TODO: add tests // @TODO: make it like a cron job to fetch the sport courses once a week -func RetrieveAndFetchAllSportCourses(app *pocketbase.PocketBase) []model.Event { +func FetchAndUpdateSportEvents(app *pocketbase.PocketBase) []model.Event { var sportCourseLinks = fetchAllAvailableSportCourses() sportEntries := fetchHTWKSportCourses(sportCourseLinks) @@ -55,6 +56,11 @@ func RetrieveAndFetchAllSportCourses(app *pocketbase.PocketBase) []model.Event { } } + err = db.DeleteAllEventsForCourse(app, "Sport", functions.GetCurrentSemesterString()) + if err != nil { + return nil + } + // save events to database savedEvents, err := db.SaveEvents(events, app) diff --git a/backend/service/functions/semester.go b/backend/service/functions/semester.go new file mode 100644 index 0000000..e0a457f --- /dev/null +++ b/backend/service/functions/semester.go @@ -0,0 +1,14 @@ +package functions + +import "time" + +// GetCurrentSemesterString returns the current semester as string +// if current month is between 10 and 03 -> winter semester "ws" +func GetCurrentSemesterString() string { + + if time.Now().Month() >= 10 || time.Now().Month() <= 3 { + return "ws" + } else { + return "ss" + } +} From 8463ec446fc64c985a2fe9407ecab4c0c5bd4661 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:14:57 +0100 Subject: [PATCH 11/18] fix:#82 slowed down sport update schedule --- backend/service/addSchedule.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/service/addSchedule.go b/backend/service/addSchedule.go index 0b62f39..b1263c5 100644 --- a/backend/service/addSchedule.go +++ b/backend/service/addSchedule.go @@ -28,8 +28,8 @@ func AddSchedules(app *pocketbase.PocketBase) { feed.ClearFeeds(app.Dao(), 6, time.RealClock{}) }) - // Every day at 2am update all Sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * *" - scheduler.MustAdd("fetchEvents", "0 2 * * *", func() { + // Every sunday at 2am fetch all sport events (5 segments - minute, hour, day, month, weekday) "0 2 * * 0" + scheduler.MustAdd("fetchEvents", "0 2 * * 0", func() { sport.FetchAndUpdateSportEvents(app) }) From 96873a326078a1665f007f96e8f1649a3d60f6a9 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:30:50 +0100 Subject: [PATCH 12/18] fix:#82 sport could be filtered by course sport --- backend/service/fetch/sport/sportFetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go index ddc7377..f3ea1f7 100644 --- a/backend/service/fetch/sport/sportFetcher.go +++ b/backend/service/fetch/sport/sportFetcher.go @@ -399,7 +399,7 @@ func fetchHtwkSportCourse(doc *goquery.Document) ([]model.SportEntry, error) { fullTitle := strings.TrimSpace(s.Find("h3").Text()) titleParts := strings.Split(fullTitle, "-") if len(titleParts) > 0 { - event.Title = "Sport: " + strings.TrimSpace(titleParts[0]) + event.Title = strings.TrimSpace(titleParts[0]) } if len(titleParts) > 2 { From 11779016988ae5d2541a0663ffc500ec501ecde1 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:37:57 +0100 Subject: [PATCH 13/18] fix:#82 updated readme sports endpoint --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 67fa014..4db9b2b 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,32 @@ Stay for this time on the page and wait for the response. http://127.0.0.1/api/fetch/events +After you fetched the events you can optional fetch sport events. +This will scrape the sport events from the HTWK website and store them in the database. +This will take a few seconds (2s-10s). + +http://127.0.0.1/api/fetch/sports + + ### View/Filter/Search in Admin UI If you want some easy first api endpoints and data views, you can use the Admin UI. + + +### Schedules + +In our project we used schedules to fetch data from HTWK and store it in the database. +So they get periodically executed and keep the database up to date. + +You can find the schedules in the following directory: + +``` +service/addSchedule.go +``` + +Currently, there are 3 schedules: + +- FetchGroupsSchedule +- FetchEventsSchedule +- FetchSportsSchedule + From b7bf1bc4099bc1fbbca52672738e0ec24249bdc5 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:39:06 +0100 Subject: [PATCH 14/18] doc:#82 updated openapi.yml --- backend/openapi.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/openapi.yml b/backend/openapi.yml index 69af1c5..c32b761 100644 --- a/backend/openapi.yml +++ b/backend/openapi.yml @@ -8,7 +8,7 @@ servers: - url: http://localhost:8090 description: Local server paths: - /api/fetchPlans: + /api/fetch/events: get: summary: Fetch Seminar Plans security: @@ -16,7 +16,7 @@ paths: responses: '200': description: Successful response - /api/fetchGroups: + /api/fetch/groups: get: summary: Fetch Seminar Groups security: @@ -24,6 +24,14 @@ paths: responses: '200': description: Successful response + /api/fetch/sports: + get: + summary: Fetch Sport Events from HTWK Leipzig + security: + - ApiKeyAuth: [ ] + responses: + '200': + description: Successful response /api/modules: delete: summary: Delete Module From 23b568338751fbe9c689ffc2789d5275c6801cd7 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:43:32 +0100 Subject: [PATCH 15/18] fix:#82 added type to title for unique identification in uuid --- backend/service/fetch/sport/sportFetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go index f3ea1f7..c3fe4a5 100644 --- a/backend/service/fetch/sport/sportFetcher.go +++ b/backend/service/fetch/sport/sportFetcher.go @@ -84,7 +84,7 @@ func formatEntriesToEvents(entries []model.SportEntry) []model.Event { end, _ := types.ParseDateTime(eventEnds[j].In(time.UTC)) var event = model.Event{ - UUID: uuid.NewSHA1(uuid.NameSpaceDNS, []byte(entry.Title+strconv.FormatInt(int64(i), 10))).String(), + UUID: uuid.NewSHA1(uuid.NameSpaceDNS, []byte(entry.Title+strconv.FormatInt(int64(i), 10)+entry.Details.Type)).String(), Day: toGermanWeekdayString(entry.Details.DateRange.Start.Weekday()), Week: strconv.Itoa(23), Start: start, From 7ddc227ee42b575678e4f86a2e08ca517b2ae385 Mon Sep 17 00:00:00 2001 From: masterelmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:11:06 +0100 Subject: [PATCH 16/18] fix:#82 added type to title --- backend/service/fetch/sport/sportFetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go index c3fe4a5..b70bbb0 100644 --- a/backend/service/fetch/sport/sportFetcher.go +++ b/backend/service/fetch/sport/sportFetcher.go @@ -89,7 +89,7 @@ func formatEntriesToEvents(entries []model.SportEntry) []model.Event { Week: strconv.Itoa(23), Start: start, End: end, - Name: entry.Title, + Name: entry.Title + " " + entry.Details.Type, EventType: entry.Details.Type, Prof: entry.Details.CourseLead.Name, Rooms: entry.Details.Location.Name, From cded924094dae40146e275e747c5e9ff7142bdcb Mon Sep 17 00:00:00 2001 From: masterElmar <18119527+masterElmar@users.noreply.github.com> Date: Wed, 13 Dec 2023 19:33:00 +0100 Subject: [PATCH 17/18] fix:#82 fetched internal sport course id for unique identification --- backend/model/sportFetcherModel.go | 1 + backend/service/fetch/sport/sportFetcher.go | 32 +++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/backend/model/sportFetcherModel.go b/backend/model/sportFetcherModel.go index 600f6ba..6b9da82 100644 --- a/backend/model/sportFetcherModel.go +++ b/backend/model/sportFetcherModel.go @@ -9,6 +9,7 @@ type SportEntry struct { Title string Details EventDetails AdditionalNote string + ID string } // EventDetails represents detailed information about the event. diff --git a/backend/service/fetch/sport/sportFetcher.go b/backend/service/fetch/sport/sportFetcher.go index b70bbb0..47ba277 100644 --- a/backend/service/fetch/sport/sportFetcher.go +++ b/backend/service/fetch/sport/sportFetcher.go @@ -76,7 +76,7 @@ func formatEntriesToEvents(entries []model.SportEntry) []model.Event { var events []model.Event - for i, entry := range entries { + for _, entry := range entries { eventStarts, eventEnds := getWeekEvents(entry.Details.DateRange.Start, entry.Details.DateRange.End, entry.Details.Cycle) for j := range eventStarts { @@ -84,12 +84,12 @@ func formatEntriesToEvents(entries []model.SportEntry) []model.Event { end, _ := types.ParseDateTime(eventEnds[j].In(time.UTC)) var event = model.Event{ - UUID: uuid.NewSHA1(uuid.NameSpaceDNS, []byte(entry.Title+strconv.FormatInt(int64(i), 10)+entry.Details.Type)).String(), + UUID: uuid.NewSHA1(uuid.NameSpaceDNS, []byte(entry.Title+entry.ID+entry.Details.Type)).String(), Day: toGermanWeekdayString(entry.Details.DateRange.Start.Weekday()), Week: strconv.Itoa(23), Start: start, End: end, - Name: entry.Title + " " + entry.Details.Type, + Name: entry.Title + " " + entry.Details.Type + " (" + entry.ID + ")", EventType: entry.Details.Type, Prof: entry.Details.CourseLead.Name, Rooms: entry.Details.Location.Name, @@ -387,6 +387,7 @@ func htmlRequest(url string) (*goquery.Document, error) { // May be improved in the future. func fetchHtwkSportCourse(doc *goquery.Document) ([]model.SportEntry, error) { var events []model.SportEntry + germanTime, _ := time.LoadLocation("Europe/Berlin") if doc.Find("h1").Text() == "Aktuelle Sportangebote" { return nil, errors.New("not a sport course page") @@ -406,6 +407,8 @@ func fetchHtwkSportCourse(doc *goquery.Document) ([]model.SportEntry, error) { details.Type = strings.TrimSpace(titleParts[len(titleParts)-1]) } + event.ID = parseEventID(fullTitle) + s.NextFiltered("table.eventDetails").Find("tr").Each(func(i int, s *goquery.Selection) { key := strings.TrimSpace(s.Find("td").First().Text()) value := strings.TrimSpace(s.Find("td").Last().Text()) @@ -414,8 +417,8 @@ func fetchHtwkSportCourse(doc *goquery.Document) ([]model.SportEntry, error) { case "Zeitraum": dates := strings.Split(value, "-") if len(dates) == 2 { - startDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[0])) - endDate, _ := time.Parse("02.01.2006", strings.TrimSpace(dates[1])) + startDate, _ := time.ParseInLocation("02.01.2006", strings.TrimSpace(dates[0]), germanTime) + endDate, _ := time.ParseInLocation("02.01.2006", strings.TrimSpace(dates[1]), germanTime) details.DateRange = model.DateRange{Start: startDate, End: endDate} } case "Zyklus": @@ -471,3 +474,22 @@ func fetchHtwkSportCourse(doc *goquery.Document) ([]model.SportEntry, error) { return events, nil } + +// parseEventID from fulltitle +// the event id is a number in the fulltitle thats not a time like HH:MM and shoudl be found after Nr. or Nr: +func parseEventID(fulltitle string) string { + var eventID string + var numberRegExp = regexp.MustCompile("[0-9]{1,4}") + var fulltitleParts = strings.Split(fulltitle, " ") + for i, part := range fulltitleParts { + if part == "Nr." || part == "Nr:" { + eventID = fulltitleParts[i+1] + break + } + } + if eventID == "" { + eventID = numberRegExp.FindString(fulltitle) + } + return eventID + +} From 4c15ccda62b441499744e052acbc8458a91fda7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:44:23 +0000 Subject: [PATCH 18/18] build(deps-dev): bump vite from 4.4.9 to 4.4.12 in /frontend Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.4.9 to 4.4.12. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.4.12/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.4.12/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- frontend/package-lock.json | 8 ++++---- frontend/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1a3f9e4..b27fcd7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -34,7 +34,7 @@ "sass": "^1.69.5", "sass-loader": "^13.3.2", "typescript": "^5.0.2", - "vite": "^4.4.5", + "vite": "^4.4.12", "vue-tsc": "^1.8.5" } }, @@ -3588,9 +3588,9 @@ "dev": true }, "node_modules/vite": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", - "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "version": "4.4.12", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.12.tgz", + "integrity": "sha512-KtPlUbWfxzGVul8Nut8Gw2Qe8sBzWY+8QVc5SL8iRFnpnrcoCaNlzO40c1R6hPmcdTwIPEDkq0Y9+27a5tVbdQ==", "dev": true, "dependencies": { "esbuild": "^0.18.10", diff --git a/frontend/package.json b/frontend/package.json index 3c1e442..b93816b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -37,7 +37,7 @@ "sass": "^1.69.5", "sass-loader": "^13.3.2", "typescript": "^5.0.2", - "vite": "^4.4.5", + "vite": "^4.4.12", "vue-tsc": "^1.8.5" } }