From 2ff37ea44105848f38a75f473b5e82734bbb0b92 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 10:23:04 +0200 Subject: [PATCH 1/8] fix:#41 added error if html request isn't 200 --- .../service/events/eventService.go | 6 +- .../service/fetch/htmlDownloader.go | 15 ++++- .../service/fetch/htmlDownloader_test.go | 63 +++++++++++++++++++ .../fetch/v1/fetchSeminarEventService.go | 52 +++++++++++---- services/go.mod | 9 +-- services/go.sum | 12 ++-- 6 files changed, 125 insertions(+), 32 deletions(-) create mode 100644 services/data-manager/service/fetch/htmlDownloader_test.go diff --git a/services/data-manager/service/events/eventService.go b/services/data-manager/service/events/eventService.go index 383640c..afa318a 100644 --- a/services/data-manager/service/events/eventService.go +++ b/services/data-manager/service/events/eventService.go @@ -122,10 +122,12 @@ func DeleteAllEvents(app *pocketbase.PocketBase) error { // If the update was not successful, an error is returned func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Events, error) { - seminarGroup := v1.GetSeminarGroupEventsFromHTML(course) + seminarGroup, err := v1.GetSeminarGroupEventsFromHTML(course) + if err != nil { + return nil, err + } seminarGroup = v1.ClearEmptySeminarGroups(seminarGroup) - seminarGroup = v1.ReplaceEmptyEventNames(seminarGroup) //check if events in the seminarGroups Events are already in the database diff --git a/services/data-manager/service/fetch/htmlDownloader.go b/services/data-manager/service/fetch/htmlDownloader.go index ffe1cf9..c12fe3d 100644 --- a/services/data-manager/service/fetch/htmlDownloader.go +++ b/services/data-manager/service/fetch/htmlDownloader.go @@ -17,8 +17,10 @@ package fetch import ( + "errors" "fmt" "io" + "log/slog" "net/http" "time" ) @@ -26,18 +28,27 @@ import ( // getPlanHTML Get the HTML document from the specified URL func GetHTML(url string) (string, error) { - // Create HTTP client with timeout of 5 seconds client := http.Client{ Timeout: 30 * time.Second, } + return GetHTMLWithClient(url, &client) +} + +func GetHTMLWithClient(url string, client *http.Client) (string, error) { // Send GET request response, err := client.Get(url) if err != nil { - fmt.Printf("Error occurred while making the request: %s\n", err.Error()) + slog.Error("Error occurred while fetching the HTML document:", "error", err) return "", err } + + if response.StatusCode != 200 { + slog.Warn("While fetching the HTML document, the server responded with status code: ", "status", response.StatusCode) + return "", errors.New(fmt.Sprintf("Server responded with status code: %d", response.StatusCode)) + } + defer func(Body io.ReadCloser) { err := Body.Close() if err != nil { diff --git a/services/data-manager/service/fetch/htmlDownloader_test.go b/services/data-manager/service/fetch/htmlDownloader_test.go new file mode 100644 index 0000000..b6c9a30 --- /dev/null +++ b/services/data-manager/service/fetch/htmlDownloader_test.go @@ -0,0 +1,63 @@ +package fetch + +import ( + "github.com/jarcoal/httpmock" + "net/http" + "testing" +) + +func TestGetHTMLWithClient(t *testing.T) { + + client := &http.Client{} + httpmock.ActivateNonDefault(client) + + type args struct { + url string + statusCode int + method string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Test GetHTML with status code 200", + args: args{ + url: "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Listen;Studenten-Sets;name;1-1?template=sws_semgrp&weeks=1-65", + method: "GET", + statusCode: 200, + }, + want: "", + wantErr: false, + }, + { + name: "Test GetHTML with status code 404", + args: args{ + url: "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Lists;Studenten-Sets;name;1-1?template=sws_semgrp&weeks=1-65", + method: "GET", + statusCode: 404, + }, + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + httpmock.RegisterResponder(tt.args.method, tt.args.url, + httpmock.NewStringResponder(tt.args.statusCode, tt.want)) + got, err := GetHTMLWithClient(tt.args.url, client) + if (err != nil) != tt.wantErr { + t.Errorf("GetHTML() error = %v, wantNoErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetHTML() got = %v, want %v", got, tt.want) + } + }) + } + + httpmock.DeactivateAndReset() +} diff --git a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go index 4f3679d..801bd2a 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go @@ -50,25 +50,51 @@ func ClearEmptySeminarGroups(seminarGroup model.SeminarGroup) model.SeminarGroup return newSeminarGroup } -func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) model.SeminarGroup { - var seminarGroup model.SeminarGroup +func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) { + url := "https://stundenplan.htwk-leipzig.de/" + semester + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65" + result, err := fetch.GetHTML(url) + if err != nil { + slog.Error("Error occurred while fetching the HTML document:", "error", err) + return "", err + } + return result, nil +} - if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) { - ssUrl := "https://stundenplan.htwk-leipzig.de/" + string("ss") + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65" - result, getError := fetch.GetHTML(ssUrl) - if getError == nil { - seminarGroup = parseSeminarGroup(result) +func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) (model.SeminarGroup, error) { + var seminarGroup [2]model.SeminarGroup + var result string + var errSS error + var errWS error + + currentMonth := time.Now().Month() + if currentMonth >= 3 && currentMonth <= 10 { + result, errSS = fetchHTMLFromURL("ss", seminarGroupLabel) + if errSS == nil { + seminarGroup[0] = parseSeminarGroup(result) + } + + } + if currentMonth >= 9 || currentMonth <= 4 { + result, errWS = fetchHTMLFromURL("ws", seminarGroupLabel) + if errWS == nil { + seminarGroup[1] = parseSeminarGroup(result) } } - if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) { - wsUrl := "https://stundenplan.htwk-leipzig.de/" + string("ws") + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65" - result, getError := fetch.GetHTML(wsUrl) - if getError == nil { - seminarGroup = parseSeminarGroup(result) + if errSS != nil { + if errWS != nil { + return model.SeminarGroup{}, errWS + } else { + return seminarGroup[1], nil + } + } else { + if errWS != nil { + return seminarGroup[0], nil + } else { + seminarGroup[0].Events = append(seminarGroup[0].Events, seminarGroup[1].Events...) + return seminarGroup[0], nil } } - return seminarGroup } func SplitEventType(events []model.Event) ([]model.Event, error) { diff --git a/services/go.mod b/services/go.mod index 015e6a1..8d8ee37 100644 --- a/services/go.mod +++ b/services/go.mod @@ -4,13 +4,14 @@ go 1.21 require ( github.com/PuerkitoBio/goquery v1.9.2 + github.com/goccy/go-json v0.10.2 github.com/gofiber/fiber/v3 v3.0.0-beta.2 github.com/google/uuid v1.6.0 + github.com/jarcoal/httpmock v1.3.1 github.com/jordic/goics v0.0.0-20210404174824-5a0337b716a0 github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 github.com/pocketbase/dbx v1.10.1 github.com/pocketbase/pocketbase v0.22.12 - github.com/samber/slog-fiber v1.15.3 golang.org/x/net v0.26.0 google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.34.1 @@ -47,8 +48,6 @@ require ( github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/ganigeorgiev/fexpr v0.4.0 // indirect github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect - github.com/gofiber/fiber/v2 v2.52.1 // indirect github.com/gofiber/utils/v2 v2.0.0-beta.4 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -60,12 +59,10 @@ require ( github.com/klauspost/compress v1.17.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -74,8 +71,6 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/valyala/tcplisten v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect gocloud.dev v0.37.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/image v0.16.0 // indirect diff --git a/services/go.sum b/services/go.sum index 2d8a28a..50f1700 100644 --- a/services/go.sum +++ b/services/go.sum @@ -109,8 +109,6 @@ github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVH github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/gofiber/fiber/v2 v2.52.1 h1:1RoU2NS+b98o1L77sdl5mboGPiW+0Ypsi5oLmcYlgHI= -github.com/gofiber/fiber/v2 v2.52.1/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/gofiber/fiber/v3 v3.0.0-beta.2 h1:mVVgt8PTaHGup3NGl/+7U7nEoZaXJ5OComV4E+HpAao= github.com/gofiber/fiber/v3 v3.0.0-beta.2/go.mod h1:w7sdfTY0okjZ1oVH6rSOGvuACUIt0By1iK0HKUb3uqM= github.com/gofiber/utils/v2 v2.0.0-beta.4 h1:1gjbVFFwVwUb9arPcqiB6iEjHBwo7cHsyS41NeIW3co= @@ -162,6 +160,8 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= +github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -187,11 +187,11 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= +github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -206,13 +206,9 @@ github.com/pocketbase/pocketbase v0.22.12/go.mod h1:yY/3IGi1tUbcI6yGVFspAyKi/IDH github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/samber/slog-fiber v1.15.3 h1:RbfC0v2QPIEXoRdort2QxAsRG42LVaFTEgTNS/0GwRQ= -github.com/samber/slog-fiber v1.15.3/go.mod h1:I0b8eJ060SlpA65LXiqH7lZixUCkAPKiEGZqkT9QJOM= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= From 3eb3fe2b5e2f7487c3d185222a3a87f13c287c40 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 10:25:19 +0200 Subject: [PATCH 2/8] fix:#41 fixed linter --- services/data-manager/service/fetch/htmlDownloader.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/data-manager/service/fetch/htmlDownloader.go b/services/data-manager/service/fetch/htmlDownloader.go index c12fe3d..f7936cf 100644 --- a/services/data-manager/service/fetch/htmlDownloader.go +++ b/services/data-manager/service/fetch/htmlDownloader.go @@ -17,7 +17,6 @@ package fetch import ( - "errors" "fmt" "io" "log/slog" @@ -46,7 +45,7 @@ func GetHTMLWithClient(url string, client *http.Client) (string, error) { if response.StatusCode != 200 { slog.Warn("While fetching the HTML document, the server responded with status code: ", "status", response.StatusCode) - return "", errors.New(fmt.Sprintf("Server responded with status code: %d", response.StatusCode)) + return "", fmt.Errorf("Server responded with status code: %d", response.StatusCode) } defer func(Body io.ReadCloser) { From 381e5c6d2b897d76bacea5ee13bcd3f0bdcd504c Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 10:25:34 +0200 Subject: [PATCH 3/8] fix:#41 fixed lang --- services/data-manager/service/fetch/htmlDownloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/data-manager/service/fetch/htmlDownloader.go b/services/data-manager/service/fetch/htmlDownloader.go index f7936cf..2c46c67 100644 --- a/services/data-manager/service/fetch/htmlDownloader.go +++ b/services/data-manager/service/fetch/htmlDownloader.go @@ -45,7 +45,7 @@ func GetHTMLWithClient(url string, client *http.Client) (string, error) { if response.StatusCode != 200 { slog.Warn("While fetching the HTML document, the server responded with status code: ", "status", response.StatusCode) - return "", fmt.Errorf("Server responded with status code: %d", response.StatusCode) + return "", fmt.Errorf("server responded with status code: %d", response.StatusCode) } defer func(Body io.ReadCloser) { From cb8de60d210032494b04810509b12611302b6aa8 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 10:50:18 +0200 Subject: [PATCH 4/8] fix:#41 refactored fetcher functions --- .../fetch/v1/fetchSeminarEventService.go | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go index 801bd2a..e2a8780 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go @@ -62,38 +62,47 @@ func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) { func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) (model.SeminarGroup, error) { var seminarGroup [2]model.SeminarGroup - var result string - var errSS error - var errWS error + var errSS, errWS error currentMonth := time.Now().Month() - if currentMonth >= 3 && currentMonth <= 10 { - result, errSS = fetchHTMLFromURL("ss", seminarGroupLabel) - if errSS == nil { - seminarGroup[0] = parseSeminarGroup(result) - } + if isSummerSemester(currentMonth) { + seminarGroup[0], errSS = fetchAndParse("ss", seminarGroupLabel) } - if currentMonth >= 9 || currentMonth <= 4 { - result, errWS = fetchHTMLFromURL("ws", seminarGroupLabel) - if errWS == nil { - seminarGroup[1] = parseSeminarGroup(result) - } + if isWinterSemester(currentMonth) { + seminarGroup[1], errWS = fetchAndParse("ws", seminarGroupLabel) } - if errSS != nil { - if errWS != nil { - return model.SeminarGroup{}, errWS - } else { - return seminarGroup[1], nil - } - } else { - if errWS != nil { - return seminarGroup[0], nil - } else { - seminarGroup[0].Events = append(seminarGroup[0].Events, seminarGroup[1].Events...) - return seminarGroup[0], nil - } + return checkForSuccessfulFetch(errSS, errWS, seminarGroup) +} + +func isSummerSemester(month time.Month) bool { + return month >= 3 && month <= 10 +} + +func isWinterSemester(month time.Month) bool { + return month >= 9 || month <= 4 +} + +func fetchAndParse(season, label string) (model.SeminarGroup, error) { + result, err := fetchHTMLFromURL(season, label) + if err != nil { + return model.SeminarGroup{}, err + } + return parseSeminarGroup(result), nil +} + +func checkForSuccessfulFetch(errSS error, errWS error, seminarGroup [2]model.SeminarGroup) (model.SeminarGroup, error) { + switch { + case errSS != nil && errWS != nil: + return model.SeminarGroup{}, errWS + case errSS != nil: + return seminarGroup[1], nil + case errWS != nil: + return seminarGroup[0], nil + default: + seminarGroup[0].Events = append(seminarGroup[0].Events, seminarGroup[1].Events...) + return seminarGroup[0], nil } } From ff2fa1b67dffa76ac741ba0b7e5e6c495710e99f Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 12:18:48 +0200 Subject: [PATCH 5/8] fix:#41 fixed sonarqube hotspots --- docker-compose.yml | 1 + services/data-manager/Dockerfile | 17 ++- services/data-manager/main.go | 8 +- services/data-manager/main_test.go | 42 +++++++ .../fetch/v1/fetchSeminarEventService_test.go | 119 ++++++++++++++++++ .../data-manager/sonar-project.properties | 1 + 6 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 services/data-manager/main_test.go diff --git a/docker-compose.yml b/docker-compose.yml index 922610f..7ec3d59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ services: volumes: - pb_data:/htwkalender-data-manager/data # for production with volume # - ./data-manager:/htwkalender/data # for development with bind mount from project directory + user: "ical" htwkalender-ical: build: diff --git a/services/data-manager/Dockerfile b/services/data-manager/Dockerfile index f1bcae8..9e52f5d 100644 --- a/services/data-manager/Dockerfile +++ b/services/data-manager/Dockerfile @@ -29,19 +29,20 @@ COPY common/. ./common RUN CGO_ENABLED=1 GOOS=linux go build -o /htwkalender-data-manager data-manager/main.go # production stage -FROM alpine:latest AS prod +FROM alpine:3.20.1 AS prod WORKDIR /htwkalender-data-manager ARG USER=ical -RUN adduser -Ds /bin/sh $USER && \ - chown $USER:$USER ./ +RUN adduser -Ds /bin/sh "$USER" && \ + chown "$USER":"$USER" ./ USER $USER RUN mkdir -p data # copies executable from build container -COPY --chown=$USER:$USER --from=build /htwkalender-data-manager ./ +COPY --chown=$USER:$USER --chmod=644 --from=build /htwkalender-data-manager ./ +RUN chmod +x main # Expose port 8090 to the outside world EXPOSE 8090 @@ -54,6 +55,12 @@ FROM golang:1.21.6 AS dev # Set the Current Working Directory inside the container WORKDIR /htwkalender-data-manager +ARG USER=ical +RUN adduser "$USER" && \ + chown "$USER":"$USER" ./ \ + && mkdir -p /htwkalender-data-manager/data \ + && chown "$USER":"$USER" /htwkalender-data-manager/data + # Copy go mod and sum files COPY go.mod go.sum ./ RUN go mod download @@ -68,5 +75,7 @@ RUN CGO_ENABLED=1 GOOS=linux go build -o /htwkalender-data-manager data-manager/ # Expose port 8091 to the outside world EXPOSE 8091 +USER $USER + # Entry point ENTRYPOINT ["./main", "serve"] \ No newline at end of file diff --git a/services/data-manager/main.go b/services/data-manager/main.go index c49c157..1e50b50 100644 --- a/services/data-manager/main.go +++ b/services/data-manager/main.go @@ -27,7 +27,7 @@ import ( "strings" ) -func main() { +func setupApp() *pocketbase.PocketBase { app := pocketbase.New() // loosely check if it was executed using "go run" @@ -44,8 +44,12 @@ func main() { service.AddRoutes(app) service.AddSchedules(app) + return app +} + +func main() { + app := setupApp() if err := app.Start(); err != nil { slog.Error("Failed to start app: ", "error", err) } - } diff --git a/services/data-manager/main_test.go b/services/data-manager/main_test.go new file mode 100644 index 0000000..773d375 --- /dev/null +++ b/services/data-manager/main_test.go @@ -0,0 +1,42 @@ +//Calendar implementation for the HTWK Leipzig timetable. Evaluation and display of the individual dates in iCal format. +//Copyright (C) 2024 HTWKalender support@htwkalender.de + +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. + +//This program is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. + +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . + +package main + +import ( + "testing" +) + +func Test_setupApp(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "Test setupApp", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + app := setupApp() + go func() { + if err := app.Start(); err != nil { + t.Errorf("Failed to start app: %v", err) + return + } + }() + }) + } +} diff --git a/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go b/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go index 10b55cd..8f88b4a 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go @@ -501,3 +501,122 @@ func Test_replaceTimeForDate(t *testing.T) { }) } } + +func Test_isSummerSemester(t *testing.T) { + type args struct { + month time.Month + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Test Summer March", + args: args{ + month: time.March, + }, + want: true, + }, + { + name: "Test Summer September", + args: args{ + month: time.September, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isSummerSemester(tt.args.month); got != tt.want { + t.Errorf("isSummerSemester() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_isWinterSemester(t *testing.T) { + type args struct { + month time.Month + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Test Winter March", + args: args{ + month: time.March, + }, + want: true, + }, + { + name: "Test Winter September", + args: args{ + month: time.September, + }, + want: true, + }, + { + name: "Test Winter November", + args: args{ + month: time.November, + }, + want: true, + }, + { + name: "Test Winter February", + args: args{ + month: time.February, + }, + want: true, + }, + { + name: "Test Winter June", + args: args{ + month: time.June, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isWinterSemester(tt.args.month); got != tt.want { + t.Errorf("isWinterSemester() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_parseSeminarGroup(t *testing.T) { + type args struct { + result string + } + tests := []struct { + name string + args args + want model.SeminarGroup + }{ + { + name: "Test 1", + args: args{ + result: "B435 SBB (wpf) & B348 BIB (pf) 5. FS", + }, + want: model.SeminarGroup{ + Events: []model.Event{ + { + Name: "B435 SBB (wpf) & B348 BIB (pf) 5. FS", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parseSeminarGroup(tt.args.result); !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseSeminarGroup() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/services/data-manager/sonar-project.properties b/services/data-manager/sonar-project.properties index a0ad568..c9b68d8 100644 --- a/services/data-manager/sonar-project.properties +++ b/services/data-manager/sonar-project.properties @@ -1,2 +1,3 @@ sonar.projectKey=HTWKalender sonar.qualitygate.wait=true +sonar.exclusions=migrations/** \ No newline at end of file From 3e2b3c37203645f4aca95fefd6d14b5217da463b Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 15:13:57 +0200 Subject: [PATCH 6/8] fix:#41 fixed missing test case --- .../fetch/v1/fetchSeminarEventService_test.go | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go b/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go index 8f88b4a..8398adb 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go @@ -523,7 +523,7 @@ func Test_isSummerSemester(t *testing.T) { args: args{ month: time.September, }, - want: false, + want: true, }, } for _, tt := range tests { @@ -588,35 +588,3 @@ func Test_isWinterSemester(t *testing.T) { }) } } - -func Test_parseSeminarGroup(t *testing.T) { - type args struct { - result string - } - tests := []struct { - name string - args args - want model.SeminarGroup - }{ - { - name: "Test 1", - args: args{ - result: "B435 SBB (wpf) & B348 BIB (pf) 5. FS", - }, - want: model.SeminarGroup{ - Events: []model.Event{ - { - Name: "B435 SBB (wpf) & B348 BIB (pf) 5. FS", - }, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := parseSeminarGroup(tt.args.result); !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseSeminarGroup() = %v, want %v", got, tt.want) - } - }) - } -} From bfb05b6704f7751881cf6eccf3a8c496fc8f1554 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 15:29:58 +0200 Subject: [PATCH 7/8] fix:#41 fixed naming issue sonarqube --- services/data-manager/main_test.go | 2 +- .../data-manager/model/eventModel_test.go | 16 +++++++-------- .../service/date/dateFormat_test.go | 2 +- .../data-manager/service/db/dbEvents_test.go | 2 +- .../service/events/courseService_test.go | 2 +- .../service/fetch/sport/sportFetcher_test.go | 2 +- .../fetch/v1/fetchSeminarEventService_test.go | 20 +++++++++---------- .../fetch/v1/fetchSeminarGroupService_test.go | 2 +- .../service/fetch/v2/fetcher_test.go | 2 +- .../service/functions/semester_test.go | 2 +- .../service/room/roomService_test.go | 6 +++--- .../ical/service/functions/semester_test.go | 2 +- 12 files changed, 30 insertions(+), 30 deletions(-) diff --git a/services/data-manager/main_test.go b/services/data-manager/main_test.go index 773d375..2275da5 100644 --- a/services/data-manager/main_test.go +++ b/services/data-manager/main_test.go @@ -20,7 +20,7 @@ import ( "testing" ) -func Test_setupApp(t *testing.T) { +func TestSetupApp(t *testing.T) { tests := []struct { name string }{ diff --git a/services/data-manager/model/eventModel_test.go b/services/data-manager/model/eventModel_test.go index c153f23..2749143 100644 --- a/services/data-manager/model/eventModel_test.go +++ b/services/data-manager/model/eventModel_test.go @@ -24,7 +24,7 @@ import ( "github.com/pocketbase/pocketbase/tools/types" ) -func TestEvents_Contains(t *testing.T) { +func TestEventsContains(t *testing.T) { specificTime, _ := types.ParseDateTime("2020-01-01 12:00:00.000Z") type args struct { @@ -70,7 +70,7 @@ func TestEvents_Contains(t *testing.T) { } } -func TestEvent_Equals(t *testing.T) { +func TestEventEquals(t *testing.T) { specificTime, _ := types.ParseDateTime("2020-01-01 12:00:00.000Z") type fields struct { @@ -148,7 +148,7 @@ func TestEvent_Equals(t *testing.T) { } } -func TestEvent_AnonymizeEvent(t *testing.T) { +func TestEventAnonymizeEvent(t *testing.T) { type fields struct { UUID string Day string @@ -218,7 +218,7 @@ func TestEvent_AnonymizeEvent(t *testing.T) { } } -func TestEvent_GetName(t *testing.T) { +func TestEventGetName(t *testing.T) { type fields struct { UUID string Day string @@ -278,7 +278,7 @@ func TestEvent_GetName(t *testing.T) { } } -func TestEvent_SetCourse(t *testing.T) { +func TestEventSetCourse(t *testing.T) { type fields struct { UUID string Day string @@ -338,7 +338,7 @@ func TestEvent_SetCourse(t *testing.T) { } } -func TestEvent_SetName(t *testing.T) { +func TestEventSetName(t *testing.T) { type fields struct { UUID string Day string @@ -398,7 +398,7 @@ func TestEvent_SetName(t *testing.T) { } } -func TestEvent_TableName(t *testing.T) { +func TestEventTableName(t *testing.T) { type fields struct { UUID string Day string @@ -453,7 +453,7 @@ func TestEvent_TableName(t *testing.T) { } } -func TestEvents_Contains1(t *testing.T) { +func TestEventsContains1(t *testing.T) { type args struct { event Event } diff --git a/services/data-manager/service/date/dateFormat_test.go b/services/data-manager/service/date/dateFormat_test.go index 4ab97be..f43a77b 100644 --- a/services/data-manager/service/date/dateFormat_test.go +++ b/services/data-manager/service/date/dateFormat_test.go @@ -23,7 +23,7 @@ import ( _ "time/tzdata" ) -func Test_getDateFromWeekNumber(t *testing.T) { +func TestGetDateFromWeekNumber(t *testing.T) { europeTime, _ := time.LoadLocation("Europe/Berlin") type args struct { diff --git a/services/data-manager/service/db/dbEvents_test.go b/services/data-manager/service/db/dbEvents_test.go index 8708394..7347062 100644 --- a/services/data-manager/service/db/dbEvents_test.go +++ b/services/data-manager/service/db/dbEvents_test.go @@ -22,7 +22,7 @@ import ( "testing" ) -func Test_buildIcalQueryForModules(t *testing.T) { +func TestBuildIcalQueryForModules(t *testing.T) { type args struct { modules []string } diff --git a/services/data-manager/service/events/courseService_test.go b/services/data-manager/service/events/courseService_test.go index 3d5e033..724f0d1 100644 --- a/services/data-manager/service/events/courseService_test.go +++ b/services/data-manager/service/events/courseService_test.go @@ -21,7 +21,7 @@ import ( "testing" ) -func Test_removeEmptyCourses(t *testing.T) { +func TestRemoveEmptyCourses(t *testing.T) { type args struct { courses []string } diff --git a/services/data-manager/service/fetch/sport/sportFetcher_test.go b/services/data-manager/service/fetch/sport/sportFetcher_test.go index 0f4c06d..bd0d512 100644 --- a/services/data-manager/service/fetch/sport/sportFetcher_test.go +++ b/services/data-manager/service/fetch/sport/sportFetcher_test.go @@ -21,7 +21,7 @@ import ( "testing" ) -func Test_splitByCommaWithTime(t *testing.T) { +func TestSplitByCommaWithTime(t *testing.T) { type args struct { input string } diff --git a/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go b/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go index 8398adb..f625705 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarEventService_test.go @@ -25,7 +25,7 @@ import ( "time" ) -func Test_extractSemesterAndYear(t *testing.T) { +func TestExtractSemesterAndYear(t *testing.T) { type args struct { semesterString string } @@ -73,7 +73,7 @@ func Test_extractSemesterAndYear(t *testing.T) { } } -func Test_replaceEmptyEventNames(t *testing.T) { +func TestReplaceEmptyEventNames(t *testing.T) { type args struct { group model.SeminarGroup } @@ -130,7 +130,7 @@ func Test_replaceEmptyEventNames(t *testing.T) { } } -func Test_splitEventType(t *testing.T) { +func TestSplitEventType(t *testing.T) { type args struct { events []model.Event } @@ -220,7 +220,7 @@ func Test_splitEventType(t *testing.T) { } } -func Test_generateUUIDs(t *testing.T) { +func TestGenerateUUIDs(t *testing.T) { type args struct { events []model.Event course string @@ -274,7 +274,7 @@ func Test_generateUUIDs(t *testing.T) { } } -func Test_createTimeFromHourAndMinuteString(t *testing.T) { +func TestCreateTimeFromHourAndMinuteString(t *testing.T) { type args struct { tableTime string } @@ -314,7 +314,7 @@ func Test_createTimeFromHourAndMinuteString(t *testing.T) { } } -func Test_replaceTimeInDate(t *testing.T) { +func TestReplaceTimeInDate(t *testing.T) { type args struct { date time.Time time time.Time @@ -350,7 +350,7 @@ func Test_replaceTimeInDate(t *testing.T) { } } -func Test_convertWeeksToDates(t *testing.T) { +func TestConvertWeeksToDates(t *testing.T) { type args struct { events []model.Event semester string @@ -450,7 +450,7 @@ func Test_convertWeeksToDates(t *testing.T) { } } -func Test_replaceTimeForDate(t *testing.T) { +func TestReplaceTimeForDate(t *testing.T) { type args struct { date time.Time replacementTime time.Time @@ -502,7 +502,7 @@ func Test_replaceTimeForDate(t *testing.T) { } } -func Test_isSummerSemester(t *testing.T) { +func TestIsSummerSemester(t *testing.T) { type args struct { month time.Month } @@ -535,7 +535,7 @@ func Test_isSummerSemester(t *testing.T) { } } -func Test_isWinterSemester(t *testing.T) { +func TestIsWinterSemester(t *testing.T) { type args struct { month time.Month } diff --git a/services/data-manager/service/fetch/v1/fetchSeminarGroupService_test.go b/services/data-manager/service/fetch/v1/fetchSeminarGroupService_test.go index 6c2cfbe..f75c3a0 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarGroupService_test.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarGroupService_test.go @@ -21,7 +21,7 @@ import ( "testing" ) -func Test_contains(t *testing.T) { +func TestContains(t *testing.T) { type args struct { groups []model.SeminarGroup group model.SeminarGroup diff --git a/services/data-manager/service/fetch/v2/fetcher_test.go b/services/data-manager/service/fetch/v2/fetcher_test.go index 0a3e2ad..a9f901a 100644 --- a/services/data-manager/service/fetch/v2/fetcher_test.go +++ b/services/data-manager/service/fetch/v2/fetcher_test.go @@ -22,7 +22,7 @@ import ( "testing" ) -func Test_switchNameAndNotesForExam(t *testing.T) { +func TestSwitchNameAndNotesForExam(t *testing.T) { type args struct { events []model.Event } diff --git a/services/data-manager/service/functions/semester_test.go b/services/data-manager/service/functions/semester_test.go index 35a8f9c..d075825 100644 --- a/services/data-manager/service/functions/semester_test.go +++ b/services/data-manager/service/functions/semester_test.go @@ -7,7 +7,7 @@ import ( "time" ) -func Test_calculateSemesterList(t *testing.T) { +func TestCalculateSemesterList(t *testing.T) { type args struct { clock mockTime.Clock } diff --git a/services/data-manager/service/room/roomService_test.go b/services/data-manager/service/room/roomService_test.go index b4bf733..3cade0e 100644 --- a/services/data-manager/service/room/roomService_test.go +++ b/services/data-manager/service/room/roomService_test.go @@ -23,7 +23,7 @@ import ( "testing" ) -func Test_anonymizeRooms(t *testing.T) { +func TestAnonymizeRooms(t *testing.T) { type args struct { events []model.Event } @@ -139,7 +139,7 @@ func Test_anonymizeRooms(t *testing.T) { } } -func Test_isRoomInSchedule(t *testing.T) { +func TestIsRoomInSchedule(t *testing.T) { type args struct { room string schedule []model.Event @@ -245,7 +245,7 @@ func Test_isRoomInSchedule(t *testing.T) { } } -func Test_getFreeRooms(t *testing.T) { +func TestGetFreeRooms(t *testing.T) { type args struct { rooms []string schedule []model.Event diff --git a/services/ical/service/functions/semester_test.go b/services/ical/service/functions/semester_test.go index 0f46f11..9595ede 100644 --- a/services/ical/service/functions/semester_test.go +++ b/services/ical/service/functions/semester_test.go @@ -7,7 +7,7 @@ import ( "time" ) -func Test_calculateSemesterList(t *testing.T) { +func TestCalculateSemesterList(t *testing.T) { type args struct { clock mockTime.Clock } From a65a57bec68fd98b323f96854253770f7083de28 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 15:34:24 +0200 Subject: [PATCH 8/8] fix:#41 refactored function --- .../service/fetch/v2/eventParser.go | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/services/data-manager/service/fetch/v2/eventParser.go b/services/data-manager/service/fetch/v2/eventParser.go index 3482bd7..8572e3d 100644 --- a/services/data-manager/service/fetch/v2/eventParser.go +++ b/services/data-manager/service/fetch/v2/eventParser.go @@ -28,40 +28,48 @@ import ( func toEvents(tables [][]*html.Node, days []string) []model.Event { var events []model.Event - for table := range tables { - for row := range tables[table] { + for tableIndex, table := range tables { + day := days[tableIndex] - tableData := findTableData(tables[table][row]) - if len(tableData) > 0 { - start, _ := types.ParseDateTime(date.CreateTimeFromHourAndMinuteString(getTextContent(tableData[1]))) - end, _ := types.ParseDateTime(date.CreateTimeFromHourAndMinuteString(getTextContent(tableData[2]))) - - courses := getTextContent(tableData[7]) - name := getTextContent(tableData[3]) - if functions.OnlyWhitespace(name) { - name = "Sonderveranstaltung" - } - - if len(courses) > 0 { - for _, course := range strings.Split(courses, " ") { - events = append(events, model.Event{ - Day: days[table], - Week: getTextContent(tableData[0]), - Start: start, - End: end, - Name: name, - EventType: getTextContent(tableData[4]), - Notes: getTextContent(tableData[5]), - Prof: getTextContent(tableData[6]), - Rooms: getTextContent(tableData[8]), - BookedAt: getTextContent(tableData[10]), - Course: strings.TrimSpace(course), - }) - } - } + for _, row := range table { + tableData := findTableData(row) + if len(tableData) == 0 { + continue } + events = append(events, createEventsFromTableData(tableData, day)...) } - } return events } + +func createEventsFromTableData(tableData []*html.Node, day string) []model.Event { + var events []model.Event + + start, _ := types.ParseDateTime(date.CreateTimeFromHourAndMinuteString(getTextContent(tableData[1]))) + end, _ := types.ParseDateTime(date.CreateTimeFromHourAndMinuteString(getTextContent(tableData[2]))) + name := getTextContent(tableData[3]) + if functions.OnlyWhitespace(name) { + name = "Sonderveranstaltung" + } + + courses := getTextContent(tableData[7]) + if len(courses) > 0 { + for _, course := range strings.Split(courses, " ") { + events = append(events, model.Event{ + Day: day, + Week: getTextContent(tableData[0]), + Start: start, + End: end, + Name: name, + EventType: getTextContent(tableData[4]), + Notes: getTextContent(tableData[5]), + Prof: getTextContent(tableData[6]), + Rooms: getTextContent(tableData[8]), + BookedAt: getTextContent(tableData[10]), + Course: strings.TrimSpace(course), + }) + } + } + + return events +}