From 2ff37ea44105848f38a75f473b5e82734bbb0b92 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 24 Jun 2024 10:23:04 +0200 Subject: [PATCH] fix:#41 added error if html request isn't 200 --- .../service/events/eventService.go | 6 +- .../service/fetch/htmlDownloader.go | 15 ++++- .../service/fetch/htmlDownloader_test.go | 63 +++++++++++++++++++ .../fetch/v1/fetchSeminarEventService.go | 52 +++++++++++---- services/go.mod | 9 +-- services/go.sum | 12 ++-- 6 files changed, 125 insertions(+), 32 deletions(-) create mode 100644 services/data-manager/service/fetch/htmlDownloader_test.go diff --git a/services/data-manager/service/events/eventService.go b/services/data-manager/service/events/eventService.go index 383640c..afa318a 100644 --- a/services/data-manager/service/events/eventService.go +++ b/services/data-manager/service/events/eventService.go @@ -122,10 +122,12 @@ func DeleteAllEvents(app *pocketbase.PocketBase) error { // If the update was not successful, an error is returned func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Events, error) { - seminarGroup := v1.GetSeminarGroupEventsFromHTML(course) + seminarGroup, err := v1.GetSeminarGroupEventsFromHTML(course) + if err != nil { + return nil, err + } seminarGroup = v1.ClearEmptySeminarGroups(seminarGroup) - seminarGroup = v1.ReplaceEmptyEventNames(seminarGroup) //check if events in the seminarGroups Events are already in the database diff --git a/services/data-manager/service/fetch/htmlDownloader.go b/services/data-manager/service/fetch/htmlDownloader.go index ffe1cf9..c12fe3d 100644 --- a/services/data-manager/service/fetch/htmlDownloader.go +++ b/services/data-manager/service/fetch/htmlDownloader.go @@ -17,8 +17,10 @@ package fetch import ( + "errors" "fmt" "io" + "log/slog" "net/http" "time" ) @@ -26,18 +28,27 @@ import ( // getPlanHTML Get the HTML document from the specified URL func GetHTML(url string) (string, error) { - // Create HTTP client with timeout of 5 seconds client := http.Client{ Timeout: 30 * time.Second, } + return GetHTMLWithClient(url, &client) +} + +func GetHTMLWithClient(url string, client *http.Client) (string, error) { // Send GET request response, err := client.Get(url) if err != nil { - fmt.Printf("Error occurred while making the request: %s\n", err.Error()) + slog.Error("Error occurred while fetching the HTML document:", "error", err) return "", err } + + if response.StatusCode != 200 { + slog.Warn("While fetching the HTML document, the server responded with status code: ", "status", response.StatusCode) + return "", errors.New(fmt.Sprintf("Server responded with status code: %d", response.StatusCode)) + } + defer func(Body io.ReadCloser) { err := Body.Close() if err != nil { diff --git a/services/data-manager/service/fetch/htmlDownloader_test.go b/services/data-manager/service/fetch/htmlDownloader_test.go new file mode 100644 index 0000000..b6c9a30 --- /dev/null +++ b/services/data-manager/service/fetch/htmlDownloader_test.go @@ -0,0 +1,63 @@ +package fetch + +import ( + "github.com/jarcoal/httpmock" + "net/http" + "testing" +) + +func TestGetHTMLWithClient(t *testing.T) { + + client := &http.Client{} + httpmock.ActivateNonDefault(client) + + type args struct { + url string + statusCode int + method string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Test GetHTML with status code 200", + args: args{ + url: "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Listen;Studenten-Sets;name;1-1?template=sws_semgrp&weeks=1-65", + method: "GET", + statusCode: 200, + }, + want: "", + wantErr: false, + }, + { + name: "Test GetHTML with status code 404", + args: args{ + url: "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Lists;Studenten-Sets;name;1-1?template=sws_semgrp&weeks=1-65", + method: "GET", + statusCode: 404, + }, + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + httpmock.RegisterResponder(tt.args.method, tt.args.url, + httpmock.NewStringResponder(tt.args.statusCode, tt.want)) + got, err := GetHTMLWithClient(tt.args.url, client) + if (err != nil) != tt.wantErr { + t.Errorf("GetHTML() error = %v, wantNoErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetHTML() got = %v, want %v", got, tt.want) + } + }) + } + + httpmock.DeactivateAndReset() +} diff --git a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go index 4f3679d..801bd2a 100644 --- a/services/data-manager/service/fetch/v1/fetchSeminarEventService.go +++ b/services/data-manager/service/fetch/v1/fetchSeminarEventService.go @@ -50,25 +50,51 @@ func ClearEmptySeminarGroups(seminarGroup model.SeminarGroup) model.SeminarGroup return newSeminarGroup } -func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) model.SeminarGroup { - var seminarGroup model.SeminarGroup +func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) { + url := "https://stundenplan.htwk-leipzig.de/" + semester + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65" + result, err := fetch.GetHTML(url) + if err != nil { + slog.Error("Error occurred while fetching the HTML document:", "error", err) + return "", err + } + return result, nil +} - if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) { - ssUrl := "https://stundenplan.htwk-leipzig.de/" + string("ss") + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65" - result, getError := fetch.GetHTML(ssUrl) - if getError == nil { - seminarGroup = parseSeminarGroup(result) +func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) (model.SeminarGroup, error) { + var seminarGroup [2]model.SeminarGroup + var result string + var errSS error + var errWS error + + currentMonth := time.Now().Month() + if currentMonth >= 3 && currentMonth <= 10 { + result, errSS = fetchHTMLFromURL("ss", seminarGroupLabel) + if errSS == nil { + seminarGroup[0] = parseSeminarGroup(result) + } + + } + if currentMonth >= 9 || currentMonth <= 4 { + result, errWS = fetchHTMLFromURL("ws", seminarGroupLabel) + if errWS == nil { + seminarGroup[1] = parseSeminarGroup(result) } } - if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) { - wsUrl := "https://stundenplan.htwk-leipzig.de/" + string("ws") + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65" - result, getError := fetch.GetHTML(wsUrl) - if getError == nil { - seminarGroup = parseSeminarGroup(result) + if errSS != nil { + if errWS != nil { + return model.SeminarGroup{}, errWS + } else { + return seminarGroup[1], nil + } + } else { + if errWS != nil { + return seminarGroup[0], nil + } else { + seminarGroup[0].Events = append(seminarGroup[0].Events, seminarGroup[1].Events...) + return seminarGroup[0], nil } } - return seminarGroup } func SplitEventType(events []model.Event) ([]model.Event, error) { diff --git a/services/go.mod b/services/go.mod index 015e6a1..8d8ee37 100644 --- a/services/go.mod +++ b/services/go.mod @@ -4,13 +4,14 @@ go 1.21 require ( github.com/PuerkitoBio/goquery v1.9.2 + github.com/goccy/go-json v0.10.2 github.com/gofiber/fiber/v3 v3.0.0-beta.2 github.com/google/uuid v1.6.0 + github.com/jarcoal/httpmock v1.3.1 github.com/jordic/goics v0.0.0-20210404174824-5a0337b716a0 github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 github.com/pocketbase/dbx v1.10.1 github.com/pocketbase/pocketbase v0.22.12 - github.com/samber/slog-fiber v1.15.3 golang.org/x/net v0.26.0 google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.34.1 @@ -47,8 +48,6 @@ require ( github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/ganigeorgiev/fexpr v0.4.0 // indirect github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect - github.com/gofiber/fiber/v2 v2.52.1 // indirect github.com/gofiber/utils/v2 v2.0.0-beta.4 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -60,12 +59,10 @@ require ( github.com/klauspost/compress v1.17.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -74,8 +71,6 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/valyala/tcplisten v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect gocloud.dev v0.37.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/image v0.16.0 // indirect diff --git a/services/go.sum b/services/go.sum index 2d8a28a..50f1700 100644 --- a/services/go.sum +++ b/services/go.sum @@ -109,8 +109,6 @@ github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVH github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/gofiber/fiber/v2 v2.52.1 h1:1RoU2NS+b98o1L77sdl5mboGPiW+0Ypsi5oLmcYlgHI= -github.com/gofiber/fiber/v2 v2.52.1/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/gofiber/fiber/v3 v3.0.0-beta.2 h1:mVVgt8PTaHGup3NGl/+7U7nEoZaXJ5OComV4E+HpAao= github.com/gofiber/fiber/v3 v3.0.0-beta.2/go.mod h1:w7sdfTY0okjZ1oVH6rSOGvuACUIt0By1iK0HKUb3uqM= github.com/gofiber/utils/v2 v2.0.0-beta.4 h1:1gjbVFFwVwUb9arPcqiB6iEjHBwo7cHsyS41NeIW3co= @@ -162,6 +160,8 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= +github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -187,11 +187,11 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= +github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -206,13 +206,9 @@ github.com/pocketbase/pocketbase v0.22.12/go.mod h1:yY/3IGi1tUbcI6yGVFspAyKi/IDH github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/samber/slog-fiber v1.15.3 h1:RbfC0v2QPIEXoRdort2QxAsRG42LVaFTEgTNS/0GwRQ= -github.com/samber/slog-fiber v1.15.3/go.mod h1:I0b8eJ060SlpA65LXiqH7lZixUCkAPKiEGZqkT9QJOM= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=