mirror of
https://gitlab.dit.htwk-leipzig.de/htwk-software/htwkalender.git
synced 2025-07-16 17:48:49 +02:00
fix:#41 added error if html request isn't 200
This commit is contained in:
Elmar Kresse
@ -122,10 +122,12 @@ func DeleteAllEvents(app *pocketbase.PocketBase) error {
|
||||
// If the update was not successful, an error is returned
|
||||
func UpdateModulesForCourse(app *pocketbase.PocketBase, course string) (model.Events, error) {
|
||||
|
||||
seminarGroup := v1.GetSeminarGroupEventsFromHTML(course)
|
||||
seminarGroup, err := v1.GetSeminarGroupEventsFromHTML(course)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
seminarGroup = v1.ClearEmptySeminarGroups(seminarGroup)
|
||||
|
||||
seminarGroup = v1.ReplaceEmptyEventNames(seminarGroup)
|
||||
|
||||
//check if events in the seminarGroups Events are already in the database
|
||||
|
||||
@ -17,8 +17,10 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
@ -26,18 +28,27 @@ import (
|
||||
// getPlanHTML Get the HTML document from the specified URL
|
||||
|
||||
func GetHTML(url string) (string, error) {
|
||||
|
||||
// Create HTTP client with timeout of 5 seconds
|
||||
client := http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
return GetHTMLWithClient(url, &client)
|
||||
}
|
||||
|
||||
func GetHTMLWithClient(url string, client *http.Client) (string, error) {
|
||||
|
||||
// Send GET request
|
||||
response, err := client.Get(url)
|
||||
if err != nil {
|
||||
fmt.Printf("Error occurred while making the request: %s\n", err.Error())
|
||||
slog.Error("Error occurred while fetching the HTML document:", "error", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
if response.StatusCode != 200 {
|
||||
slog.Warn("While fetching the HTML document, the server responded with status code: ", "status", response.StatusCode)
|
||||
return "", errors.New(fmt.Sprintf("Server responded with status code: %d", response.StatusCode))
|
||||
}
|
||||
|
||||
defer func(Body io.ReadCloser) {
|
||||
err := Body.Close()
|
||||
if err != nil {
|
||||
|
||||
63
services/data-manager/service/fetch/htmlDownloader_test.go
Normal file
63
services/data-manager/service/fetch/htmlDownloader_test.go
Normal file
@ -0,0 +1,63 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"github.com/jarcoal/httpmock"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetHTMLWithClient(t *testing.T) {
|
||||
|
||||
client := &http.Client{}
|
||||
httpmock.ActivateNonDefault(client)
|
||||
|
||||
type args struct {
|
||||
url string
|
||||
statusCode int
|
||||
method string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test GetHTML with status code 200",
|
||||
args: args{
|
||||
url: "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Listen;Studenten-Sets;name;1-1?template=sws_semgrp&weeks=1-65",
|
||||
method: "GET",
|
||||
statusCode: 200,
|
||||
},
|
||||
want: "",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test GetHTML with status code 404",
|
||||
args: args{
|
||||
url: "https://stundenplan.htwk-leipzig.de/ss/Berichte/Text-Lists;Studenten-Sets;name;1-1?template=sws_semgrp&weeks=1-65",
|
||||
method: "GET",
|
||||
statusCode: 404,
|
||||
},
|
||||
want: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
httpmock.RegisterResponder(tt.args.method, tt.args.url,
|
||||
httpmock.NewStringResponder(tt.args.statusCode, tt.want))
|
||||
got, err := GetHTMLWithClient(tt.args.url, client)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("GetHTML() error = %v, wantNoErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("GetHTML() got = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
httpmock.DeactivateAndReset()
|
||||
}
|
||||
@ -50,25 +50,51 @@ func ClearEmptySeminarGroups(seminarGroup model.SeminarGroup) model.SeminarGroup
|
||||
return newSeminarGroup
|
||||
}
|
||||
|
||||
func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) model.SeminarGroup {
|
||||
var seminarGroup model.SeminarGroup
|
||||
func fetchHTMLFromURL(semester, seminarGroupLabel string) (string, error) {
|
||||
url := "https://stundenplan.htwk-leipzig.de/" + semester + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65"
|
||||
result, err := fetch.GetHTML(url)
|
||||
if err != nil {
|
||||
slog.Error("Error occurred while fetching the HTML document:", "error", err)
|
||||
return "", err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
if (time.Now().Month() >= 3) && (time.Now().Month() <= 10) {
|
||||
ssUrl := "https://stundenplan.htwk-leipzig.de/" + string("ss") + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65"
|
||||
result, getError := fetch.GetHTML(ssUrl)
|
||||
if getError == nil {
|
||||
seminarGroup = parseSeminarGroup(result)
|
||||
func GetSeminarGroupEventsFromHTML(seminarGroupLabel string) (model.SeminarGroup, error) {
|
||||
var seminarGroup [2]model.SeminarGroup
|
||||
var result string
|
||||
var errSS error
|
||||
var errWS error
|
||||
|
||||
currentMonth := time.Now().Month()
|
||||
if currentMonth >= 3 && currentMonth <= 10 {
|
||||
result, errSS = fetchHTMLFromURL("ss", seminarGroupLabel)
|
||||
if errSS == nil {
|
||||
seminarGroup[0] = parseSeminarGroup(result)
|
||||
}
|
||||
|
||||
}
|
||||
if currentMonth >= 9 || currentMonth <= 4 {
|
||||
result, errWS = fetchHTMLFromURL("ws", seminarGroupLabel)
|
||||
if errWS == nil {
|
||||
seminarGroup[1] = parseSeminarGroup(result)
|
||||
}
|
||||
}
|
||||
|
||||
if (time.Now().Month() >= 9) || (time.Now().Month() <= 4) {
|
||||
wsUrl := "https://stundenplan.htwk-leipzig.de/" + string("ws") + "/Berichte/Text-Listen;Studenten-Sets;name;" + seminarGroupLabel + "?template=sws_semgrp&weeks=1-65"
|
||||
result, getError := fetch.GetHTML(wsUrl)
|
||||
if getError == nil {
|
||||
seminarGroup = parseSeminarGroup(result)
|
||||
if errSS != nil {
|
||||
if errWS != nil {
|
||||
return model.SeminarGroup{}, errWS
|
||||
} else {
|
||||
return seminarGroup[1], nil
|
||||
}
|
||||
} else {
|
||||
if errWS != nil {
|
||||
return seminarGroup[0], nil
|
||||
} else {
|
||||
seminarGroup[0].Events = append(seminarGroup[0].Events, seminarGroup[1].Events...)
|
||||
return seminarGroup[0], nil
|
||||
}
|
||||
}
|
||||
return seminarGroup
|
||||
}
|
||||
|
||||
func SplitEventType(events []model.Event) ([]model.Event, error) {
|
||||
|
||||
@ -4,13 +4,14 @@ go 1.21
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.2
|
||||
github.com/goccy/go-json v0.10.2
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.2
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jarcoal/httpmock v1.3.1
|
||||
github.com/jordic/goics v0.0.0-20210404174824-5a0337b716a0
|
||||
github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61
|
||||
github.com/pocketbase/dbx v1.10.1
|
||||
github.com/pocketbase/pocketbase v0.22.12
|
||||
github.com/samber/slog-fiber v1.15.3
|
||||
golang.org/x/net v0.26.0
|
||||
google.golang.org/grpc v1.63.2
|
||||
google.golang.org/protobuf v1.34.1
|
||||
@ -47,8 +48,6 @@ require (
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/ganigeorgiev/fexpr v0.4.0 // indirect
|
||||
github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/gofiber/fiber/v2 v2.52.1 // indirect
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.4 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
@ -60,12 +59,10 @@ require (
|
||||
github.com/klauspost/compress v1.17.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.22 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/spf13/cobra v1.8.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
@ -74,8 +71,6 @@ require (
|
||||
github.com/valyala/fasttemplate v1.2.2 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.opentelemetry.io/otel v1.24.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.24.0 // indirect
|
||||
gocloud.dev v0.37.0 // indirect
|
||||
golang.org/x/crypto v0.24.0 // indirect
|
||||
golang.org/x/image v0.16.0 // indirect
|
||||
|
||||
@ -109,8 +109,6 @@ github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVH
|
||||
github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/gofiber/fiber/v2 v2.52.1 h1:1RoU2NS+b98o1L77sdl5mboGPiW+0Ypsi5oLmcYlgHI=
|
||||
github.com/gofiber/fiber/v2 v2.52.1/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.2 h1:mVVgt8PTaHGup3NGl/+7U7nEoZaXJ5OComV4E+HpAao=
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.2/go.mod h1:w7sdfTY0okjZ1oVH6rSOGvuACUIt0By1iK0HKUb3uqM=
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.4 h1:1gjbVFFwVwUb9arPcqiB6iEjHBwo7cHsyS41NeIW3co=
|
||||
@ -162,6 +160,8 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u
|
||||
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
|
||||
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
@ -187,11 +187,11 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
|
||||
github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
@ -206,13 +206,9 @@ github.com/pocketbase/pocketbase v0.22.12/go.mod h1:yY/3IGi1tUbcI6yGVFspAyKi/IDH
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/samber/slog-fiber v1.15.3 h1:RbfC0v2QPIEXoRdort2QxAsRG42LVaFTEgTNS/0GwRQ=
|
||||
github.com/samber/slog-fiber v1.15.3/go.mod h1:I0b8eJ060SlpA65LXiqH7lZixUCkAPKiEGZqkT9QJOM=
|
||||
github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
|
||||
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
||||
|
||||
Reference in New Issue
Block a user