From e3b3e891b5379f18fe768f867b9b522a37bd4dd3 Mon Sep 17 00:00:00 2001 From: Elmar Kresse Date: Mon, 3 Jun 2024 23:54:22 +0200 Subject: [PATCH] fix:#34 added transaction for main fetch datasource --- backend/service/addCalDavRoutes.go | 17 +++++----- backend/service/db/dbEvents.go | 49 ++++++++++++++++++++-------- backend/service/fetch/v2/fetcher.go | 50 +++++++++++++++++++++++------ docker-compose.yml | 2 -- 4 files changed, 85 insertions(+), 33 deletions(-) diff --git a/backend/service/addCalDavRoutes.go b/backend/service/addCalDavRoutes.go index 1b72e06..8f91f50 100644 --- a/backend/service/addCalDavRoutes.go +++ b/backend/service/addCalDavRoutes.go @@ -60,18 +60,17 @@ func addFeedRoutes(app *pocketbase.PocketBase) { if err != nil { return c.JSON(http.StatusInternalServerError, "Failed to get feed") } - var responseWriter = c.Response().Writer - responseWriter.Header().Set("Content-type", "text/calendar") - responseWriter.Header().Set("charset", "utf-8") - responseWriter.Header().Set("Content-Disposition", "inline") - responseWriter.Header().Set("filename", "calendar.ics") - responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write([]byte(result)) + c.Response().Header().Set("Content-type", "text/calendar") + c.Response().Header().Set("charset", "utf-8") + c.Response().Header().Set("Content-Disposition", "inline") + c.Response().Header().Set("filename", "calendar.ics") + + _, err = c.Response().Write([]byte(result)) + if err != nil { - return c.JSON(http.StatusInternalServerError, "Failed to write feed") + return err } - c.Response().Writer = responseWriter return nil }, Middlewares: []echo.MiddlewareFunc{ diff --git a/backend/service/db/dbEvents.go b/backend/service/db/dbEvents.go index 8db7b64..ac4b793 100644 --- a/backend/service/db/dbEvents.go +++ b/backend/service/db/dbEvents.go @@ -18,6 +18,7 @@ package db import ( "fmt" + "github.com/pocketbase/pocketbase/daos" "github.com/pocketbase/pocketbase/tools/types" "htwkalender/model" "log/slog" @@ -34,7 +35,7 @@ func SaveSeminarGroupEvents(seminarGroup model.SeminarGroup, app *pocketbase.Poc // check if event is already in database and add to toBeSavedEvents if not for _, event := range seminarGroup.Events { event = event.SetCourse(seminarGroup.Course) - existsInDatabase, err := findEventByDayWeekStartEndNameCourse(event, seminarGroup.Course, app) + existsInDatabase, err := findEventByDayWeekStartEndNameCourse(event, seminarGroup.Course, app.Dao()) alreadyAddedToSave := toBeSavedEvents.Contains(event) if err != nil { @@ -61,13 +62,44 @@ func SaveSeminarGroupEvents(seminarGroup model.SeminarGroup, app *pocketbase.Poc return savedRecords, nil } +func SaveEventsTxDao(events []model.Event, txDao *daos.Dao) ([]model.Event, error) { + var toBeSavedEvents model.Events + var savedRecords model.Events + + // check if event is already in database and add to toBeSavedEvents if not + for _, event := range events { + existsInDatabase, err := findEventByDayWeekStartEndNameCourse(event, event.Course, txDao) + alreadyAddedToSave := toBeSavedEvents.Contains(event) + + if err != nil { + return nil, err + } + + if !existsInDatabase && !alreadyAddedToSave { + toBeSavedEvents = append(toBeSavedEvents, event) + } + } + // create record for each event that's not already in the database + for _, event := range toBeSavedEvents { + event.MarkAsNew() + // auto mapping for event fields to record fields + err := txDao.Save(&event) + if err != nil { + return nil, err + } else { + savedRecords = append(savedRecords, event) + } + } + return savedRecords, nil +} + func SaveEvents(events []model.Event, app *pocketbase.PocketBase) ([]model.Event, error) { var toBeSavedEvents model.Events var savedRecords model.Events // check if event is already in database and add to toBeSavedEvents if not for _, event := range events { - existsInDatabase, err := findEventByDayWeekStartEndNameCourse(event, event.Course, app) + existsInDatabase, err := findEventByDayWeekStartEndNameCourse(event, event.Course, app.Dao()) alreadyAddedToSave := toBeSavedEvents.Contains(event) if err != nil { @@ -94,11 +126,11 @@ func SaveEvents(events []model.Event, app *pocketbase.PocketBase) ([]model.Event } // check if event is already in database and return true if it is and false if it's not -func findEventByDayWeekStartEndNameCourse(event model.Event, course string, app *pocketbase.PocketBase) (bool, error) { +func findEventByDayWeekStartEndNameCourse(event model.Event, course string, dao *daos.Dao) (bool, error) { var dbEvent model.Event - err := app.Dao().DB().Select("*").From("events"). + err := dao.DB().Select("*").From("events"). Where(dbx.NewExp( "Day = {:day} AND "+ "Week = {:week} AND "+ @@ -239,15 +271,6 @@ func DeleteAllEventsByCourse(app *pocketbase.PocketBase, course string, semester return nil } -func DeleteAllEventsBySemesterWithoutCourse(app *pocketbase.PocketBase, course string, semester string) error { - _, err := app.Dao().DB().Delete("events", dbx.NewExp("course != {:course} AND semester = {:semester}", dbx.Params{"course": course, "semester": semester})).Execute() - - if err != nil { - return err - } - return nil -} - func DeleteAllEvents(app *pocketbase.PocketBase) error { _, err := app.Dao().DB().Delete("events", dbx.NewExp("1=1")).Execute() diff --git a/backend/service/fetch/v2/fetcher.go b/backend/service/fetch/v2/fetcher.go index b26fe8c..131b85f 100644 --- a/backend/service/fetch/v2/fetcher.go +++ b/backend/service/fetch/v2/fetcher.go @@ -17,9 +17,12 @@ package v2 import ( + "database/sql" "fmt" "github.com/google/uuid" + "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/daos" "golang.org/x/net/html" "htwkalender/model" "htwkalender/service/db" @@ -89,21 +92,50 @@ func fetchAndSaveAllEventsForSemester( var savedRecords []model.Event url := stubUrl[0] + semester + stubUrl[1] events, err := parseEventForOneSemester(url) + if err != nil { return nil, fmt.Errorf("failed to parse events for "+semester+": %w", err) } - err = db.DeleteAllEventsBySemesterWithoutCourse(app, "Sport", semester) - if err != nil { - return nil, fmt.Errorf("failed to delete all events for "+semester+": %w", err) - } - savedEvents, dbError := db.SaveEvents(events, app) - if dbError != nil { - return nil, fmt.Errorf("failed to save events for "+semester+": %w", dbError) - } - savedRecords = append(savedRecords, savedEvents...) + + err = updateDatabase(app, events, "Sport", semester) + return savedRecords, err } +func updateDatabase(app *pocketbase.PocketBase, eventsToBeAdded []model.Event, course string, semester string) error { + + var addedEvents []model.Event + var err error + + // to in transaction the events will be added and deleted + err = app.Dao().RunInTransaction(func(txDao *daos.Dao) error { + var execute sql.Result + execute, err = txDao.DB().Delete("events", + dbx.NewExp("course != {:course} AND semester = {:semester}", + dbx.Params{"course": course, "semester": semester})).Execute() + if err != nil { + return err + } + var rows int64 + rows, err = execute.RowsAffected() + slog.Info("Deleted events: ", "events", rows) + + addedEvents, err = db.SaveEventsTxDao(eventsToBeAdded, txDao) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + slog.Info("Added events: ", "events", len(addedEvents)) + + return nil +} + func parseEventForOneSemester(url string) ([]model.Event, error) { // Fetch Webpage from URL webpage, err := fetch.GetHTML(url) diff --git a/docker-compose.yml b/docker-compose.yml index 5b2ce9b..9eb3608 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,8 +14,6 @@ #You should have received a copy of the GNU Affero General Public License #along with this program. If not, see . -version: "3.9" - services: htwkalender-backend: build: