htwkalender/backend/service/fetch/v2/fetcher.go

package v2

import (
	"fmt"
	"github.com/google/uuid"
	"github.com/pocketbase/pocketbase"
	"golang.org/x/net/html"
	"htwkalender/model"
	"htwkalender/service/db"
	"htwkalender/service/fetch"
	v1 "htwkalender/service/fetch/v1"
	localTime "htwkalender/service/functions/time"
	"log/slog"
	"strings"
	"time"
)

func ParseEventsFromRemote(app *pocketbase.PocketBase) (model.Events, error) {
	savedRecords, err := FetchAllEventsAndSave(app, localTime.RealClock{})
	if err != nil {
		return nil, err
	}
	return savedRecords, nil
}

func FetchAllEventsAndSave(app *pocketbase.PocketBase, clock localTime.Clock) ([]model.Event, error) {
	var savedRecords []model.Event
	var err error = nil

	var stubUrl = [2]string{
		"https://stundenplan.htwk-leipzig.de/",
		"/Berichte/Text-Listen;Veranstaltungsarten;name;" +
			"Vp%0A" +
			"Vw%0A" +
			"V%0A" +
			"Sp%0A" +
			"Sw%0A" +
			"S%0A" +
			"Pp%0A" +
			"Pw%0A" +
			"P%0A" +
			"ZV%0A" +
			"Tut%0A" +
			"Sperr%0A" +
			"pf%0A" +
			"wpf%0A" +
			"fak%0A" +
			"Pruefung%0A" +
			"gebucht%0A" +
			"Vertretung%0A" +
			"Fremdveranst.%0A" +
			"Buchen%0A" +
			"%0A?&template=sws_modul&weeks=1-65&combined=yes",
	}

	savedRecordsSs, errSs := fetchAndSaveAllEventsForSemester(app, time.March, time.October, "ss", clock, stubUrl)
	if errSs == nil {
		savedRecords = append(savedRecords, savedRecordsSs...)
	}
	savedRecordsWs, errWs := fetchAndSaveAllEventsForSemester(app, time.September, time.April, "ws", clock, stubUrl)
	if errWs == nil {
		savedRecords = append(savedRecords, savedRecordsWs...)
	}
	return savedRecords, err
}

func fetchAndSaveAllEventsForSemester(
	app *pocketbase.PocketBase,
	before time.Month,
	after time.Month,
	semester string,
	clock localTime.Clock,
	stubUrl [2]string,
) ([]model.Event, error) {
	var err error = nil
	var savedRecords []model.Event

	if (clock.Now().Month() >= after) || (clock.Now().Month() <= before) {
		url := stubUrl[0] + semester + stubUrl[1]
		var events []model.Event
		events, err = parseEventForOneSemester(url)
		if err != nil {
			return nil, fmt.Errorf("failed to parse events for "+semester+": %w", err)
		}
		err = db.DeleteAllEventsBySemesterWithoutCourse(app, "Sport", semester)
		if err != nil {
			return nil, fmt.Errorf("failed to delete all events for "+semester+": %w", err)
		}
		savedEvents, dbError := db.SaveEvents(events, app)
		if dbError != nil {
			return nil, fmt.Errorf("failed to save events for "+semester+": %w", dbError)
		}
		savedRecords = append(savedRecords, savedEvents...)
	}
	return savedRecords, err
}

func parseEventForOneSemester(url string) ([]model.Event, error) {
	// Fetch Webpage from URL
	webpage, err := fetch.GetHTML(url)
	if err != nil {
		return nil, err
	}

	// Parse HTML to Node Tree
	var doc *html.Node
	doc, err = parseHTML(webpage)
	if err != nil {
		return nil, err
	}

	// Get all event tables and all day labels
	eventTables := getEventTables(doc)
	allDayLabels := getAllDayLabels(doc)

	eventsWithCombinedWeeks := toEvents(eventTables, allDayLabels)

	splitEventsByWeekVal := splitEventsByWeek(eventsWithCombinedWeeks)
	events := splitEventsBySingleWeek(splitEventsByWeekVal)

	if events == nil {
		return nil, err
	}

	table := findFirstTable(doc)

	if table == nil {
		return nil, fmt.Errorf("failed to find first table")
	}

	semesterString := findFirstSpanWithClass(table, "header-0-2-0").FirstChild.Data
	semester, year := extractSemesterAndYear(semesterString)
	events = convertWeeksToDates(events, semester, year)
	events, err = v1.SplitEventType(events)
	if err != nil {
		slog.Error("Error occurred while splitting event types: %s", err)
		return nil, err
	}
	events = switchNameAndNotesForExam(events)
	events = generateUUIDs(events)

	return events, nil
}

// switch name and notes for Pruefung events when Note is not empty and Name starts with "Prüfungen" and contains email
func switchNameAndNotesForExam(events []model.Event) []model.Event {
	for i, event := range events {
		if event.EventType == "Pruefung" {
			if event.Notes != "" && strings.HasPrefix(event.Name, "Prüfungen") && strings.Contains(event.Name, "@") {
				events[i].Name = event.Notes
				events[i].Notes = event.Name
			}
		}
	}
	return events
}

func parseHTML(webpage string) (*html.Node, error) {
	doc, err := html.Parse(strings.NewReader(webpage))
	if err != nil {
		return nil, err
	}
	return doc, nil
}

func generateUUIDs(events []model.Event) []model.Event {
	for i, event := range events {
		// generate a hash value from the event name, course and semester
		hash := uuid.NewSHA1(uuid.NameSpaceOID, []byte(event.Name+event.Course))
		events[i].UUID = hash.String()
	}
	return events
}