masterElmar/flatscraper
1
0
Fork 0
mirror of https://gitlab.dit.htwk-leipzig.de/fsr-im/tools/flatscraper.git synced 2025-07-16 03:28:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add BGL scraper and integrate into main scraping process; update image scraping logic

Browse Source
This commit is contained in:
Elmar Kresse
2025-02-09 19:33:53 +01:00
parent 010a3249cf
commit f1c9816d9e
7 changed files with 147 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
main.py
View File

@ -5,6 +5,7 @@ import src.wogetra.scraper as wogetra_scraper
import src.lwb.scraper as lwb_scraper
import src.discord.webhook as localwebhook
import src.lipsia.lipsia as lipsia_scraper
import src.bgl.bgl as bgl_scraper
# URL of the website to scrape
TARGET_URL = "https://www.wogetra.de/immobilien-vermarktungsart/miete/"
@ -26,12 +27,14 @@ def main():
while True:
current_time = time.strftime("%H:%M:%S", time.localtime())
print("Scraping properties at " + current_time)
print("Scraping properties from Wogetra...")
properties = wogetra_scraper.scrape_wogetra()
print("Scraping properties from LWB...")
properties += lwb_scraper.scrape_easysquare()
print("Scraping properties from Lipsia...")
properties += lipsia_scraper.scrape_lipsia()
# print("Scraping properties from Wogetra...")
# properties = wogetra_scraper.scrape_wogetra()
# print("Scraping properties from LWB...")
# properties += lwb_scraper.scrape_easysquare()
# print("Scraping properties from Lipsia...")
# properties += lipsia_scraper.scrape_lipsia()
print("Scraping properties from BGL...")
properties = bgl_scraper.fetch_all_properties()
for prop in properties:
if prop["id"] not in known_properties: