masterElmar/flatscraper
1
0
Fork 0
mirror of https://gitlab.dit.htwk-leipzig.de/fsr-im/tools/flatscraper.git synced 2025-07-16 11:38:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: update Docker setup, add environment variables, and enhance property scraping logic

Browse Source
This commit is contained in:
Elmar Kresse
2025-02-10 10:02:53 +01:00
parent f16116040d
commit 92bb5b2e85
11 changed files with 66 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
main.py
View File

@ -27,14 +27,23 @@ def main():
while True:
current_time = time.strftime("%H:%M:%S", time.localtime())
print("Scraping properties at " + current_time)
print("Scraping properties from Wogetra...")
properties = wogetra_scraper.scrape_wogetra()
print("Scraping properties from LWB...")
properties += lwb_scraper.scrape_easysquare()
print("Scraping properties from Lipsia...")
properties += lipsia_scraper.scrape_lipsia()
print("Scraping properties from BGL...")
properties = bgl_scraper.fetch_all_properties()
properties_wogetra = wogetra_scraper.scrape_wogetra()
print("Scraped " + str(len(properties_wogetra)) + " properties from Wogetra")
properties = properties_wogetra
properties_lwb = lwb_scraper.scrape_easysquare()
print("Scraped " + str(len(properties_lwb)) + " properties from LWB")
properties += properties_lwb
properties_lipsia = lipsia_scraper.scrape_lipsia()
print("Scraped " + str(len(properties_lipsia)) + " properties from Lipsia")
properties += properties_lipsia
properties_bgl = bgl_scraper.fetch_all_properties()
print("Scraped " + str(len(properties_bgl)) + " properties from BGL")
properties += properties_bgl
for prop in properties:
if prop["id"] not in known_properties: