feat: Change timeout error to warning and continue scraping

This commit is contained in:
Paul Gauthier (aider) 2025-03-28 15:34:18 -10:00
parent 6689f001cf
commit fa256eb1a7

View file

@ -159,7 +159,8 @@ class Scraper:
try:
response = page.goto(url, wait_until="networkidle", timeout=5000)
except PlaywrightTimeoutError:
self.print_error(f"Timeout while loading {url}")
self.print_error(f"Page didn't quiesce, scraping content anyway")
response = None
except PlaywrightError as e:
self.print_error(f"Error navigating to {url}: {str(e)}")
return None, None