From dece830cb8c91949f730ab84ca6dd7567d4a645b Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Tue, 11 Jun 2024 06:37:41 -0700 Subject: [PATCH] wait up to 5sec --- aider/scrape.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aider/scrape.py b/aider/scrape.py index 4dc5bd1a7..147203c48 100755 --- a/aider/scrape.py +++ b/aider/scrape.py @@ -4,6 +4,7 @@ import re import sys import httpx +import playwright import pypandoc from bs4 import BeautifulSoup from playwright.sync_api import sync_playwright @@ -81,7 +82,10 @@ class Scraper: user_agent += " " + aider_user_agent page = browser.new_page(user_agent=user_agent) - page.goto(url, wait_until="networkidle", timeout=5000) + try: + page.goto(url, wait_until="networkidle", timeout=5000) + except playwright._impl._errors.TimeoutError: + pass content = page.content() browser.close()