wip

aider/commands.py

@@ -1,9 +1,3 @@
-from pathlib import Path
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.chrome.service import Service
-from selenium.webdriver.common.by import By
-from webdriver_manager.chrome import ChromeDriverManager
 import re
 import subprocess
 import sys
@@ -13,7 +7,7 @@ import git
 from prompt_toolkit.completion import Completion
 
 from aider import prompts, voice
-from aider.utils import is_gpt4_with_openai_base_url, is_image_file
+from aider.utils import is_gpt4_with_openai_base_url, is_image_file, scrape
 
 from .dump import dump  # noqa: F401
 
@@ -31,18 +25,6 @@ class Commands:
 
         self.voice_language = voice_language
         self.tokenizer = coder.main_model.tokenizer
-        self.initialize_web_driver()
-
-    def initialize_web_driver(self):
-        chrome_options = Options()
-        chrome_options.add_argument("--headless")
-        chrome_options.add_argument("--disable-gpu")
-        chrome_options.add_argument("--no-sandbox")
-        chrome_options.add_argument("--disable-dev-shm-usage")
-        self.web_driver = webdriver.Chrome(
-            service=Service(ChromeDriverManager().install()),
-            options=chrome_options
-        )
 
     def cmd_web(self, args):
         "Use headless selenium to scrape a webpage and add the content to the chat"
@@ -51,13 +33,9 @@ class Commands:
             self.io.tool_error("Please provide a URL to scrape.")
             return
 
-        try:
+        content = scrape(url)
-            self.web_driver.get(url)
+        print(content)
-            page_content = self.web_driver.find_element(By.TAG_NAME, "body").text
+        return content
-            self.io.tool_output(f"Content from {url}:\n{page_content}")
-            return page_content
-        except Exception as e:
-            self.io.tool_error(f"Error scraping {url}: {e}")
 
     def is_command(self, inp):
         return inp[0] in "/!"

aider/utils.py

@@ -1,9 +1,32 @@
 import os
 import tempfile
 from pathlib import Path
-import git
+from typing import Type
 
-IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.bmp', '.tiff', '.webp'}
+import git
+from bs4 import BeautifulSoup
+from selenium.common.exceptions import WebDriverException
+from selenium.webdriver.chrome.options import Options as ChromeOptions
+from selenium.webdriver.chrome.service import Service as ChromeDriverService
+from selenium.webdriver.chrome.webdriver import WebDriver as ChromeDriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.options import ArgOptions as BrowserOptions
+from selenium.webdriver.edge.options import Options as EdgeOptions
+from selenium.webdriver.edge.service import Service as EdgeDriverService
+from selenium.webdriver.edge.webdriver import WebDriver as EdgeDriver
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.firefox.service import Service as GeckoDriverService
+from selenium.webdriver.firefox.webdriver import WebDriver as FirefoxDriver
+from selenium.webdriver.remote.webdriver import WebDriver
+from selenium.webdriver.safari.options import Options as SafariOptions
+from selenium.webdriver.safari.webdriver import WebDriver as SafariDriver
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+from webdriver_manager.chrome import ChromeDriverManager
+from webdriver_manager.firefox import GeckoDriverManager
+from webdriver_manager.microsoft import EdgeChromiumDriverManager as EdgeDriverManager
+
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".webp"}
 
 from aider.dump import dump  # noqa: F401
 
@@ -65,10 +88,11 @@ def make_repo(path=None):
 
     return repo
 
+
 def is_image_file(file_name):
     """
     Check if the given file name has an image file extension.
-    
+
     :param file_name: The name of the file to check.
     :return: True if the file is an image, False otherwise.
     """
@@ -103,14 +127,116 @@ def show_messages(messages, title=None, functions=None):
     if functions:
         dump(functions)
 
+
 def is_gpt4_with_openai_base_url(model_name, client):
     """
     Check if the model_name starts with 'gpt-4' and the client base URL includes 'api.openai.com'.
-    
+
     :param model_name: The name of the model to check.
     :param client: The OpenAI client instance.
     :return: True if conditions are met, False otherwise.
     """
-    if client is None or not hasattr(client, 'base_url'):
+    if client is None or not hasattr(client, "base_url"):
         return False
     return model_name.startswith("gpt-4") and "api.openai.com" in client.base_url.host
+
+
+# Taken from AutoGPT, MIT License
+def open_page_in_browser(
+    url: str,
+    selenium_web_browser="chrome",
+    selenium_headless=True,
+    platform="linux",
+    user_agent="Aider CLI 0.23.0",
+) -> WebDriver:
+    """Open a browser window and load a web page using Selenium
+
+    Params:
+        url (str): The URL of the page to load
+        config (Config): The applicable application configuration
+
+    Returns:
+        driver (WebDriver): A driver object representing the browser window to scrape
+    """
+    options_available: dict[str, Type[BrowserOptions]] = {
+        "chrome": ChromeOptions,
+        "edge": EdgeOptions,
+        "firefox": FirefoxOptions,
+        "safari": SafariOptions,
+    }
+
+    options: BrowserOptions = options_available[selenium_web_browser]()
+    options.add_argument(f"user-agent={user_agent}")
+
+    if selenium_web_browser == "firefox":
+        if selenium_headless:
+            options.headless = True
+            options.add_argument("--disable-gpu")
+        driver = FirefoxDriver(
+            service=GeckoDriverService(GeckoDriverManager().install()), options=options
+        )
+    elif selenium_web_browser == "edge":
+        driver = EdgeDriver(
+            service=EdgeDriverService(EdgeDriverManager().install()), options=options
+        )
+    elif selenium_web_browser == "safari":
+        # Requires a bit more setup on the users end.
+        # See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari  # noqa: E501
+        driver = SafariDriver(options=options)
+    else:
+        if platform == "linux" or platform == "linux2":
+            options.add_argument("--disable-dev-shm-usage")
+            options.add_argument("--remote-debugging-port=9222")
+
+        options.add_argument("--no-sandbox")
+        if selenium_headless:
+            options.add_argument("--headless=new")
+            options.add_argument("--disable-gpu")
+
+        chromium_driver_path = Path("/usr/bin/chromedriver")
+
+        driver = ChromeDriver(
+            service=(
+                ChromeDriverService(str(chromium_driver_path))
+                if chromium_driver_path.exists()
+                else ChromeDriverService(ChromeDriverManager().install())
+            ),
+            options=options,
+        )
+    driver.get(url)
+
+    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+
+    return driver
+
+
+# Taken from AutoGPT, MIT License
+def scrape_text_with_selenium(driver: WebDriver) -> str:
+    """Scrape text from a browser window using selenium
+
+    Args:
+        driver (WebDriver): A driver object representing the browser window to scrape
+
+    Returns:
+        str: the text scraped from the website
+    """
+
+    # Get the HTML content directly from the browser's DOM
+    page_source = driver.execute_script("return document.body.outerHTML;")
+    soup = BeautifulSoup(page_source, "html.parser")
+
+    for script in soup(["script", "style"]):
+        script.extract()
+
+    text = soup.get_text()
+    lines = (line.strip() for line in text.splitlines())
+    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+    text = "\n".join(chunk for chunk in chunks if chunk)
+    return text
+
+
+def scrape(url: str):
+    driver = open_page_in_browser(url)
+    text = scrape_text_with_selenium(driver)
+    driver.quit()
+    return text

requirements.txt

@@ -17,15 +17,11 @@ attrs==23.2.0
     #   referencing
     #   trio
 backoff==2.2.1
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 beautifulsoup4==4.12.3
     # via bs4
 bs4==0.0.2
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 certifi==2023.11.17
     # via
     #   httpcore
@@ -38,30 +34,22 @@ cffi==1.16.0
     #   soundfile
 charset-normalizer==3.3.2
     # via requests
+chromedriver-autoinstaller==0.3.1
+    # via -r requirements.in
 configargparse==1.7
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 diff-match-patch==20230430
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 diskcache==5.6.3
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 distro==1.9.0
     # via openai
 gitdb==4.0.11
     # via gitpython
 gitpython==3.1.40
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 grep-ast==0.2.4
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 h11==0.14.0
     # via
     #   httpcore
@@ -77,9 +65,7 @@ idna==3.6
     #   requests
     #   trio
 jsonschema==4.20.0
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 jsonschema-specifications==2023.12.1
     # via jsonschema
 markdown-it-py==3.0.0
@@ -87,17 +73,13 @@ markdown-it-py==3.0.0
 mdurl==0.1.2
     # via markdown-it-py
 networkx==3.2.1
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 numpy==1.26.3
     # via
     #   -r requirements.in
     #   scipy
 openai==1.6.1
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 outcome==1.3.0.post0
     # via trio
 packaging==23.2
@@ -109,13 +91,9 @@ pathspec==0.12.1
     #   -r requirements.in
     #   grep-ast
 pillow==10.2.0
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 prompt-toolkit==3.0.43
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 pycparser==2.21
     # via cffi
 pydantic==2.5.3
@@ -129,9 +107,7 @@ pysocks==1.7.1
 python-dotenv==1.0.1
     # via webdriver-manager
 pyyaml==6.0.1
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 referencing==0.32.0
     # via
     #   jsonschema
@@ -143,21 +119,15 @@ requests==2.31.0
     #   tiktoken
     #   webdriver-manager
 rich==13.7.0
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 rpds-py==0.16.2
     # via
     #   jsonschema
     #   referencing
 scipy==1.11.4
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 selenium==4.17.2
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 smmap==5.0.1
     # via gitdb
 sniffio==1.3.0
@@ -169,19 +139,13 @@ sniffio==1.3.0
 sortedcontainers==2.4.0
     # via trio
 sounddevice==0.4.6
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 soundfile==0.12.1
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 soupsieve==2.5
     # via beautifulsoup4
 tiktoken==0.5.2
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 tqdm==4.66.1
     # via openai
 tree-sitter==0.20.4
@@ -208,8 +172,6 @@ urllib3[socks]==2.1.0
 wcwidth==0.2.12
     # via prompt-toolkit
 webdriver-manager==4.0.1
- selenium==4.17.2
+    # via -r requirements.in
-chromedriver-autoinstaller==0.3.1
-   # via -r requirements.in
 wsproto==1.2.0
     # via trio-websocket