feat: Add option to return web scrape content instead of printing

aider/coders/base_coder.py

@@ -820,7 +820,7 @@ class Coder:
                     "Add URL to the chat?", subject=url, group=group, allow_never=True
                 ):
                     inp += "\n\n"
-                    inp += self.commands.cmd_web(url)
+                    inp += self.commands.cmd_web(url, return_content=True)
                     added_urls.append(url)
                 else:
                     self.rejected_urls.add(url)

aider/commands.py

@@ -139,7 +139,7 @@ class Commands:
         else:
             self.io.tool_output("Please provide a partial model name to search for.")
 
-    def cmd_web(self, args):
+    def cmd_web(self, args, return_content=False):
         "Scrape a webpage, convert to markdown and send in a message"
 
         url = args.strip()
@@ -159,6 +159,8 @@ class Commands:
 
         content = self.scraper.scrape(url) or ""
         content = f"Here is the content of {url}:\n\n" + content
+        if return_content:
+            return content
 
         self.io.tool_output("... added to chat.")
 

tests/scrape/test_scrape.py

@@ -44,7 +44,7 @@ class TestScrape(unittest.TestCase):
         self.commands.io.tool_error = mock_print_error
 
         # Run the cmd_web command
-        result = self.commands.cmd_web("https://example.com")
+        result = self.commands.cmd_web("https://example.com", return_content=True)
 
         # Assert that the result contains some content
         self.assertIsNotNone(result)