mirror of
https://github.com/Aider-AI/aider.git
synced 2025-05-31 01:35:00 +00:00
aider: Implemented HTTP request using httpx
library in scrape_with_httpx
method.
# Aider chat conversation: USER: finish `scrape_with_httpx` ASSISTANT: To finish the `scrape_with_httpx` method, I will implement the HTTP request using the `httpx` library. This will involve making a GET request to the provided URL and returning the content of the response. Here is the diff for the changes: ```diff --- aider/scrape.py +++ aider/scrape.py @@ ... @@ def scrape_with_httpx(self, url): - pass + import httpx + headers = { + 'User-Agent': aider_user_agent + } + try: + with httpx.Client(headers=headers) as client: + response = client.get(url) + response.raise_for_status() + return response.text + except httpx.HTTPError as http_err: + self.print_error(f'HTTP error occurred: {http_err}') + except Exception as err: + self.print_error(f'An error occurred: {err}') + return None ``` This implementation uses `httpx.Client` to send a GET request with a custom User-Agent header. It handles any HTTP errors by printing them out and returns `None` in case of an error. If the request is successful, it returns the text content of the response.
This commit is contained in:
parent
e04187d1ad
commit
9bf3a6e0c6
1 changed files with 14 additions and 1 deletions
|
@ -56,7 +56,20 @@ class Scraper:
|
|||
self.print_error(PLAYWRIGHT_INFO)
|
||||
|
||||
def scrape_with_httpx(self, url):
|
||||
pass
|
||||
import httpx
|
||||
headers = {
|
||||
'User-Agent': aider_user_agent
|
||||
}
|
||||
try:
|
||||
with httpx.Client(headers=headers) as client:
|
||||
response = client.get(url)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
except httpx.HTTPError as http_err:
|
||||
self.print_error(f'HTTP error occurred: {http_err}')
|
||||
except Exception as err:
|
||||
self.print_error(f'An error occurred: {err}')
|
||||
return None
|
||||
|
||||
def scrape(self, url):
|
||||
if self.playwright_available is None:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue