Roughly working scraper

requirements.txt

@@ -13,9 +13,7 @@ anyio==4.2.0
 attrs==23.2.0
     # via
     #   jsonschema
-    #   outcome
     #   referencing
-    #   trio
 backoff==2.2.1
     # via -r requirements.in
 beautifulsoup4==4.12.3
@@ -27,15 +25,12 @@ certifi==2023.11.17
     #   httpcore
     #   httpx
     #   requests
-    #   selenium
 cffi==1.16.0
     # via
     #   sounddevice
     #   soundfile
 charset-normalizer==3.3.2
     # via requests
-chromedriver-autoinstaller==0.3.1
-    # via -r requirements.in
 configargparse==1.7
     # via -r requirements.in
 diff-match-patch==20230430
@@ -48,12 +43,12 @@ gitdb==4.0.11
     # via gitpython
 gitpython==3.1.40
     # via -r requirements.in
+greenlet==3.0.3
+    # via playwright
 grep-ast==0.2.4
     # via -r requirements.in
 h11==0.14.0
-    # via
-    #   httpcore
-    #   wsproto
+    # via httpcore
 httpcore==1.0.2
     # via httpx
 httpx==0.26.0
@@ -63,7 +58,6 @@ idna==3.6
     #   anyio
     #   httpx
     #   requests
-    #   trio
 jsonschema==4.20.0
     # via -r requirements.in
 jsonschema-specifications==2023.12.1
@@ -80,18 +74,16 @@ numpy==1.26.3
     #   scipy
 openai==1.6.1
     # via -r requirements.in
-outcome==1.3.0.post0
-    # via trio
 packaging==23.2
-    # via
-    #   -r requirements.in
-    #   webdriver-manager
+    # via -r requirements.in
 pathspec==0.12.1
     # via
     #   -r requirements.in
     #   grep-ast
 pillow==10.2.0
     # via -r requirements.in
+playwright==1.41.2
+    # via -r requirements.in
 prompt-toolkit==3.0.43
     # via -r requirements.in
 pycparser==2.21
@@ -100,12 +92,10 @@ pydantic==2.5.3
     # via openai
 pydantic-core==2.14.6
     # via pydantic
+pyee==11.0.1
+    # via playwright
 pygments==2.17.2
     # via rich
-pysocks==1.7.1
-    # via urllib3
-python-dotenv==1.0.1
-    # via webdriver-manager
 pyyaml==6.0.1
     # via -r requirements.in
 referencing==0.32.0
@@ -115,9 +105,7 @@ referencing==0.32.0
 regex==2023.12.25
     # via tiktoken
 requests==2.31.0
-    # via
-    #   tiktoken
-    #   webdriver-manager
+    # via tiktoken
 rich==13.7.0
     # via -r requirements.in
 rpds-py==0.16.2
@@ -126,8 +114,6 @@ rpds-py==0.16.2
     #   referencing
 scipy==1.11.4
     # via -r requirements.in
-selenium==4.17.2
-    # via -r requirements.in
 smmap==5.0.1
     # via gitdb
 sniffio==1.3.0
@@ -135,9 +121,6 @@ sniffio==1.3.0
     #   anyio
     #   httpx
     #   openai
-    #   trio
-sortedcontainers==2.4.0
-    # via trio
 sounddevice==0.4.6
     # via -r requirements.in
 soundfile==0.12.1
@@ -152,26 +135,13 @@ tree-sitter==0.20.4
     # via tree-sitter-languages
 tree-sitter-languages==1.9.1
     # via grep-ast
-trio==0.24.0
-    # via
-    #   selenium
-    #   trio-websocket
-trio-websocket==0.11.1
-    # via selenium
 typing-extensions==4.9.0
     # via
     #   openai
     #   pydantic
     #   pydantic-core
-    #   selenium
-urllib3[socks]==2.1.0
-    # via
-    #   requests
-    #   selenium
-    #   urllib3
+    #   pyee
+urllib3==2.1.0
+    # via requests
 wcwidth==0.2.12
     # via prompt-toolkit
-webdriver-manager==4.0.1
-    # via -r requirements.in
-wsproto==1.2.0
-    # via trio-websocket