diff --git a/app/db/models/site.py b/app/db/models/site.py index d4d1ec59..cf830d22 100644 --- a/app/db/models/site.py +++ b/app/db/models/site.py @@ -20,6 +20,7 @@ class Site(Base): ua = Column(String) proxy = Column(Integer) filter = Column(String) + render = Column(Integer) note = Column(String) limit_interval = Column(Integer) limit_count = Column(Integer) diff --git a/app/helper/browser.py b/app/helper/browser.py index 99381d5b..5ade5e8d 100644 --- a/app/helper/browser.py +++ b/app/helper/browser.py @@ -1,5 +1,5 @@ from playwright.sync_api import sync_playwright - +from cf_clearance import sync_cf_retry, sync_stealth from app.log import logger @@ -29,8 +29,11 @@ class PlaywrightHelper: if cookies: page.set_extra_http_headers({"cookie": cookies}) try: - + sync_stealth(page, pure=True) page.goto(url) + res = sync_cf_retry(page) + if not res: + logger.warn("cloudflare challenge fail!") page.wait_for_load_state("networkidle", timeout=timeout * 1000) source = page.content() except Exception as e: @@ -45,8 +48,8 @@ class PlaywrightHelper: # 示例用法 if __name__ == "__main__": utils = PlaywrightHelper() - test_url = "https://www.baidu.com" - test_cookies = "cookie1=value1; cookie2=value2" + test_url = "https://piggo.me" + test_cookies = "" test_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36" source_code = utils.get_page_source(test_url, cookies=test_cookies, ua=test_user_agent) print(source_code) diff --git a/app/schemas/site.py b/app/schemas/site.py index b264cc0e..3c598ece 100644 --- a/app/schemas/site.py +++ b/app/schemas/site.py @@ -14,6 +14,7 @@ class Site(BaseModel): ua: Optional[str] = None proxy: Optional[int] = 0 filter: Optional[str] = None + render: Optional[int] = 0 note: Optional[str] = None limit_interval: Optional[int] = 0 limit_count: Optional[int] = 0 diff --git a/requirements.txt b/requirements.txt index b246ef1e..fa5b0337 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,4 +36,5 @@ bs4~=0.0.1 beautifulsoup4~=4.12.2 pillow~=9.5.0 pyTelegramBotAPI~=4.12.0 -playwright~=1.34.0 \ No newline at end of file +playwright~=1.34.0 +cf_clearance~=0.29.2 \ No newline at end of file