remove selenium

app/helper/cloudflare.py

@@ -0,0 +1,44 @@
+import os
+
+from pyquery import PyQuery
+
+from app.log import logger
+
+CHALLENGE_TITLES = [
+    # Cloudflare
+    'Just a moment...',
+    '请稍候…',
+    # DDoS-GUARD
+    'DDOS-GUARD',
+]
+CHALLENGE_SELECTORS = [
+    # Cloudflare
+    '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js',
+    # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+    'td.info #js_info',
+    # Fairlane / pararius.com
+    'div.vc div.text-box h2'
+]
+SHORT_TIMEOUT = 6
+CF_TIMEOUT = int(os.getenv("NASTOOL_CF_TIMEOUT", "60"))
+
+
+def under_challenge(html_text: str):
+    """
+    Check if the page is under challenge
+    :param html_text:
+    :return:
+    """
+    # get the page title
+    if not html_text:
+        return False
+    page_title = PyQuery(html_text)('title').text()
+    logger.debug("under_challenge page_title=" + page_title)
+    for title in CHALLENGE_TITLES:
+        if page_title.lower() == title.lower():
+            return True
+    for selector in CHALLENGE_SELECTORS:
+        html_doc = PyQuery(html_text)
+        if html_doc(selector):
+            return True
+    return False

app/plugins/autosignin/__init__.py

@@ -9,8 +9,9 @@ from ruamel.yaml import CommentedMap
 
 from app.core.event_manager import EventManager, eventmanager
 from app.core.config import settings
-from app.helper.module import ModuleHelper
+from app.helper.browser import PlaywrightHelper
 from app.helper.cloudflare import under_challenge
+from app.helper.module import ModuleHelper
 from app.helper.sites import SitesHelper
 from app.log import logger
 from app.plugins import _PluginBase
@@ -136,6 +137,8 @@ class AutoSignIn(_PluginBase):
         site_url = site_info.get("url")
         site_cookie = site_info.get("cookie")
         ua = site_info.get("ua")
+        render = site_info.get("render")
+        proxy = settings.PROXY if site_info.get("proxy") else None
         if not site_url or not site_cookie:
             logger.warn(f"未配置 {site} 的站点地址或Cookie，无法签到")
             return ""
@@ -147,9 +150,19 @@ class AutoSignIn(_PluginBase):
                 # 拼登签到地址
                 checkin_url = urljoin(site_url, "attendance.php")
             logger.info(f"开始站点签到：{site}，地址：{checkin_url}...")
+            if render:
+                page_source = PlaywrightHelper().get_page_source(url=checkin_url,
+                                                                 cookies=site_cookie,
+                                                                 ua=ua,
+                                                                 proxy=proxy)
+                if not SiteUtils.is_logged_in(page_source):
+                    if under_challenge(page_source):
+                        return f"【{site}】无法通过Cloudflare！"
+                    return f"【{site}】仿真登录失败，Cookie已失效！"
+            else:
                 res = RequestUtils(cookies=site_cookie,
                                    headers=ua,
-                               proxies=settings.PROXY if site_info.get("proxy") else None
+                                   proxies=proxy
                                    ).get_res(url=checkin_url)
                 if not res and site_url != checkin_url:
                     logger.info(f"开始站点模拟登录：{site}，地址：{site_url}...")
@@ -161,9 +174,9 @@ class AutoSignIn(_PluginBase):
                 if res and res.status_code in [200, 500, 403]:
                     if not SiteUtils.is_logged_in(res.text):
                         if under_challenge(res.text):
-                        msg = "站点被Cloudflare防护，请更换Cookie和UA！"
+                            msg = "站点被Cloudflare防护，请打开站点浏览器仿真！"
                         elif res.status_code == 200:
-                        msg = "Cookie已失效"
+                            msg = "Cookie已失效！"
                         else:
                             msg = f"状态码：{res.status_code}"
                         logger.warn(f"{site} 签到失败，{msg}")

app/plugins/sitestatistic/__init__.py

@@ -10,6 +10,7 @@ from ruamel.yaml import CommentedMap
 from app.core.config import settings
 from app.core.event_manager import eventmanager
 from app.core.event_manager import Event
+from app.helper.browser import PlaywrightHelper
 from app.helper.module import ModuleHelper
 from app.helper.sites import SitesHelper
 from app.log import logger
@@ -87,18 +88,30 @@ class SiteStatistic(_PluginBase):
                 logger.error(f"站点匹配失败 {e}")
         return None
 
-    def build(self, url: str, site_name: str,
+    def build(self, site_info: CommentedMap) -> Optional[ISiteUserInfo]:
-              site_cookie: str = None,
-              ua: str = None,
-              proxy: bool = False) -> Optional[ISiteUserInfo]:
         """
         构建站点信息
         """
+        site_cookie = site_info.get("cookie")
         if not site_cookie:
             return None
+        site_name = site_info.get("name")
+        url = site_info.get("url")
+        proxy = site_info.get("proxy")
+        ua = site_info.get("ua")
         session = requests.Session()
-        logger.debug(f"站点 {site_name} url={url} site_cookie={site_cookie} ua={ua}")
         proxies = settings.PROXY if proxy else None
+        render = site_info.get("render")
+
+        logger.debug(f"站点 {site_name} url={url} site_cookie={site_cookie} ua={ua}")
+        if render:
+            # 演染模式
+            html_text = PlaywrightHelper().get_page_source(url=url,
+                                                           cookies=site_cookie,
+                                                           ua=ua,
+                                                           proxy=proxies)
+        else:
+            # 普通模式
             res = RequestUtils(cookies=site_cookie,
                                session=session,
                                headers=ua,
@@ -115,8 +128,11 @@ class SiteStatistic(_PluginBase):
                     i = html_text.find("window.location")
                     if i == -1:
                         return None
-                tmp_url = url + html_text[i:html_text.find(";")] \
+                    tmp_url = url + html_text[i:html_text.find(";")]\
-                    .replace("\"", "").replace("+", "").replace(" ", "").replace("window.location=", "")
+                        .replace("\"", "")\
+                        .replace("+", "")\
+                        .replace(" ", "")\
+                        .replace("window.location=", "")
                     res = RequestUtils(cookies=site_cookie,
                                        session=session,
                                        headers=ua,
@@ -168,20 +184,13 @@ class SiteStatistic(_PluginBase):
         :param site_info:
         :return:
         """
-        site_name = site_info.get("name")
+        site_name = site_info.get('name')
-        site_url = site_info.get("url")
+        site_url = site_info.get('url')
         if not site_url:
             return None
-        site_cookie = site_info.get("cookie")
-        ua = site_info.get("ua")
         unread_msg_notify = True
-        proxy = site_info.get("proxy")
         try:
-            site_user_info: ISiteUserInfo = self.build(url=site_url,
+            site_user_info: ISiteUserInfo = self.build(site_info=site_info)
-                                                       site_name=site_name,
-                                                       site_cookie=site_cookie,
-                                                       ua=ua,
-                                                       proxy=proxy)
             if site_user_info:
                 logger.debug(f"站点 {site_name} 开始以 {site_user_info.site_schema()} 模型解析")
                 # 开始解析

requirements.txt

@@ -31,7 +31,6 @@ transmission-rpc~=4.3.0
 Jinja2~=3.1.2
 pyparsing~=3.0.9
 func_timeout==4.3.5
-selenium~=4.9.1
 bs4~=0.0.1
 beautifulsoup4~=4.12.2
 pillow~=9.5.0