add 站点连通性

app/api/endpoints/site.py

@@ -9,6 +9,7 @@ from app.chain.site import SiteChain
 from app.db import get_db
 from app.db.models.site import Site
 from app.db.models.user import User
+from app.db.siteicon_oper import SiteIconOper
 from app.db.userauth import get_current_active_user, get_current_active_superuser
 
 router = APIRouter()
@@ -97,3 +98,25 @@ async def update_cookie(
         return schemas.Response(success=False, message=msg)
     else:
         return schemas.Response(success=True, message=msg)
+
+
+@router.get("/test", summary="连接测试", response_model=schemas.Response)
+async def test_site(domain: str, _: User = Depends(get_current_active_user)) -> Any:
+    """
+    测试站点是否可用
+    """
+    status, message = SiteChain().test(domain)
+    return schemas.Response(success=status, message=message)
+
+
+@router.get("/icon", summary="站点图标", response_model=schemas.Response)
+async def site_icon(domain: str, _: User = Depends(get_current_active_user)) -> Any:
+    """
+    获取站点图标：base64或者url
+    """
+    icon = SiteIconOper().get_by_domain(domain)
+    if not icon:
+        return schemas.Response(success=False, message="站点图标不存在！")
+    return schemas.Response(success=True, data={
+        "icon": icon.base64 if icon.base64 else icon.url
+    })

app/chain/cookiecloud.py

@@ -5,6 +5,7 @@ from urllib.parse import urljoin
 from lxml import etree
 
 from app.chain import ChainBase
+from app.chain.site import SiteChain
 from app.core.config import settings
 from app.db.siteicon_oper import SiteIconOper
 from app.db.site_oper import SiteOper
@@ -24,6 +25,7 @@ class CookieCloudChain(ChainBase):
         self.siteoper = SiteOper()
         self.siteiconoper = SiteIconOper()
         self.siteshelper = SitesHelper()
+        self.sitechain = SiteChain()
         self.cookiecloud = CookieCloudHelper(
             server=settings.COOKIECLOUD_HOST,
             key=settings.COOKIECLOUD_KEY,
@@ -58,6 +60,11 @@ class CookieCloudChain(ChainBase):
             # 获取站点信息
             indexer = self.siteshelper.get_indexer(domain)
             if self.siteoper.exists(domain):
+                # 检查站点连通性
+                status, msg = self.sitechain.test(domain)
+                if status:
+                    logger.info(f"站点【{indexer.get('name')}】连通性正常，不同步CookieCloud数据")
+                    continue
                 # 更新站点Cookie
                 self.siteoper.update_cookie(domain=domain, cookies=cookie)
                 _update_count += 1
@@ -70,14 +77,17 @@ class CookieCloudChain(ChainBase):
                 _add_count += 1
             # 保存站点图标
             if indexer:
-                icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
+                site_icon = self.siteiconoper.get_by_domain(domain)
-                                                             cookie=cookie,
+                if not site_icon or not site_icon.base64:
-                                                             ua=settings.USER_AGENT)
+                    logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...")
-                if icon_url:
+                    icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
-                    self.siteiconoper.update_icon(name=indexer.get("name"),
+                                                                 cookie=cookie,
-                                                  domain=domain,
+                                                                 ua=settings.USER_AGENT)
-                                                  icon_url=icon_url,
+                    if icon_url:
-                                                  icon_base64=icon_base64)
+                        self.siteiconoper.update_icon(name=indexer.get("name"),
+                                                      domain=domain,
+                                                      icon_url=icon_url,
+                                                      icon_base64=icon_base64)
         # 处理完成
         ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点"
         logger.info(f"CookieCloud同步成功：{ret_msg}")

app/chain/site.py

@@ -4,8 +4,13 @@ from app.chain import ChainBase
 from app.core.config import settings
 from app.db.models.site import Site
 from app.db.site_oper import SiteOper
+from app.helper.browser import PlaywrightHelper
+from app.helper.cloudflare import under_challenge
 from app.helper.cookie import CookieHelper
 from app.log import logger
+from app.utils.http import RequestUtils
+from app.utils.site import SiteUtils
+from app.utils.string import StringUtils
 
 
 class SiteChain(ChainBase):
@@ -21,6 +26,60 @@ class SiteChain(ChainBase):
         self._siteoper = SiteOper()
         self._cookiehelper = CookieHelper()
 
+    def test(self, url: str) -> Tuple[bool, str]:
+        """
+        测试站点是否可用
+        :param url: 站点域名
+        :return: (是否可用, 错误信息)
+        """
+        # 检查域名是否可用
+        domain = StringUtils.get_url_domain(url)
+        site_info = self._siteoper.get_by_domain(domain)
+        if not site_info:
+            return False, f"站点【{url}】不存在"
+        site_url = site_info.url
+        site_cookie = site_info.cookie
+        ua = site_info.ua
+        render = site_info.render
+        proxies = settings.PROXY if site_info.proxy else None
+        proxy_server = settings.PROXY_SERVER if site_info.proxy else None
+        # 模拟登录
+        try:
+            # 访问链接
+            if render:
+                page_source = PlaywrightHelper().get_page_source(url=site_url,
+                                                                 cookies=site_cookie,
+                                                                 ua=ua,
+                                                                 proxies=proxy_server)
+                if not SiteUtils.is_logged_in(page_source):
+                    if under_challenge(page_source):
+                        return False, f"无法通过Cloudflare！"
+                    return False, f"仿真登录失败，Cookie已失效！"
+            else:
+                res = RequestUtils(cookies=site_cookie,
+                                   ua=ua,
+                                   proxies=proxies
+                                   ).get_res(url=site_url)
+                # 判断登录状态
+                if res and res.status_code in [200, 500, 403]:
+                    if not SiteUtils.is_logged_in(res.text):
+                        if under_challenge(res.text):
+                            msg = "站点被Cloudflare防护，请打开站点浏览器仿真"
+                        elif res.status_code == 200:
+                            msg = "Cookie已失效"
+                        else:
+                            msg = f"状态码：{res.status_code}"
+                        return False, f"连接失败，{msg}！"
+                    else:
+                        return True, f"连接成功"
+                elif res is not None:
+                    return False, f"连接失败，状态码：{res.status_code}！"
+                else:
+                    return False, f"连接失败，无法打开网站！"
+        except Exception as e:
+            return False, f"连接失败：{str(e)}！"
+        return True, "连接成功"
+
     def remote_list(self, userid: Union[str, int] = None):
         """
         查询所有站点，发送消息