From 35f5f46873a58bcb35e355708759606bb30574ac Mon Sep 17 00:00:00 2001 From: jxxghp Date: Tue, 20 Jun 2023 07:15:10 +0800 Subject: [PATCH] =?UTF-8?q?add=20=E7=AB=99=E7=82=B9=E8=BF=9E=E9=80=9A?= =?UTF-8?q?=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/endpoints/site.py | 23 +++++++++++++++ app/chain/cookiecloud.py | 26 +++++++++++------ app/chain/site.py | 59 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/app/api/endpoints/site.py b/app/api/endpoints/site.py index 60f23a31..7e95f971 100644 --- a/app/api/endpoints/site.py +++ b/app/api/endpoints/site.py @@ -9,6 +9,7 @@ from app.chain.site import SiteChain from app.db import get_db from app.db.models.site import Site from app.db.models.user import User +from app.db.siteicon_oper import SiteIconOper from app.db.userauth import get_current_active_user, get_current_active_superuser router = APIRouter() @@ -97,3 +98,25 @@ async def update_cookie( return schemas.Response(success=False, message=msg) else: return schemas.Response(success=True, message=msg) + + +@router.get("/test", summary="连接测试", response_model=schemas.Response) +async def test_site(domain: str, _: User = Depends(get_current_active_user)) -> Any: + """ + 测试站点是否可用 + """ + status, message = SiteChain().test(domain) + return schemas.Response(success=status, message=message) + + +@router.get("/icon", summary="站点图标", response_model=schemas.Response) +async def site_icon(domain: str, _: User = Depends(get_current_active_user)) -> Any: + """ + 获取站点图标：base64或者url + """ + icon = SiteIconOper().get_by_domain(domain) + if not icon: + return schemas.Response(success=False, message="站点图标不存在！") + return schemas.Response(success=True, data={ + "icon": icon.base64 if icon.base64 else icon.url + }) diff --git a/app/chain/cookiecloud.py b/app/chain/cookiecloud.py index f63db85b..264aa5f2 100644 --- a/app/chain/cookiecloud.py +++ b/app/chain/cookiecloud.py @@ -5,6 +5,7 @@ from urllib.parse import urljoin from lxml import etree from app.chain import ChainBase +from app.chain.site import SiteChain from app.core.config import settings from app.db.siteicon_oper import SiteIconOper from app.db.site_oper import SiteOper @@ -24,6 +25,7 @@ class CookieCloudChain(ChainBase): self.siteoper = SiteOper() self.siteiconoper = SiteIconOper() self.siteshelper = SitesHelper() + self.sitechain = SiteChain() self.cookiecloud = CookieCloudHelper( server=settings.COOKIECLOUD_HOST, key=settings.COOKIECLOUD_KEY, @@ -58,6 +60,11 @@ class CookieCloudChain(ChainBase): # 获取站点信息 indexer = self.siteshelper.get_indexer(domain) if self.siteoper.exists(domain): + # 检查站点连通性 + status, msg = self.sitechain.test(domain) + if status: + logger.info(f"站点【{indexer.get('name')}】连通性正常，不同步CookieCloud数据") + continue # 更新站点Cookie self.siteoper.update_cookie(domain=domain, cookies=cookie) _update_count += 1 @@ -70,14 +77,17 @@ class CookieCloudChain(ChainBase): _add_count += 1 # 保存站点图标 if indexer: - icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"), - cookie=cookie, - ua=settings.USER_AGENT) - if icon_url: - self.siteiconoper.update_icon(name=indexer.get("name"), - domain=domain, - icon_url=icon_url, - icon_base64=icon_base64) + site_icon = self.siteiconoper.get_by_domain(domain) + if not site_icon or not site_icon.base64: + logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...") + icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"), + cookie=cookie, + ua=settings.USER_AGENT) + if icon_url: + self.siteiconoper.update_icon(name=indexer.get("name"), + domain=domain, + icon_url=icon_url, + icon_base64=icon_base64) # 处理完成 ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点" logger.info(f"CookieCloud同步成功：{ret_msg}") diff --git a/app/chain/site.py b/app/chain/site.py index 800bd56e..ad983d58 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -4,8 +4,13 @@ from app.chain import ChainBase from app.core.config import settings from app.db.models.site import Site from app.db.site_oper import SiteOper +from app.helper.browser import PlaywrightHelper +from app.helper.cloudflare import under_challenge from app.helper.cookie import CookieHelper from app.log import logger +from app.utils.http import RequestUtils +from app.utils.site import SiteUtils +from app.utils.string import StringUtils class SiteChain(ChainBase): @@ -21,6 +26,60 @@ class SiteChain(ChainBase): self._siteoper = SiteOper() self._cookiehelper = CookieHelper() + def test(self, url: str) -> Tuple[bool, str]: + """ + 测试站点是否可用 + :param url: 站点域名 + :return: (是否可用, 错误信息) + """ + # 检查域名是否可用 + domain = StringUtils.get_url_domain(url) + site_info = self._siteoper.get_by_domain(domain) + if not site_info: + return False, f"站点【{url}】不存在" + site_url = site_info.url + site_cookie = site_info.cookie + ua = site_info.ua + render = site_info.render + proxies = settings.PROXY if site_info.proxy else None + proxy_server = settings.PROXY_SERVER if site_info.proxy else None + # 模拟登录 + try: + # 访问链接 + if render: + page_source = PlaywrightHelper().get_page_source(url=site_url, + cookies=site_cookie, + ua=ua, + proxies=proxy_server) + if not SiteUtils.is_logged_in(page_source): + if under_challenge(page_source): + return False, f"无法通过Cloudflare！" + return False, f"仿真登录失败，Cookie已失效！" + else: + res = RequestUtils(cookies=site_cookie, + ua=ua, + proxies=proxies + ).get_res(url=site_url) + # 判断登录状态 + if res and res.status_code in [200, 500, 403]: + if not SiteUtils.is_logged_in(res.text): + if under_challenge(res.text): + msg = "站点被Cloudflare防护，请打开站点浏览器仿真" + elif res.status_code == 200: + msg = "Cookie已失效" + else: + msg = f"状态码：{res.status_code}" + return False, f"连接失败，{msg}！" + else: + return True, f"连接成功" + elif res is not None: + return False, f"连接失败，状态码：{res.status_code}！" + else: + return False, f"连接失败，无法打开网站！" + except Exception as e: + return False, f"连接失败：{str(e)}！" + return True, "连接成功" + def remote_list(self, userid: Union[str, int] = None): """ 查询所有站点，发送消息