diff --git a/app/api/endpoints/site.py b/app/api/endpoints/site.py index c8023b0d..93a2d853 100644 --- a/app/api/endpoints/site.py +++ b/app/api/endpoints/site.py @@ -54,6 +54,10 @@ def add_site( site_in.id = None site = Site(**site_in.dict()) site.create(db) + # 通知缓存站点图标 + EventManager().send_event(EventType.CacheSiteIcon, { + "domain": domain + }) return schemas.Response(success=True) @@ -71,6 +75,10 @@ def update_site( if not site: return schemas.Response(success=False, message="站点不存在") site.update(db, site_in.dict()) + # 通知缓存站点图标 + EventManager().send_event(EventType.CacheSiteIcon, { + "domain": site_in.domain + }) return schemas.Response(success=True) diff --git a/app/chain/cookiecloud.py b/app/chain/cookiecloud.py index 5e65848d..dadf52f4 100644 --- a/app/chain/cookiecloud.py +++ b/app/chain/cookiecloud.py @@ -1,20 +1,17 @@ -import base64 -from typing import Tuple, Optional -from urllib.parse import urljoin - -from lxml import etree +from typing import Tuple from app.chain import ChainBase from app.chain.site import SiteChain from app.core.config import settings +from app.core.event import EventManager from app.db.site_oper import SiteOper -from app.db.siteicon_oper import SiteIconOper from app.helper.cloudflare import under_challenge from app.helper.cookiecloud import CookieCloudHelper from app.helper.message import MessageHelper from app.helper.rss import RssHelper from app.helper.sites import SitesHelper from app.log import logger +from app.schemas.types import EventType from app.utils.http import RequestUtils from app.utils.site import SiteUtils @@ -27,7 +24,6 @@ class CookieCloudChain(ChainBase): def __init__(self): super().__init__() self.siteoper = SiteOper() - self.siteiconoper = SiteIconOper() self.siteshelper = SitesHelper() self.rsshelper = RssHelper() self.sitechain = SiteChain() @@ -123,22 +119,11 @@ class CookieCloudChain(ChainBase): public=1 if indexer.get("public") else 0) _add_count += 1 - # 保存站点图标 + # 通知缓存站点图标 if indexer: - site_icon = self.siteiconoper.get_by_domain(domain) - if not site_icon or not site_icon.base64: - logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...") - icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"), - cookie=cookie, - ua=settings.USER_AGENT) - if icon_url: - self.siteiconoper.update_icon(name=indexer.get("name"), - domain=domain, - icon_url=icon_url, - icon_base64=icon_base64) - logger.info(f"缓存站点 {indexer.get('name')} 图标成功") - else: - logger.warn(f"缓存站点 {indexer.get('name')} 图标失败") + EventManager().send_event(EventType.CacheSiteIcon, { + "domain": domain, + }) # 处理完成 ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点" if _fail_count > 0: @@ -147,32 +132,3 @@ class CookieCloudChain(ChainBase): self.message.put(f"CookieCloud同步成功, {ret_msg}") logger.info(f"CookieCloud同步成功：{ret_msg}") return True, ret_msg - - @staticmethod - def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]: - """ - 解析站点favicon,返回base64 fav图标 - :param url: 站点地址 - :param cookie: Cookie - :param ua: User-Agent - :return: - """ - favicon_url = urljoin(url, "favicon.ico") - res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url) - if res: - html_text = res.text - else: - logger.error(f"获取站点页面失败：{url}") - return favicon_url, None - html = etree.HTML(html_text) - if html: - fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href') - if fav_link: - favicon_url = urljoin(url, fav_link[0]) - - res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url) - if res: - return favicon_url, base64.b64encode(res.content).decode() - else: - logger.error(f"获取站点图标失败：{favicon_url}") - return favicon_url, None diff --git a/app/chain/site.py b/app/chain/site.py index 68bb0b03..c913b72e 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -1,16 +1,25 @@ +import base64 import re -from typing import Union, Tuple +from typing import Tuple, Optional +from typing import Union +from urllib.parse import urljoin + +from lxml import etree from app.chain import ChainBase from app.core.config import settings +from app.core.event import eventmanager, Event from app.db.models.site import Site from app.db.site_oper import SiteOper +from app.db.siteicon_oper import SiteIconOper from app.helper.browser import PlaywrightHelper from app.helper.cloudflare import under_challenge from app.helper.cookie import CookieHelper from app.helper.message import MessageHelper +from app.helper.sites import SitesHelper from app.log import logger from app.schemas import MessageChannel, Notification +from app.schemas.types import EventType from app.utils.http import RequestUtils from app.utils.site import SiteUtils from app.utils.string import StringUtils @@ -24,6 +33,8 @@ class SiteChain(ChainBase): def __init__(self): super().__init__() self.siteoper = SiteOper() + self.siteiconoper = SiteIconOper() + self.siteshelper = SitesHelper() self.cookiehelper = CookieHelper() self.message = MessageHelper() @@ -87,6 +98,77 @@ class SiteChain(ChainBase): return True, "连接成功" return False, "Cookie已失效" + @staticmethod + def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]: + """ + 解析站点favicon,返回base64 fav图标 + :param url: 站点地址 + :param cookie: Cookie + :param ua: User-Agent + :return: + """ + favicon_url = urljoin(url, "favicon.ico") + res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url) + if res: + html_text = res.text + else: + logger.error(f"获取站点页面失败：{url}") + return favicon_url, None + html = etree.HTML(html_text) + if html: + fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href') + if fav_link: + favicon_url = urljoin(url, fav_link[0]) + + res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url) + if res: + return favicon_url, base64.b64encode(res.content).decode() + else: + logger.error(f"获取站点图标失败：{favicon_url}") + return favicon_url, None + + @eventmanager.register(EventType.CacheSiteIcon) + def cache_site_icon(self, event: Event): + """ + 缓存站点图标 + """ + if not event: + return + event_data = event.event_data or {} + # 主域名 + domain = event_data.get("domain") + if not domain: + return + if str(domain).startswith("http"): + domain = StringUtils.get_url_domain(domain) + # 站点信息 + siteinfo = self.siteoper.get_by_domain(domain) + if not siteinfo: + logger.warn(f"未维护站点 {domain} 信息！") + return + # Cookie + cookie = siteinfo.cookie + # 索引器 + indexer = self.siteshelper.get_indexer(domain) + if not indexer: + logger.warn(f"站点 {domain} 索引器不存在！") + return + # 查询站点图标 + site_icon = self.siteiconoper.get_by_domain(domain) + if not site_icon or not site_icon.base64: + logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...") + icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"), + cookie=cookie, + ua=settings.USER_AGENT) + if icon_url: + self.siteiconoper.update_icon(name=indexer.get("name"), + domain=domain, + icon_url=icon_url, + icon_base64=icon_base64) + logger.info(f"缓存站点 {indexer.get('name')} 图标成功") + else: + logger.warn(f"缓存站点 {indexer.get('name')} 图标失败") + def test(self, url: str) -> Tuple[bool, str]: """ 测试站点是否可用 diff --git a/app/schemas/types.py b/app/schemas/types.py index 02d866f7..ee64ed62 100644 --- a/app/schemas/types.py +++ b/app/schemas/types.py @@ -40,6 +40,8 @@ class EventType(Enum): NameRecognize = "name.recognize" # 名称识别结果 NameRecognizeResult = "name.recognize.result" + # 缓存站点图标 + CacheSiteIcon = "cache.siteicon" # 系统配置Key字典