fix 手动维护站点时缓存站点图标
This commit is contained in:
jxxghp
parent
928aaf0c19
commit
c43a1411c9
@ -54,6 +54,10 @@ def add_site(
|
||||
site_in.id = None
|
||||
site = Site(**site_in.dict())
|
||||
site.create(db)
|
||||
# 通知缓存站点图标
|
||||
EventManager().send_event(EventType.CacheSiteIcon, {
|
||||
"domain": domain
|
||||
})
|
||||
return schemas.Response(success=True)
|
||||
|
||||
|
||||
@ -71,6 +75,10 @@ def update_site(
|
||||
if not site:
|
||||
return schemas.Response(success=False, message="站点不存在")
|
||||
site.update(db, site_in.dict())
|
||||
# 通知缓存站点图标
|
||||
EventManager().send_event(EventType.CacheSiteIcon, {
|
||||
"domain": site_in.domain
|
||||
})
|
||||
return schemas.Response(success=True)
|
||||
|
||||
|
||||
|
||||
@ -1,20 +1,17 @@
|
||||
import base64
|
||||
from typing import Tuple, Optional
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from lxml import etree
|
||||
from typing import Tuple
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.chain.site import SiteChain
|
||||
from app.core.config import settings
|
||||
from app.core.event import EventManager
|
||||
from app.db.site_oper import SiteOper
|
||||
from app.db.siteicon_oper import SiteIconOper
|
||||
from app.helper.cloudflare import under_challenge
|
||||
from app.helper.cookiecloud import CookieCloudHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.helper.rss import RssHelper
|
||||
from app.helper.sites import SitesHelper
|
||||
from app.log import logger
|
||||
from app.schemas.types import EventType
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.site import SiteUtils
|
||||
|
||||
@ -27,7 +24,6 @@ class CookieCloudChain(ChainBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.siteoper = SiteOper()
|
||||
self.siteiconoper = SiteIconOper()
|
||||
self.siteshelper = SitesHelper()
|
||||
self.rsshelper = RssHelper()
|
||||
self.sitechain = SiteChain()
|
||||
@ -123,22 +119,11 @@ class CookieCloudChain(ChainBase):
|
||||
public=1 if indexer.get("public") else 0)
|
||||
_add_count += 1
|
||||
|
||||
# 保存站点图标
|
||||
# 通知缓存站点图标
|
||||
if indexer:
|
||||
site_icon = self.siteiconoper.get_by_domain(domain)
|
||||
if not site_icon or not site_icon.base64:
|
||||
logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...")
|
||||
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
|
||||
cookie=cookie,
|
||||
ua=settings.USER_AGENT)
|
||||
if icon_url:
|
||||
self.siteiconoper.update_icon(name=indexer.get("name"),
|
||||
domain=domain,
|
||||
icon_url=icon_url,
|
||||
icon_base64=icon_base64)
|
||||
logger.info(f"缓存站点 {indexer.get('name')} 图标成功")
|
||||
else:
|
||||
logger.warn(f"缓存站点 {indexer.get('name')} 图标失败")
|
||||
EventManager().send_event(EventType.CacheSiteIcon, {
|
||||
"domain": domain,
|
||||
})
|
||||
# 处理完成
|
||||
ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点"
|
||||
if _fail_count > 0:
|
||||
@ -147,32 +132,3 @@ class CookieCloudChain(ChainBase):
|
||||
self.message.put(f"CookieCloud同步成功, {ret_msg}")
|
||||
logger.info(f"CookieCloud同步成功：{ret_msg}")
|
||||
return True, ret_msg
|
||||
|
||||
@staticmethod
|
||||
def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]:
|
||||
"""
|
||||
解析站点favicon,返回base64 fav图标
|
||||
:param url: 站点地址
|
||||
:param cookie: Cookie
|
||||
:param ua: User-Agent
|
||||
:return:
|
||||
"""
|
||||
favicon_url = urljoin(url, "favicon.ico")
|
||||
res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url)
|
||||
if res:
|
||||
html_text = res.text
|
||||
else:
|
||||
logger.error(f"获取站点页面失败：{url}")
|
||||
return favicon_url, None
|
||||
html = etree.HTML(html_text)
|
||||
if html:
|
||||
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
|
||||
if fav_link:
|
||||
favicon_url = urljoin(url, fav_link[0])
|
||||
|
||||
res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url)
|
||||
if res:
|
||||
return favicon_url, base64.b64encode(res.content).decode()
|
||||
else:
|
||||
logger.error(f"获取站点图标失败：{favicon_url}")
|
||||
return favicon_url, None
|
||||
|
||||
@ -1,16 +1,25 @@
|
||||
import base64
|
||||
import re
|
||||
from typing import Union, Tuple
|
||||
from typing import Tuple, Optional
|
||||
from typing import Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.core.config import settings
|
||||
from app.core.event import eventmanager, Event
|
||||
from app.db.models.site import Site
|
||||
from app.db.site_oper import SiteOper
|
||||
from app.db.siteicon_oper import SiteIconOper
|
||||
from app.helper.browser import PlaywrightHelper
|
||||
from app.helper.cloudflare import under_challenge
|
||||
from app.helper.cookie import CookieHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.helper.sites import SitesHelper
|
||||
from app.log import logger
|
||||
from app.schemas import MessageChannel, Notification
|
||||
from app.schemas.types import EventType
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.site import SiteUtils
|
||||
from app.utils.string import StringUtils
|
||||
@ -24,6 +33,8 @@ class SiteChain(ChainBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.siteoper = SiteOper()
|
||||
self.siteiconoper = SiteIconOper()
|
||||
self.siteshelper = SitesHelper()
|
||||
self.cookiehelper = CookieHelper()
|
||||
self.message = MessageHelper()
|
||||
|
||||
@ -87,6 +98,77 @@ class SiteChain(ChainBase):
|
||||
return True, "连接成功"
|
||||
return False, "Cookie已失效"
|
||||
|
||||
@staticmethod
|
||||
def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]:
|
||||
"""
|
||||
解析站点favicon,返回base64 fav图标
|
||||
:param url: 站点地址
|
||||
:param cookie: Cookie
|
||||
:param ua: User-Agent
|
||||
:return:
|
||||
"""
|
||||
favicon_url = urljoin(url, "favicon.ico")
|
||||
res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url)
|
||||
if res:
|
||||
html_text = res.text
|
||||
else:
|
||||
logger.error(f"获取站点页面失败：{url}")
|
||||
return favicon_url, None
|
||||
html = etree.HTML(html_text)
|
||||
if html:
|
||||
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
|
||||
if fav_link:
|
||||
favicon_url = urljoin(url, fav_link[0])
|
||||
|
||||
res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url)
|
||||
if res:
|
||||
return favicon_url, base64.b64encode(res.content).decode()
|
||||
else:
|
||||
logger.error(f"获取站点图标失败：{favicon_url}")
|
||||
return favicon_url, None
|
||||
|
||||
@eventmanager.register(EventType.CacheSiteIcon)
|
||||
def cache_site_icon(self, event: Event):
|
||||
"""
|
||||
缓存站点图标
|
||||
"""
|
||||
if not event:
|
||||
return
|
||||
event_data = event.event_data or {}
|
||||
# 主域名
|
||||
domain = event_data.get("domain")
|
||||
if not domain:
|
||||
return
|
||||
if str(domain).startswith("http"):
|
||||
domain = StringUtils.get_url_domain(domain)
|
||||
# 站点信息
|
||||
siteinfo = self.siteoper.get_by_domain(domain)
|
||||
if not siteinfo:
|
||||
logger.warn(f"未维护站点 {domain} 信息！")
|
||||
return
|
||||
# Cookie
|
||||
cookie = siteinfo.cookie
|
||||
# 索引器
|
||||
indexer = self.siteshelper.get_indexer(domain)
|
||||
if not indexer:
|
||||
logger.warn(f"站点 {domain} 索引器不存在！")
|
||||
return
|
||||
# 查询站点图标
|
||||
site_icon = self.siteiconoper.get_by_domain(domain)
|
||||
if not site_icon or not site_icon.base64:
|
||||
logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...")
|
||||
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
|
||||
cookie=cookie,
|
||||
ua=settings.USER_AGENT)
|
||||
if icon_url:
|
||||
self.siteiconoper.update_icon(name=indexer.get("name"),
|
||||
domain=domain,
|
||||
icon_url=icon_url,
|
||||
icon_base64=icon_base64)
|
||||
logger.info(f"缓存站点 {indexer.get('name')} 图标成功")
|
||||
else:
|
||||
logger.warn(f"缓存站点 {indexer.get('name')} 图标失败")
|
||||
|
||||
def test(self, url: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
测试站点是否可用
|
||||
|
||||
@ -40,6 +40,8 @@ class EventType(Enum):
|
||||
NameRecognize = "name.recognize"
|
||||
# 名称识别结果
|
||||
NameRecognizeResult = "name.recognize.result"
|
||||
# 缓存站点图标
|
||||
CacheSiteIcon = "cache.siteicon"
|
||||
|
||||
|
||||
# 系统配置Key字典
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user