codfrm/MoviePilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix 手动维护站点时缓存站点图标

Browse Source
This commit is contained in:
jxxghp 2024-02-13 10:18:27 +08:00
parent 928aaf0c19
commit c43a1411c9
4 changed files with 100 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/api/endpoints/site.py
View File

@ -54,6 +54,10 @@ def add_site(
site_in.id = None
site = Site(**site_in.dict())
site.create(db)
# 通知缓存站点图标
EventManager().send_event(EventType.CacheSiteIcon, {
"domain": domain
})
return schemas.Response(success=True)
@ -71,6 +75,10 @@ def update_site(
if not site:
return schemas.Response(success=False, message="站点不存在")
site.update(db, site_in.dict())
# 通知缓存站点图标
EventManager().send_event(EventType.CacheSiteIcon, {
"domain": site_in.domain
})
return schemas.Response(success=True)

58
app/chain/cookiecloud.py
View File

@ -1,20 +1,17 @@
import base64
from typing import Tuple, Optional
from urllib.parse import urljoin
from lxml import etree
from typing import Tuple
from app.chain import ChainBase
from app.chain.site import SiteChain
from app.core.config import settings
from app.core.event import EventManager
from app.db.site_oper import SiteOper
from app.db.siteicon_oper import SiteIconOper
from app.helper.cloudflare import under_challenge
from app.helper.cookiecloud import CookieCloudHelper
from app.helper.message import MessageHelper
from app.helper.rss import RssHelper
from app.helper.sites import SitesHelper
from app.log import logger
from app.schemas.types import EventType
from app.utils.http import RequestUtils
from app.utils.site import SiteUtils
@ -27,7 +24,6 @@ class CookieCloudChain(ChainBase):
def __init__(self):
super().__init__()
self.siteoper = SiteOper()
self.siteiconoper = SiteIconOper()
self.siteshelper = SitesHelper()
self.rsshelper = RssHelper()
self.sitechain = SiteChain()
@ -123,22 +119,11 @@ class CookieCloudChain(ChainBase):
public=1 if indexer.get("public") else 0)
_add_count += 1
# 保存站点图标
# 通知缓存站点图标
if indexer:
site_icon = self.siteiconoper.get_by_domain(domain)
if not site_icon or not site_icon.base64:
logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...")
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
cookie=cookie,
ua=settings.USER_AGENT)
if icon_url:
self.siteiconoper.update_icon(name=indexer.get("name"),
domain=domain,
icon_url=icon_url,
icon_base64=icon_base64)
logger.info(f"缓存站点 {indexer.get('name')} 图标成功")
else:
logger.warn(f"缓存站点 {indexer.get('name')} 图标失败")
EventManager().send_event(EventType.CacheSiteIcon, {
"domain": domain,
})
# 处理完成
ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点"
if _fail_count > 0:
@ -147,32 +132,3 @@ class CookieCloudChain(ChainBase):
self.message.put(f"CookieCloud同步成功, {ret_msg}")
logger.info(f"CookieCloud同步成功：{ret_msg}")
return True, ret_msg
@staticmethod
def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]:
"""
解析站点favicon,返回base64 fav图标
:param url: 站点地址
:param cookie: Cookie
:param ua: User-Agent
:return:
"""
favicon_url = urljoin(url, "favicon.ico")
res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url)
if res:
html_text = res.text
else:
logger.error(f"获取站点页面失败：{url}")
return favicon_url, None
html = etree.HTML(html_text)
if html:
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
if fav_link:
favicon_url = urljoin(url, fav_link[0])
res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url)
if res:
return favicon_url, base64.b64encode(res.content).decode()
else:
logger.error(f"获取站点图标失败：{favicon_url}")
return favicon_url, None

84
app/chain/site.py
View File

@ -1,16 +1,25 @@
import base64
import re
from typing import Union, Tuple
from typing import Tuple, Optional
from typing import Union
from urllib.parse import urljoin
from lxml import etree
from app.chain import ChainBase
from app.core.config import settings
from app.core.event import eventmanager, Event
from app.db.models.site import Site
from app.db.site_oper import SiteOper
from app.db.siteicon_oper import SiteIconOper
from app.helper.browser import PlaywrightHelper
from app.helper.cloudflare import under_challenge
from app.helper.cookie import CookieHelper
from app.helper.message import MessageHelper
from app.helper.sites import SitesHelper
from app.log import logger
from app.schemas import MessageChannel, Notification
from app.schemas.types import EventType
from app.utils.http import RequestUtils
from app.utils.site import SiteUtils
from app.utils.string import StringUtils
@ -24,6 +33,8 @@ class SiteChain(ChainBase):
def __init__(self):
super().__init__()
self.siteoper = SiteOper()
self.siteiconoper = SiteIconOper()
self.siteshelper = SitesHelper()
self.cookiehelper = CookieHelper()
self.message = MessageHelper()
@ -87,6 +98,77 @@ class SiteChain(ChainBase):
return True, "连接成功"
return False, "Cookie已失效"
@staticmethod
def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]:
"""
解析站点favicon,返回base64 fav图标
:param url: 站点地址
:param cookie: Cookie
:param ua: User-Agent
:return:
"""
favicon_url = urljoin(url, "favicon.ico")
res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url)
if res:
html_text = res.text
else:
logger.error(f"获取站点页面失败：{url}")
return favicon_url, None
html = etree.HTML(html_text)
if html:
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
if fav_link:
favicon_url = urljoin(url, fav_link[0])
res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url)
if res:
return favicon_url, base64.b64encode(res.content).decode()
else:
logger.error(f"获取站点图标失败：{favicon_url}")
return favicon_url, None
@eventmanager.register(EventType.CacheSiteIcon)
def cache_site_icon(self, event: Event):
"""
缓存站点图标
"""
if not event:
return
event_data = event.event_data or {}
# 主域名
domain = event_data.get("domain")
if not domain:
return
if str(domain).startswith("http"):
domain = StringUtils.get_url_domain(domain)
# 站点信息
siteinfo = self.siteoper.get_by_domain(domain)
if not siteinfo:
logger.warn(f"未维护站点 {domain} 信息！")
return
# Cookie
cookie = siteinfo.cookie
# 索引器
indexer = self.siteshelper.get_indexer(domain)
if not indexer:
logger.warn(f"站点 {domain} 索引器不存在！")
return
# 查询站点图标
site_icon = self.siteiconoper.get_by_domain(domain)
if not site_icon or not site_icon.base64:
logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...")
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
cookie=cookie,
ua=settings.USER_AGENT)
if icon_url:
self.siteiconoper.update_icon(name=indexer.get("name"),
domain=domain,
icon_url=icon_url,
icon_base64=icon_base64)
logger.info(f"缓存站点 {indexer.get('name')} 图标成功")
else:
logger.warn(f"缓存站点 {indexer.get('name')} 图标失败")
def test(self, url: str) -> Tuple[bool, str]:
"""
测试站点是否可用

2
app/schemas/types.py
View File

@ -40,6 +40,8 @@ class EventType(Enum):
NameRecognize = "name.recognize"
# 名称识别结果
NameRecognizeResult = "name.recognize.result"
# 缓存站点图标
CacheSiteIcon = "cache.siteicon"
# 系统配置Key字典