codfrm/MoviePilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add 站点图标缓存

Browse Source
This commit is contained in:
jxxghp 2023-06-12 09:00:59 +08:00
parent 7345cd237e
commit 6e8b687545
4 changed files with 86 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
app/chain/cookiecloud.py
View File

@ -1,4 +1,8 @@
from typing import Tuple
import base64
from typing import Tuple, Optional
from urllib.parse import urljoin
from lxml import etree
from app.chain import ChainBase
from app.core.config import settings
@ -6,6 +10,7 @@ from app.db.sites import Sites
from app.helper.cookiecloud import CookieCloudHelper
from app.helper.sites import SitesHelper
from app.log import logger
from app.utils.http import RequestUtils
class CookieCloudChain(ChainBase):
@ -36,20 +41,59 @@ class CookieCloudChain(ChainBase):
_update_count = 0
_add_count = 0
for domain, cookie in cookies.items():
# 获取站点信息
indexer = self.siteshelper.get_indexer(domain)
if self.sites.exists(domain):
# 更新站点Cookie
self.sites.update_cookie(domain, cookie)
self.sites.update_cookie(domain=domain, cookies=cookie)
_update_count += 1
else:
# 获取站点信息
indexer = self.siteshelper.get_indexer(domain)
if indexer:
# 新增站点
self.sites.add(name=indexer.get("name"),
url=indexer.get("domain"),
domain=domain,
cookie=cookie)
_add_count += 1
elif indexer:
# 新增站点
self.sites.add(name=indexer.get("name"),
url=indexer.get("domain"),
domain=domain,
cookie=cookie)
_add_count += 1
# 保存站点图标
if indexer:
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
cookie=cookie,
ua=settings.USER_AGENT)
if icon_url:
self.sites.update_icon(name=indexer.get("name"),
domain=domain,
icon_url=icon_url,
icon_base64=icon_base64)
# 处理完成
ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点"
logger.info(f"CookieCloud同步成功：{ret_msg}")
return True, ret_msg
@staticmethod
def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]:
"""
解析站点favicon,返回base64 fav图标
:param url: 站点地址
:param cookie: Cookie
:param ua: User-Agent
:return:
"""
favicon_url = urljoin(url, "favicon.ico")
res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url)
if res:
html_text = res.text
else:
logger.error(f"获取站点页面失败：{url}")
return favicon_url, None
html = etree.HTML(html_text)
if html:
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
if fav_link:
favicon_url = urljoin(url, fav_link[0])
res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url)
if res:
return favicon_url, base64.b64encode(res.content).decode()
else:
logger.error(f"获取站点图标失败：{favicon_url}")
return favicon_url, None

15
app/db/models/site.py
View File

@ -35,3 +35,18 @@ class Site(Base):
@staticmethod
def get_actives(db: Session):
return db.query(Site).filter(Site.is_active == 1).all()
class SiteIcon(Base):
"""
站点图标表
"""
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
name = Column(String, nullable=False)
domain = Column(String, index=True)
url = Column(String, nullable=False)
base64 = Column(String)
@staticmethod
def get_by_domain(db: Session, domain: str):
return db.query(SiteIcon).filter(SiteIcon.domain == domain).first()

16
app/db/sites.py
View File

@ -3,7 +3,7 @@ from typing import Tuple, List
from sqlalchemy.orm import Session
from app.db import SessionLocal
from app.db.models.site import Site
from app.db.models.site import Site, SiteIcon
class Sites:
@ -60,3 +60,17 @@ class Sites:
"cookie": cookies
})
return True, "更新站点Cookie成功"
def update_icon(self, name: str, domain: str, icon_url: str, icon_base64: str) -> bool:
"""
更新站点图标
"""
siteicon = SiteIcon(name=name, domain=domain, url=icon_url, base64=icon_base64)
if not siteicon.get_by_domain(self._db, domain):
siteicon.create(self._db)
elif icon_base64:
siteicon.update(self._db, {
"url": icon_url,
"base64": f"data:image/ico;base64,{icon_base64}"
})
return True

21
app/plugins/sitestatistic/siteuserinfo/__init__.py
View File

@ -38,7 +38,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
# 站点信息
self.site_name = None
self.site_url = None
self.site_favicon = None
# 用户信息
self.username = None
self.userid = None
@ -92,8 +91,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
self.site_name = site_name
self.site_url = url
self._base_url = f"{split_url.scheme}://{split_url.netloc}"
self._favicon_url = urljoin(self._base_url, "favicon.ico")
self.site_favicon = ""
self._site_cookie = site_cookie
self._index_html = index_html
self._session = session if session else requests.Session()
@ -123,7 +120,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
解析站点信息
:return:
"""
self._parse_favicon(self._index_html)
if not self._parse_logged_in(self._index_html):
return
@ -197,23 +193,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
"""
pass
def _parse_favicon(self, html_text: str):
"""
解析站点favicon,返回base64 fav图标
:param html_text:
:return:
"""
html = etree.HTML(html_text)
if html:
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
if fav_link:
self._favicon_url = urljoin(self._base_url, fav_link[0])
res = RequestUtils(cookies=self._site_cookie, session=self._session, timeout=60, headers=self._ua).get_res(
url=self._favicon_url)
if res:
self.site_favicon = base64.b64encode(res.content).decode()
def _get_page_content(self, url: str, params: dict = None, headers: dict = None):
"""
:param url: 网页地址