diff --git a/app/chain/cookiecloud.py b/app/chain/cookiecloud.py index 622acf04..cbb0ee82 100644 --- a/app/chain/cookiecloud.py +++ b/app/chain/cookiecloud.py @@ -1,4 +1,8 @@ -from typing import Tuple +import base64 +from typing import Tuple, Optional +from urllib.parse import urljoin + +from lxml import etree from app.chain import ChainBase from app.core.config import settings @@ -6,6 +10,7 @@ from app.db.sites import Sites from app.helper.cookiecloud import CookieCloudHelper from app.helper.sites import SitesHelper from app.log import logger +from app.utils.http import RequestUtils class CookieCloudChain(ChainBase): @@ -36,20 +41,59 @@ class CookieCloudChain(ChainBase): _update_count = 0 _add_count = 0 for domain, cookie in cookies.items(): + # 获取站点信息 + indexer = self.siteshelper.get_indexer(domain) if self.sites.exists(domain): # 更新站点Cookie - self.sites.update_cookie(domain, cookie) + self.sites.update_cookie(domain=domain, cookies=cookie) _update_count += 1 - else: - # 获取站点信息 - indexer = self.siteshelper.get_indexer(domain) - if indexer: - # 新增站点 - self.sites.add(name=indexer.get("name"), - url=indexer.get("domain"), - domain=domain, - cookie=cookie) - _add_count += 1 + elif indexer: + # 新增站点 + self.sites.add(name=indexer.get("name"), + url=indexer.get("domain"), + domain=domain, + cookie=cookie) + _add_count += 1 + # 保存站点图标 + if indexer: + icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"), + cookie=cookie, + ua=settings.USER_AGENT) + if icon_url: + self.sites.update_icon(name=indexer.get("name"), + domain=domain, + icon_url=icon_url, + icon_base64=icon_base64) + # 处理完成 ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点" logger.info(f"CookieCloud同步成功：{ret_msg}") return True, ret_msg + + @staticmethod + def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]: + """ + 解析站点favicon,返回base64 fav图标 + :param url: 站点地址 + :param cookie: Cookie + :param ua: User-Agent + :return: + """ + favicon_url = urljoin(url, "favicon.ico") + res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url) + if res: + html_text = res.text + else: + logger.error(f"获取站点页面失败：{url}") + return favicon_url, None + html = etree.HTML(html_text) + if html: + fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href') + if fav_link: + favicon_url = urljoin(url, fav_link[0]) + + res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url) + if res: + return favicon_url, base64.b64encode(res.content).decode() + else: + logger.error(f"获取站点图标失败：{favicon_url}") + return favicon_url, None diff --git a/app/db/models/site.py b/app/db/models/site.py index eb3d2c72..a8f23526 100644 --- a/app/db/models/site.py +++ b/app/db/models/site.py @@ -35,3 +35,18 @@ class Site(Base): @staticmethod def get_actives(db: Session): return db.query(Site).filter(Site.is_active == 1).all() + + +class SiteIcon(Base): + """ + 站点图标表 + """ + id = Column(Integer, Sequence('id'), primary_key=True, index=True) + name = Column(String, nullable=False) + domain = Column(String, index=True) + url = Column(String, nullable=False) + base64 = Column(String) + + @staticmethod + def get_by_domain(db: Session, domain: str): + return db.query(SiteIcon).filter(SiteIcon.domain == domain).first() diff --git a/app/db/sites.py b/app/db/sites.py index 67a4f114..b86e690a 100644 --- a/app/db/sites.py +++ b/app/db/sites.py @@ -3,7 +3,7 @@ from typing import Tuple, List from sqlalchemy.orm import Session from app.db import SessionLocal -from app.db.models.site import Site +from app.db.models.site import Site, SiteIcon class Sites: @@ -60,3 +60,17 @@ class Sites: "cookie": cookies }) return True, "更新站点Cookie成功" + + def update_icon(self, name: str, domain: str, icon_url: str, icon_base64: str) -> bool: + """ + 更新站点图标 + """ + siteicon = SiteIcon(name=name, domain=domain, url=icon_url, base64=icon_base64) + if not siteicon.get_by_domain(self._db, domain): + siteicon.create(self._db) + elif icon_base64: + siteicon.update(self._db, { + "url": icon_url, + "base64": f"data:image/ico;base64,{icon_base64}" + }) + return True diff --git a/app/plugins/sitestatistic/siteuserinfo/__init__.py b/app/plugins/sitestatistic/siteuserinfo/__init__.py index 6d6b380e..c82dfdb7 100644 --- a/app/plugins/sitestatistic/siteuserinfo/__init__.py +++ b/app/plugins/sitestatistic/siteuserinfo/__init__.py @@ -38,7 +38,6 @@ class ISiteUserInfo(metaclass=ABCMeta): # 站点信息 self.site_name = None self.site_url = None - self.site_favicon = None # 用户信息 self.username = None self.userid = None @@ -92,8 +91,6 @@ class ISiteUserInfo(metaclass=ABCMeta): self.site_name = site_name self.site_url = url self._base_url = f"{split_url.scheme}://{split_url.netloc}" - self._favicon_url = urljoin(self._base_url, "favicon.ico") - self.site_favicon = "" self._site_cookie = site_cookie self._index_html = index_html self._session = session if session else requests.Session() @@ -123,7 +120,6 @@ class ISiteUserInfo(metaclass=ABCMeta): 解析站点信息 :return: """ - self._parse_favicon(self._index_html) if not self._parse_logged_in(self._index_html): return @@ -197,23 +193,6 @@ class ISiteUserInfo(metaclass=ABCMeta): """ pass - def _parse_favicon(self, html_text: str): - """ - 解析站点favicon,返回base64 fav图标 - :param html_text: - :return: - """ - html = etree.HTML(html_text) - if html: - fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href') - if fav_link: - self._favicon_url = urljoin(self._base_url, fav_link[0]) - - res = RequestUtils(cookies=self._site_cookie, session=self._session, timeout=60, headers=self._ua).get_res( - url=self._favicon_url) - if res: - self.site_favicon = base64.b64encode(res.content).decode() - def _get_page_content(self, url: str, params: dict = None, headers: dict = None): """ :param url: 网页地址