add 站点图标缓存

app/chain/cookiecloud.py

@@ -1,4 +1,8 @@
-from typing import Tuple
+import base64
+from typing import Tuple, Optional
+from urllib.parse import urljoin
+
+from lxml import etree
 
 from app.chain import ChainBase
 from app.core.config import settings
@@ -6,6 +10,7 @@ from app.db.sites import Sites
 from app.helper.cookiecloud import CookieCloudHelper
 from app.helper.sites import SitesHelper
 from app.log import logger
+from app.utils.http import RequestUtils
 
 
 class CookieCloudChain(ChainBase):
@@ -36,20 +41,59 @@ class CookieCloudChain(ChainBase):
         _update_count = 0
         _add_count = 0
         for domain, cookie in cookies.items():
-            if self.sites.exists(domain):
-                # 更新站点Cookie
-                self.sites.update_cookie(domain, cookie)
-                _update_count += 1
-            else:
             # 获取站点信息
             indexer = self.siteshelper.get_indexer(domain)
-                if indexer:
+            if self.sites.exists(domain):
+                # 更新站点Cookie
+                self.sites.update_cookie(domain=domain, cookies=cookie)
+                _update_count += 1
+            elif indexer:
                 # 新增站点
                 self.sites.add(name=indexer.get("name"),
                                url=indexer.get("domain"),
                                domain=domain,
                                cookie=cookie)
                 _add_count += 1
+            # 保存站点图标
+            if indexer:
+                icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
+                                                             cookie=cookie,
+                                                             ua=settings.USER_AGENT)
+                if icon_url:
+                    self.sites.update_icon(name=indexer.get("name"),
+                                           domain=domain,
+                                           icon_url=icon_url,
+                                           icon_base64=icon_base64)
+        # 处理完成
         ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点"
         logger.info(f"CookieCloud同步成功：{ret_msg}")
         return True, ret_msg
+
+    @staticmethod
+    def __parse_favicon(url: str, cookie: str, ua: str) -> Tuple[str, Optional[str]]:
+        """
+        解析站点favicon,返回base64 fav图标
+        :param url: 站点地址
+        :param cookie: Cookie
+        :param ua: User-Agent
+        :return:
+        """
+        favicon_url = urljoin(url, "favicon.ico")
+        res = RequestUtils(cookies=cookie, timeout=60, ua=ua).get_res(url=url)
+        if res:
+            html_text = res.text
+        else:
+            logger.error(f"获取站点页面失败：{url}")
+            return favicon_url, None
+        html = etree.HTML(html_text)
+        if html:
+            fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
+            if fav_link:
+                favicon_url = urljoin(url, fav_link[0])
+
+        res = RequestUtils(cookies=cookie, timeout=20, ua=ua).get_res(url=favicon_url)
+        if res:
+            return favicon_url, base64.b64encode(res.content).decode()
+        else:
+            logger.error(f"获取站点图标失败：{favicon_url}")
+        return favicon_url, None

app/db/models/site.py

@@ -35,3 +35,18 @@ class Site(Base):
     @staticmethod
     def get_actives(db: Session):
         return db.query(Site).filter(Site.is_active == 1).all()
+
+
+class SiteIcon(Base):
+    """
+    站点图标表
+    """
+    id = Column(Integer, Sequence('id'), primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    domain = Column(String, index=True)
+    url = Column(String, nullable=False)
+    base64 = Column(String)
+
+    @staticmethod
+    def get_by_domain(db: Session, domain: str):
+        return db.query(SiteIcon).filter(SiteIcon.domain == domain).first()

app/db/sites.py

@@ -3,7 +3,7 @@ from typing import Tuple, List
 from sqlalchemy.orm import Session
 
 from app.db import SessionLocal
-from app.db.models.site import Site
+from app.db.models.site import Site, SiteIcon
 
 
 class Sites:
@@ -60,3 +60,17 @@ class Sites:
             "cookie": cookies
         })
         return True, "更新站点Cookie成功"
+
+    def update_icon(self, name: str, domain: str, icon_url: str, icon_base64: str) -> bool:
+        """
+        更新站点图标
+        """
+        siteicon = SiteIcon(name=name, domain=domain, url=icon_url, base64=icon_base64)
+        if not siteicon.get_by_domain(self._db, domain):
+            siteicon.create(self._db)
+        elif icon_base64:
+            siteicon.update(self._db, {
+                "url": icon_url,
+                "base64": f"data:image/ico;base64,{icon_base64}"
+            })
+        return True

app/plugins/sitestatistic/siteuserinfo/__init__.py

@@ -38,7 +38,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
         # 站点信息
         self.site_name = None
         self.site_url = None
-        self.site_favicon = None
         # 用户信息
         self.username = None
         self.userid = None
@@ -92,8 +91,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
         self.site_name = site_name
         self.site_url = url
         self._base_url = f"{split_url.scheme}://{split_url.netloc}"
-        self._favicon_url = urljoin(self._base_url, "favicon.ico")
-        self.site_favicon = ""
         self._site_cookie = site_cookie
         self._index_html = index_html
         self._session = session if session else requests.Session()
@@ -123,7 +120,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
         解析站点信息
         :return:
         """
-        self._parse_favicon(self._index_html)
         if not self._parse_logged_in(self._index_html):
             return
 
@@ -197,23 +193,6 @@ class ISiteUserInfo(metaclass=ABCMeta):
         """
         pass
 
-    def _parse_favicon(self, html_text: str):
-        """
-        解析站点favicon,返回base64 fav图标
-        :param html_text:
-        :return:
-        """
-        html = etree.HTML(html_text)
-        if html:
-            fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
-            if fav_link:
-                self._favicon_url = urljoin(self._base_url, fav_link[0])
-
-        res = RequestUtils(cookies=self._site_cookie, session=self._session, timeout=60, headers=self._ua).get_res(
-            url=self._favicon_url)
-        if res:
-            self.site_favicon = base64.b64encode(res.content).decode()
-
     def _get_page_content(self, url: str, params: dict = None, headers: dict = None):
         """
         :param url: 网页地址