fix plugins

app/db/init.py

@@ -1,3 +1,6 @@
+import importlib
+from pathlib import Path
+
 from alembic.command import upgrade
 from alembic.config import Config
 
@@ -13,6 +16,10 @@ def init_db():
     """
     初始化数据库
     """
+    # 导入模块，避免建表缺失
+    for module in Path(__file__).with_name("models").glob("*.py"):
+        importlib.import_module(f"app.db.models.{module.stem}")
+    # 全量建表
     Base.metadata.create_all(bind=Engine)
     # 初始化超级管理员
     _db = SessionLocal()

app/db/models/plugin.py

@@ -0,0 +1,22 @@
+from sqlalchemy import Column, Integer, String, Sequence
+from sqlalchemy.orm import Session
+
+from app.db.models import Base
+
+
+class PluginData(Base):
+    """
+    插件数据表
+    """
+    id = Column(Integer, Sequence('id'), primary_key=True, index=True)
+    plugin_id = Column(String, nullable=False, index=True)
+    key = Column(String, index=True, nullable=False)
+    value = Column(String)
+
+    @staticmethod
+    def get_plugin_data(db: Session, plugin_id: str):
+        return db.query(PluginData).filter(PluginData.plugin_id == plugin_id).all()
+
+    @staticmethod
+    def get_plugin_data_by_key(db: Session, plugin_id: str, key: str):
+        return db.query(PluginData.value).filter(PluginData.plugin_id == plugin_id, PluginData.key == key).first()

app/db/models/site.py

@@ -7,6 +7,9 @@ from app.db.models import Base
 
 
 class Site(Base):
+    """
+    站点表
+    """
     id = Column(Integer, Sequence('id'), primary_key=True, index=True)
     name = Column(String, nullable=False)
     domain = Column(String, index=True)

app/db/models/subscribe.py

@@ -5,6 +5,9 @@ from app.db.models import Base
 
 
 class Subscribe(Base):
+    """
+    订阅表
+    """
     id = Column(Integer, Sequence('id'), primary_key=True, index=True)
     name = Column(String, nullable=False, index=True)
     year = Column(String)

app/db/models/systemconfig.py

@@ -5,6 +5,9 @@ from app.db.models import Base
 
 
 class SystemConfig(Base):
+    """
+    配置表
+    """
     id = Column(Integer, Sequence('id'), primary_key=True, index=True)
     key = Column(String, index=True)
     value = Column(String, nullable=True)

app/db/models/user.py

@@ -6,6 +6,9 @@ from app.db.models import Base
 
 
 class User(Base):
+    """
+    用户表
+    """
     id = Column(Integer, Sequence('id'), primary_key=True, index=True)
     full_name = Column(String, index=True)
     email = Column(String, unique=True, index=True, nullable=False)

app/main.py

@@ -24,7 +24,9 @@ def shutdown_server():
     """
     服务关闭
     """
+    # 停止定时服务
     Scheduler().stop()
+    # 停止插件
     PluginManager().stop()
 
 

app/plugins/__init__.py

@@ -1,10 +1,15 @@
+import json
 from abc import ABCMeta, abstractmethod
 from pathlib import Path
 from typing import Any, Optional
 
 from app.chain import ChainBase
 from app.core import settings, Context
+from app.db import SessionLocal
+from app.db.models import Base
+from app.db.models.plugin import PluginData
 from app.db.systemconfigs import SystemConfigs
+from app.utils.object import ObjectUtils
 
 
 class PluginChian(ChainBase):
@@ -34,6 +39,7 @@ class _PluginBase(metaclass=ABCMeta):
     plugin_desc: str = ""
     
     def __init__(self):
+        self.db = SessionLocal()
         self.chain = PluginChian()
 
     @abstractmethod
@@ -80,3 +86,24 @@ class _PluginBase(metaclass=ABCMeta):
         if not data_path.exists():
             data_path.mkdir(parents=True)
         return data_path
+
+    def save_data(self, key: str, value: Any) -> Base:
+        """
+        保存插件数据
+        :param key: 数据key
+        :param value: 数据值
+        """
+        if ObjectUtils.is_obj(value):
+            value = json.dumps(value)
+        plugin = PluginData(plugin_id=self.__class__.__name__, key=key, value=value)
+        return plugin.create(self.db)
+
+    def get_data(self, key: str) -> Any:
+        """
+        获取插件数据
+        :param key: 数据key
+        """
+        data = PluginData.get_plugin_data_by_key(self.db, self.__class__.__name__, key)
+        if ObjectUtils.is_obj(data):
+            return json.load(data)
+        return data

app/plugins/autosignin/__init__.py

@@ -5,7 +5,6 @@ from typing import Any
 from urllib.parse import urljoin
 
 from apscheduler.schedulers.background import BackgroundScheduler
-from lxml import etree
 from ruamel.yaml import CommentedMap
 
 from app.core import EventManager, settings, eventmanager
@@ -15,6 +14,7 @@ from app.helper.sites import SitesHelper
 from app.log import logger
 from app.plugins import _PluginBase
 from app.utils.http import RequestUtils
+from app.utils.site import SiteUtils
 from app.utils.timer import TimerUtils
 from app.utils.types import EventType
 
@@ -81,6 +81,8 @@ class AutoSignIn(_PluginBase):
         """
         自动签到
         """
+        if event:
+            logger.info("收到远程签到命令，开始执行签到任务 ...")
         # 查询签到站点
         sign_sites = self.sites.get_indexers()
         if not sign_sites:
@@ -123,7 +125,8 @@ class AutoSignIn(_PluginBase):
         else:
             return self.__signin_base(site_info)
 
-    def __signin_base(self, site_info: CommentedMap) -> str:
+    @staticmethod
+    def __signin_base(site_info: CommentedMap) -> str:
         """
         通用签到处理
         :param site_info: 站点信息
@@ -158,7 +161,7 @@ class AutoSignIn(_PluginBase):
                                    ).get_res(url=site_url)
             # 判断登录状态
             if res and res.status_code in [200, 500, 403]:
-                if not self.is_logged_in(res.text):
+                if not SiteUtils.is_logged_in(res.text):
                     if under_challenge(res.text):
                         msg = "站点被Cloudflare防护，请更换Cookie和UA！"
                     elif res.status_code == 200:
@@ -194,32 +197,3 @@ class AutoSignIn(_PluginBase):
                 self._scheduler = None
         except Exception as e:
             logger.error("退出插件失败：%s" % str(e))
-
-    @classmethod
-    def is_logged_in(cls, html_text: str) -> bool:
-        """
-        判断站点是否已经登陆
-        :param html_text:
-        :return:
-        """
-        html = etree.HTML(html_text)
-        if not html:
-            return False
-        # 存在明显的密码输入框，说明未登录
-        if html.xpath("//input[@type='password']"):
-            return False
-        # 是否存在登出和用户面板等链接
-        xpaths = ['//a[contains(@href, "logout")'
-                  ' or contains(@data-url, "logout")'
-                  ' or contains(@href, "mybonus") '
-                  ' or contains(@onclick, "logout")'
-                  ' or contains(@href, "usercp")]',
-                  '//form[contains(@action, "logout")]']
-        for xpath in xpaths:
-            if html.xpath(xpath):
-                return True
-        user_info_div = html.xpath('//div[@class="user-info-side"]')
-        if user_info_div:
-            return True
-
-        return False

app/plugins/sitestatistic/__init__.py

@@ -1,9 +1,10 @@
 from datetime import datetime
 from multiprocessing.dummy import Pool as ThreadPool
 from threading import Lock
-from typing import Optional, Any
+from typing import Optional, Any, List
 
 import requests
+from apscheduler.schedulers.background import BackgroundScheduler
 from ruamel.yaml import CommentedMap
 
 from app.core import settings
@@ -11,23 +12,28 @@ from app.helper import ModuleHelper
 from app.helper.sites import SitesHelper
 from app.log import logger
 from app.plugins import _PluginBase
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo
 from app.utils.http import RequestUtils
+from app.utils.timer import TimerUtils
+
+import warnings
+warnings.filterwarnings("ignore", category=FutureWarning)
+
 
 lock = Lock()
 
 
-class SiteStatistics(_PluginBase):
+class SiteStatistic(_PluginBase):
     sites = None
-
+    _scheduler: BackgroundScheduler = None
     _MAX_CONCURRENCY: int = 10
     _last_update_time: Optional[datetime] = None
     _sites_data: dict = {}
-    _site_schema: list = None
+    _site_schema: List[ISiteUserInfo] = None
 
     def init_plugin(self, config: dict = None):
         # 加载模块
-        self._site_schema = ModuleHelper.load('app.plugins.sitestatistics.siteuserinfo',
+        self._site_schema = ModuleHelper.load('app.plugins.sitestatistic.siteuserinfo',
                                               filter_func=lambda _, obj: hasattr(obj, 'schema'))
         self._site_schema.sort(key=lambda x: x.order)
         # 站点管理
@@ -36,6 +42,20 @@ class SiteStatistics(_PluginBase):
         self._last_update_time = None
         # 站点数据
         self._sites_data = {}
+        # 定时服务
+        self._scheduler = BackgroundScheduler(timezone=settings.TZ)
+        triggers = TimerUtils.random_scheduler(num_executions=1,
+                                               begin_hour=0,
+                                               end_hour=1,
+                                               min_interval=1,
+                                               max_interval=60)
+        for trigger in triggers:
+            self._scheduler.add_job(self.refresh_all_site_data, "cron", hour=trigger.hour, minute=trigger.minute)
+
+        # 启动任务
+        if self._scheduler.get_jobs():
+            self._scheduler.print_jobs()
+            self._scheduler.start()
 
     def stop_service(self):
         pass
@@ -46,13 +66,16 @@ class SiteStatistics(_PluginBase):
                 if site_schema.match(html_text):
                     return site_schema
             except Exception as e:
-                logger.error(f"站点 {site_schema.name} 匹配失败 {e}")
+                logger.error(f"站点匹配失败 {e}")
         return None
 
     def build(self, url: str, site_name: str,
               site_cookie: str = None,
               ua: str = None,
-              proxy: bool = False) -> Any:
+              proxy: bool = False) -> Optional[ISiteUserInfo]:
+        """
+        构建站点信息
+        """
         if not site_cookie:
             return None
         session = requests.Session()
@@ -121,26 +144,26 @@ class SiteStatistics(_PluginBase):
             return None
         return site_schema(site_name, url, site_cookie, html_text, session=session, ua=ua, proxy=proxy)
 
-    def __refresh_site_data(self, site_info: CommentedMap):
+    def __refresh_site_data(self, site_info: CommentedMap) -> Optional[ISiteUserInfo]:
         """
         更新单个site 数据信息
         :param site_info:
         :return:
         """
         site_name = site_info.get("name")
-        site_url = site_info.get("strict_url")
+        site_url = site_info.get("url")
         if not site_url:
-            return
+            return None
         site_cookie = site_info.get("cookie")
         ua = site_info.get("ua")
         unread_msg_notify = True
         proxy = site_info.get("proxy")
         try:
-            site_user_info = self.build(url=site_url,
-                                        site_name=site_name,
-                                        site_cookie=site_cookie,
-                                        ua=ua,
-                                        proxy=proxy)
+            site_user_info: ISiteUserInfo = self.build(url=site_url,
+                                                       site_name=site_name,
+                                                       site_cookie=site_cookie,
+                                                       ua=ua,
+                                                       proxy=proxy)
             if site_user_info:
                 logger.debug(f"站点 {site_name} 开始以 {site_user_info.site_schema()} 模型解析")
                 # 开始解析
@@ -150,7 +173,7 @@ class SiteStatistics(_PluginBase):
                 # 获取不到数据时，仅返回错误信息，不做历史数据更新
                 if site_user_info.err_msg:
                     self._sites_data.update({site_name: {"err_msg": site_user_info.err_msg}})
-                    return
+                    return None
 
                 # 发送通知，存在未读消息
                 self.__notify_unread_msg(site_name, site_user_info, unread_msg_notify)
@@ -173,11 +196,11 @@ class SiteStatistics(_PluginBase):
                             "message_unread": site_user_info.message_unread
                         }
                     })
-
                 return site_user_info
 
         except Exception as e:
             logger.error(f"站点 {site_name} 获取流量数据失败：{str(e)}")
+        return None
 
     def __notify_unread_msg(self, site_name: str, site_user_info: ISiteUserInfo, unread_msg_notify: bool):
         if site_user_info.message_unread <= 0:
@@ -228,35 +251,15 @@ class SiteStatistics(_PluginBase):
 
             # 并发刷新
             with ThreadPool(min(len(refresh_sites), self._MAX_CONCURRENCY)) as p:
-                site_user_infos = p.map(self.__refresh_site_data, refresh_sites)
+                site_user_infos: List[ISiteUserInfo] = p.map(self.__refresh_site_data, refresh_sites)
                 site_user_infos = [info for info in site_user_infos if info]
-
-            print(site_user_infos)
-            # TODO 登记历史数据
-            # TODO 实时用户数据
-            # TODO 更新站点图标
-            # TODO 实时做种信息
+            # 保存数据
+            for site_user_info in site_user_infos:
+                # 获取今天的日期
+                key = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                value = site_user_info.to_dict()
+                # 按日期保存为字典
+                self.save_data(key, value)
 
             # 更新时间
             self._last_update_time = datetime.now()
-
-    @staticmethod
-    def __todict(raw_statistics):
-        statistics = []
-        for site in raw_statistics:
-            statistics.append({"site": site.SITE,
-                               "username": site.USERNAME,
-                               "user_level": site.USER_LEVEL,
-                               "join_at": site.JOIN_AT,
-                               "update_at": site.UPDATE_AT,
-                               "upload": site.UPLOAD,
-                               "download": site.DOWNLOAD,
-                               "ratio": site.RATIO,
-                               "seeding": site.SEEDING,
-                               "leeching": site.LEECHING,
-                               "seeding_size": site.SEEDING_SIZE,
-                               "bonus": site.BONUS,
-                               "url": site.URL,
-                               "msg_unread": site.MSG_UNREAD
-                               })
-        return statistics

app/plugins/sitestatistic/siteuserinfo/__init__.py

@@ -14,6 +14,7 @@ from app.core import settings
 from app.helper.cloudflare import under_challenge
 from app.log import logger
 from app.utils.http import RequestUtils
+from app.utils.site import SiteUtils
 from app.utils.types import SiteSchema
 
 SITE_BASE_ORDER = 1000
@@ -101,7 +102,7 @@ class ISiteUserInfo(metaclass=ABCMeta):
         self._emulate = emulate
         self._proxy = proxy
 
-    def site_schema(self):
+    def site_schema(self) -> SiteSchema:
         """
         站点解析模型
         :return: 站点解析模型
@@ -196,7 +197,7 @@ class ISiteUserInfo(metaclass=ABCMeta):
         """
         pass
 
-    def _parse_favicon(self, html_text):
+    def _parse_favicon(self, html_text: str):
         """
         解析站点favicon,返回base64 fav图标
         :param html_text:
@@ -213,7 +214,7 @@ class ISiteUserInfo(metaclass=ABCMeta):
         if res:
             self.site_favicon = base64.b64encode(res.content).decode()
 
-    def _get_page_content(self, url, params=None, headers=None):
+    def _get_page_content(self, url: str, params: dict = None, headers: dict = None):
         """
         :param url: 网页地址
         :param params: post参数
@@ -285,7 +286,7 @@ class ISiteUserInfo(metaclass=ABCMeta):
         :param html_text:
         :return: True/False
         """
-        logged_in = self.is_logged_in(html_text)
+        logged_in = SiteUtils.is_logged_in(html_text)
         if not logged_in:
             self.err_msg = "未检测到已登陆，请检查cookies是否过期"
             logger.warn(f"{self.site_name} 未登录，跳过后续操作")
@@ -330,31 +331,16 @@ class ISiteUserInfo(metaclass=ABCMeta):
         """
         pass
 
-    @classmethod
-    def is_logged_in(cls, html_text: str) -> bool:
+    def to_dict(self):
         """
-        判断站点是否已经登陆
-        :param html_text:
-        :return:
+        转化为字典
         """
-        html = etree.HTML(html_text)
-        if not html:
-            return False
-        # 存在明显的密码输入框，说明未登录
-        if html.xpath("//input[@type='password']"):
-            return False
-        # 是否存在登出和用户面板等链接
-        xpaths = ['//a[contains(@href, "logout")'
-                  ' or contains(@data-url, "logout")'
-                  ' or contains(@href, "mybonus") '
-                  ' or contains(@onclick, "logout")'
-                  ' or contains(@href, "usercp")]',
-                  '//form[contains(@action, "logout")]']
-        for xpath in xpaths:
-            if html.xpath(xpath):
-                return True
-        user_info_div = html.xpath('//div[@class="user-info-side"]')
-        if user_info_div:
-            return True
-
-        return False
+        attributes = [
+            attr for attr in dir(self)
+            if not callable(getattr(self, attr)) and not attr.startswith("_")
+        ]
+        return {
+            attr: getattr(self, attr).value
+            if isinstance(getattr(self, attr), SiteSchema)
+            else getattr(self, attr) for attr in attributes
+        }

app/plugins/sitestatistic/siteuserinfo/discuz.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 from lxml import etree
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/file_list.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 from lxml import etree
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/gazelle.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 from lxml import etree
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/ipt_project.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 from lxml import etree
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/nexus_php.py

@@ -5,7 +5,7 @@ from typing import Optional
 from lxml import etree
 
 from app.log import logger
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/nexus_project.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 import re
 
-from app.plugins.sitestatistics.siteuserinfo import SITE_BASE_ORDER
-from app.plugins.sitestatistics.siteuserinfo.nexus_php import NexusPhpSiteUserInfo
+from app.plugins.sitestatistic.siteuserinfo import SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo.nexus_php import NexusPhpSiteUserInfo
 from app.utils.types import SiteSchema
 
 

app/plugins/sitestatistic/siteuserinfo/nexus_rabbit.py

@@ -5,8 +5,8 @@ from typing import Optional
 from lxml import etree
 
 from app.log import logger
-from app.plugins.sitestatistics.siteuserinfo import SITE_BASE_ORDER
-from app.plugins.sitestatistics.siteuserinfo.nexus_php import NexusPhpSiteUserInfo
+from app.plugins.sitestatistic.siteuserinfo import SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo.nexus_php import NexusPhpSiteUserInfo
 from app.utils.types import SiteSchema
 
 

app/plugins/sitestatistic/siteuserinfo/small_horse.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 from lxml import etree
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/tnode.py

@@ -3,7 +3,7 @@ import json
 import re
 from typing import Optional
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/torrent_leech.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 from lxml import etree
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/plugins/sitestatistic/siteuserinfo/unit3d.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 from lxml import etree
 
-from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
+from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER
 from app.utils.string import StringUtils
 from app.utils.types import SiteSchema
 

app/utils/site.py

@@ -0,0 +1,33 @@
+from lxml import etree
+
+
+class SiteUtils:
+
+    @classmethod
+    def is_logged_in(cls, html_text: str) -> bool:
+        """
+        判断站点是否已经登陆
+        :param html_text:
+        :return:
+        """
+        html = etree.HTML(html_text)
+        if not html:
+            return False
+        # 存在明显的密码输入框，说明未登录
+        if html.xpath("//input[@type='password']"):
+            return False
+        # 是否存在登出和用户面板等链接
+        xpaths = ['//a[contains(@href, "logout")'
+                  ' or contains(@data-url, "logout")'
+                  ' or contains(@href, "mybonus") '
+                  ' or contains(@onclick, "logout")'
+                  ' or contains(@href, "usercp")]',
+                  '//form[contains(@action, "logout")]']
+        for xpath in xpaths:
+            if html.xpath(xpath):
+                return True
+        user_info_div = html.xpath('//div[@class="user-info-side"]')
+        if user_info_div:
+            return True
+
+        return False

app/utils/timer.py

@@ -30,8 +30,8 @@ class TimerUtils:
             random_interval = datetime.timedelta(minutes=interval_minutes)
             # 更新当前时间为下一个任务的时间触发器
             random_trigger += random_interval
-            # 达到结否时间时退出
-            if now.hour > end_hour:
+            # 达到结束时间时退出
+            if random_trigger.hour > end_hour:
                 break
             # 添加到队列
             trigger.append(random_trigger)

tests/run.py

@@ -15,9 +15,9 @@ if __name__ == '__main__':
     # 测试名称识别
     suite.addTest(MetaInfoTest('test_metainfo'))
     # 测试媒体识别
-    # suite.addTest(RecognizeTest('test_recognize'))
+    suite.addTest(RecognizeTest('test_recognize'))
     # 测试CookieCloud同步
-    # suite.addTest(CookieCloudTest('test_cookiecloud'))
+    suite.addTest(CookieCloudTest('test_cookiecloud'))
     # 测试文件转移
     # suite.addTest(TransferTest('test_transfer'))
     # 测试豆瓣同步