From 0d0b078a31d5c3cb653e68d9804fc03e90ebead5 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 8 Jun 2023 15:47:14 +0800 Subject: [PATCH] fix plugins --- app/db/init.py | 7 ++ app/db/models/plugin.py | 22 +++++ app/db/models/site.py | 3 + app/db/models/subscribe.py | 3 + app/db/models/systemconfig.py | 3 + app/db/models/user.py | 3 + app/main.py | 2 + app/plugins/__init__.py | 27 ++++++ app/plugins/autosignin/__init__.py | 38 ++------ .../__init__.py | 95 ++++++++++--------- .../siteuserinfo/__init__.py | 46 ++++----- .../siteuserinfo/discuz.py | 2 +- .../siteuserinfo/file_list.py | 2 +- .../siteuserinfo/gazelle.py | 2 +- .../siteuserinfo/ipt_project.py | 2 +- .../siteuserinfo/nexus_php.py | 2 +- .../siteuserinfo/nexus_project.py | 4 +- .../siteuserinfo/nexus_rabbit.py | 4 +- .../siteuserinfo/small_horse.py | 2 +- .../siteuserinfo/tnode.py | 2 +- .../siteuserinfo/torrent_leech.py | 2 +- .../siteuserinfo/unit3d.py | 2 +- app/utils/site.py | 33 +++++++ app/utils/timer.py | 4 +- tests/run.py | 4 +- 25 files changed, 191 insertions(+), 125 deletions(-) create mode 100644 app/db/models/plugin.py rename app/plugins/{sitestatistics => sitestatistic}/__init__.py (79%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/__init__.py (90%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/discuz.py (98%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/file_list.py (98%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/gazelle.py (98%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/ipt_project.py (97%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/nexus_php.py (99%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/nexus_project.py (83%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/nexus_rabbit.py (92%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/small_horse.py (98%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/tnode.py (97%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/torrent_leech.py (97%) rename app/plugins/{sitestatistics => sitestatistic}/siteuserinfo/unit3d.py (98%) create mode 100644 app/utils/site.py diff --git a/app/db/init.py b/app/db/init.py index d2583c69..d56e9293 100644 --- a/app/db/init.py +++ b/app/db/init.py @@ -1,3 +1,6 @@ +import importlib +from pathlib import Path + from alembic.command import upgrade from alembic.config import Config @@ -13,6 +16,10 @@ def init_db(): """ 初始化数据库 """ + # 导入模块，避免建表缺失 + for module in Path(__file__).with_name("models").glob("*.py"): + importlib.import_module(f"app.db.models.{module.stem}") + # 全量建表 Base.metadata.create_all(bind=Engine) # 初始化超级管理员 _db = SessionLocal() diff --git a/app/db/models/plugin.py b/app/db/models/plugin.py new file mode 100644 index 00000000..796d2228 --- /dev/null +++ b/app/db/models/plugin.py @@ -0,0 +1,22 @@ +from sqlalchemy import Column, Integer, String, Sequence +from sqlalchemy.orm import Session + +from app.db.models import Base + + +class PluginData(Base): + """ + 插件数据表 + """ + id = Column(Integer, Sequence('id'), primary_key=True, index=True) + plugin_id = Column(String, nullable=False, index=True) + key = Column(String, index=True, nullable=False) + value = Column(String) + + @staticmethod + def get_plugin_data(db: Session, plugin_id: str): + return db.query(PluginData).filter(PluginData.plugin_id == plugin_id).all() + + @staticmethod + def get_plugin_data_by_key(db: Session, plugin_id: str, key: str): + return db.query(PluginData.value).filter(PluginData.plugin_id == plugin_id, PluginData.key == key).first() diff --git a/app/db/models/site.py b/app/db/models/site.py index cdc95430..d4d1ec59 100644 --- a/app/db/models/site.py +++ b/app/db/models/site.py @@ -7,6 +7,9 @@ from app.db.models import Base class Site(Base): + """ + 站点表 + """ id = Column(Integer, Sequence('id'), primary_key=True, index=True) name = Column(String, nullable=False) domain = Column(String, index=True) diff --git a/app/db/models/subscribe.py b/app/db/models/subscribe.py index 3db64fa0..13994925 100644 --- a/app/db/models/subscribe.py +++ b/app/db/models/subscribe.py @@ -5,6 +5,9 @@ from app.db.models import Base class Subscribe(Base): + """ + 订阅表 + """ id = Column(Integer, Sequence('id'), primary_key=True, index=True) name = Column(String, nullable=False, index=True) year = Column(String) diff --git a/app/db/models/systemconfig.py b/app/db/models/systemconfig.py index cb1ae268..7e32120e 100644 --- a/app/db/models/systemconfig.py +++ b/app/db/models/systemconfig.py @@ -5,6 +5,9 @@ from app.db.models import Base class SystemConfig(Base): + """ + 配置表 + """ id = Column(Integer, Sequence('id'), primary_key=True, index=True) key = Column(String, index=True) value = Column(String, nullable=True) diff --git a/app/db/models/user.py b/app/db/models/user.py index 1be302be..8c9a9cfc 100644 --- a/app/db/models/user.py +++ b/app/db/models/user.py @@ -6,6 +6,9 @@ from app.db.models import Base class User(Base): + """ + 用户表 + """ id = Column(Integer, Sequence('id'), primary_key=True, index=True) full_name = Column(String, index=True) email = Column(String, unique=True, index=True, nullable=False) diff --git a/app/main.py b/app/main.py index 6e158ec5..3fe06c3b 100644 --- a/app/main.py +++ b/app/main.py @@ -24,7 +24,9 @@ def shutdown_server(): """ 服务关闭 """ + # 停止定时服务 Scheduler().stop() + # 停止插件 PluginManager().stop() diff --git a/app/plugins/__init__.py b/app/plugins/__init__.py index 419a1976..fe1f4413 100644 --- a/app/plugins/__init__.py +++ b/app/plugins/__init__.py @@ -1,10 +1,15 @@ +import json from abc import ABCMeta, abstractmethod from pathlib import Path from typing import Any, Optional from app.chain import ChainBase from app.core import settings, Context +from app.db import SessionLocal +from app.db.models import Base +from app.db.models.plugin import PluginData from app.db.systemconfigs import SystemConfigs +from app.utils.object import ObjectUtils class PluginChian(ChainBase): @@ -34,6 +39,7 @@ class _PluginBase(metaclass=ABCMeta): plugin_desc: str = "" def __init__(self): + self.db = SessionLocal() self.chain = PluginChian() @abstractmethod @@ -80,3 +86,24 @@ class _PluginBase(metaclass=ABCMeta): if not data_path.exists(): data_path.mkdir(parents=True) return data_path + + def save_data(self, key: str, value: Any) -> Base: + """ + 保存插件数据 + :param key: 数据key + :param value: 数据值 + """ + if ObjectUtils.is_obj(value): + value = json.dumps(value) + plugin = PluginData(plugin_id=self.__class__.__name__, key=key, value=value) + return plugin.create(self.db) + + def get_data(self, key: str) -> Any: + """ + 获取插件数据 + :param key: 数据key + """ + data = PluginData.get_plugin_data_by_key(self.db, self.__class__.__name__, key) + if ObjectUtils.is_obj(data): + return json.load(data) + return data diff --git a/app/plugins/autosignin/__init__.py b/app/plugins/autosignin/__init__.py index d2e0c0c0..1108f73c 100644 --- a/app/plugins/autosignin/__init__.py +++ b/app/plugins/autosignin/__init__.py @@ -5,7 +5,6 @@ from typing import Any from urllib.parse import urljoin from apscheduler.schedulers.background import BackgroundScheduler -from lxml import etree from ruamel.yaml import CommentedMap from app.core import EventManager, settings, eventmanager @@ -15,6 +14,7 @@ from app.helper.sites import SitesHelper from app.log import logger from app.plugins import _PluginBase from app.utils.http import RequestUtils +from app.utils.site import SiteUtils from app.utils.timer import TimerUtils from app.utils.types import EventType @@ -81,6 +81,8 @@ class AutoSignIn(_PluginBase): """ 自动签到 """ + if event: + logger.info("收到远程签到命令，开始执行签到任务 ...") # 查询签到站点 sign_sites = self.sites.get_indexers() if not sign_sites: @@ -123,7 +125,8 @@ class AutoSignIn(_PluginBase): else: return self.__signin_base(site_info) - def __signin_base(self, site_info: CommentedMap) -> str: + @staticmethod + def __signin_base(site_info: CommentedMap) -> str: """ 通用签到处理 :param site_info: 站点信息 @@ -158,7 +161,7 @@ class AutoSignIn(_PluginBase): ).get_res(url=site_url) # 判断登录状态 if res and res.status_code in [200, 500, 403]: - if not self.is_logged_in(res.text): + if not SiteUtils.is_logged_in(res.text): if under_challenge(res.text): msg = "站点被Cloudflare防护，请更换Cookie和UA！" elif res.status_code == 200: @@ -194,32 +197,3 @@ class AutoSignIn(_PluginBase): self._scheduler = None except Exception as e: logger.error("退出插件失败：%s" % str(e)) - - @classmethod - def is_logged_in(cls, html_text: str) -> bool: - """ - 判断站点是否已经登陆 - :param html_text: - :return: - """ - html = etree.HTML(html_text) - if not html: - return False - # 存在明显的密码输入框，说明未登录 - if html.xpath("//input[@type='password']"): - return False - # 是否存在登出和用户面板等链接 - xpaths = ['//a[contains(@href, "logout")' - ' or contains(@data-url, "logout")' - ' or contains(@href, "mybonus") ' - ' or contains(@onclick, "logout")' - ' or contains(@href, "usercp")]', - '//form[contains(@action, "logout")]'] - for xpath in xpaths: - if html.xpath(xpath): - return True - user_info_div = html.xpath('//div[@class="user-info-side"]') - if user_info_div: - return True - - return False diff --git a/app/plugins/sitestatistics/__init__.py b/app/plugins/sitestatistic/__init__.py similarity index 79% rename from app/plugins/sitestatistics/__init__.py rename to app/plugins/sitestatistic/__init__.py index 4abea59d..ede2a2b0 100644 --- a/app/plugins/sitestatistics/__init__.py +++ b/app/plugins/sitestatistic/__init__.py @@ -1,9 +1,10 @@ from datetime import datetime from multiprocessing.dummy import Pool as ThreadPool from threading import Lock -from typing import Optional, Any +from typing import Optional, Any, List import requests +from apscheduler.schedulers.background import BackgroundScheduler from ruamel.yaml import CommentedMap from app.core import settings @@ -11,23 +12,28 @@ from app.helper import ModuleHelper from app.helper.sites import SitesHelper from app.log import logger from app.plugins import _PluginBase -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo from app.utils.http import RequestUtils +from app.utils.timer import TimerUtils + +import warnings +warnings.filterwarnings("ignore", category=FutureWarning) + lock = Lock() -class SiteStatistics(_PluginBase): +class SiteStatistic(_PluginBase): sites = None - + _scheduler: BackgroundScheduler = None _MAX_CONCURRENCY: int = 10 _last_update_time: Optional[datetime] = None _sites_data: dict = {} - _site_schema: list = None + _site_schema: List[ISiteUserInfo] = None def init_plugin(self, config: dict = None): # 加载模块 - self._site_schema = ModuleHelper.load('app.plugins.sitestatistics.siteuserinfo', + self._site_schema = ModuleHelper.load('app.plugins.sitestatistic.siteuserinfo', filter_func=lambda _, obj: hasattr(obj, 'schema')) self._site_schema.sort(key=lambda x: x.order) # 站点管理 @@ -36,6 +42,20 @@ class SiteStatistics(_PluginBase): self._last_update_time = None # 站点数据 self._sites_data = {} + # 定时服务 + self._scheduler = BackgroundScheduler(timezone=settings.TZ) + triggers = TimerUtils.random_scheduler(num_executions=1, + begin_hour=0, + end_hour=1, + min_interval=1, + max_interval=60) + for trigger in triggers: + self._scheduler.add_job(self.refresh_all_site_data, "cron", hour=trigger.hour, minute=trigger.minute) + + # 启动任务 + if self._scheduler.get_jobs(): + self._scheduler.print_jobs() + self._scheduler.start() def stop_service(self): pass @@ -46,13 +66,16 @@ class SiteStatistics(_PluginBase): if site_schema.match(html_text): return site_schema except Exception as e: - logger.error(f"站点 {site_schema.name} 匹配失败 {e}") + logger.error(f"站点匹配失败 {e}") return None def build(self, url: str, site_name: str, site_cookie: str = None, ua: str = None, - proxy: bool = False) -> Any: + proxy: bool = False) -> Optional[ISiteUserInfo]: + """ + 构建站点信息 + """ if not site_cookie: return None session = requests.Session() @@ -121,26 +144,26 @@ class SiteStatistics(_PluginBase): return None return site_schema(site_name, url, site_cookie, html_text, session=session, ua=ua, proxy=proxy) - def __refresh_site_data(self, site_info: CommentedMap): + def __refresh_site_data(self, site_info: CommentedMap) -> Optional[ISiteUserInfo]: """ 更新单个site 数据信息 :param site_info: :return: """ site_name = site_info.get("name") - site_url = site_info.get("strict_url") + site_url = site_info.get("url") if not site_url: - return + return None site_cookie = site_info.get("cookie") ua = site_info.get("ua") unread_msg_notify = True proxy = site_info.get("proxy") try: - site_user_info = self.build(url=site_url, - site_name=site_name, - site_cookie=site_cookie, - ua=ua, - proxy=proxy) + site_user_info: ISiteUserInfo = self.build(url=site_url, + site_name=site_name, + site_cookie=site_cookie, + ua=ua, + proxy=proxy) if site_user_info: logger.debug(f"站点 {site_name} 开始以 {site_user_info.site_schema()} 模型解析") # 开始解析 @@ -150,7 +173,7 @@ class SiteStatistics(_PluginBase): # 获取不到数据时，仅返回错误信息，不做历史数据更新 if site_user_info.err_msg: self._sites_data.update({site_name: {"err_msg": site_user_info.err_msg}}) - return + return None # 发送通知，存在未读消息 self.__notify_unread_msg(site_name, site_user_info, unread_msg_notify) @@ -173,11 +196,11 @@ class SiteStatistics(_PluginBase): "message_unread": site_user_info.message_unread } }) - return site_user_info except Exception as e: logger.error(f"站点 {site_name} 获取流量数据失败：{str(e)}") + return None def __notify_unread_msg(self, site_name: str, site_user_info: ISiteUserInfo, unread_msg_notify: bool): if site_user_info.message_unread <= 0: @@ -228,35 +251,15 @@ class SiteStatistics(_PluginBase): # 并发刷新 with ThreadPool(min(len(refresh_sites), self._MAX_CONCURRENCY)) as p: - site_user_infos = p.map(self.__refresh_site_data, refresh_sites) + site_user_infos: List[ISiteUserInfo] = p.map(self.__refresh_site_data, refresh_sites) site_user_infos = [info for info in site_user_infos if info] - - print(site_user_infos) - # TODO 登记历史数据 - # TODO 实时用户数据 - # TODO 更新站点图标 - # TODO 实时做种信息 + # 保存数据 + for site_user_info in site_user_infos: + # 获取今天的日期 + key = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + value = site_user_info.to_dict() + # 按日期保存为字典 + self.save_data(key, value) # 更新时间 self._last_update_time = datetime.now() - - @staticmethod - def __todict(raw_statistics): - statistics = [] - for site in raw_statistics: - statistics.append({"site": site.SITE, - "username": site.USERNAME, - "user_level": site.USER_LEVEL, - "join_at": site.JOIN_AT, - "update_at": site.UPDATE_AT, - "upload": site.UPLOAD, - "download": site.DOWNLOAD, - "ratio": site.RATIO, - "seeding": site.SEEDING, - "leeching": site.LEECHING, - "seeding_size": site.SEEDING_SIZE, - "bonus": site.BONUS, - "url": site.URL, - "msg_unread": site.MSG_UNREAD - }) - return statistics diff --git a/app/plugins/sitestatistics/siteuserinfo/__init__.py b/app/plugins/sitestatistic/siteuserinfo/__init__.py similarity index 90% rename from app/plugins/sitestatistics/siteuserinfo/__init__.py rename to app/plugins/sitestatistic/siteuserinfo/__init__.py index 38d01073..930aaf54 100644 --- a/app/plugins/sitestatistics/siteuserinfo/__init__.py +++ b/app/plugins/sitestatistic/siteuserinfo/__init__.py @@ -14,6 +14,7 @@ from app.core import settings from app.helper.cloudflare import under_challenge from app.log import logger from app.utils.http import RequestUtils +from app.utils.site import SiteUtils from app.utils.types import SiteSchema SITE_BASE_ORDER = 1000 @@ -101,7 +102,7 @@ class ISiteUserInfo(metaclass=ABCMeta): self._emulate = emulate self._proxy = proxy - def site_schema(self): + def site_schema(self) -> SiteSchema: """ 站点解析模型 :return: 站点解析模型 @@ -196,7 +197,7 @@ class ISiteUserInfo(metaclass=ABCMeta): """ pass - def _parse_favicon(self, html_text): + def _parse_favicon(self, html_text: str): """ 解析站点favicon,返回base64 fav图标 :param html_text: @@ -213,7 +214,7 @@ class ISiteUserInfo(metaclass=ABCMeta): if res: self.site_favicon = base64.b64encode(res.content).decode() - def _get_page_content(self, url, params=None, headers=None): + def _get_page_content(self, url: str, params: dict = None, headers: dict = None): """ :param url: 网页地址 :param params: post参数 @@ -285,7 +286,7 @@ class ISiteUserInfo(metaclass=ABCMeta): :param html_text: :return: True/False """ - logged_in = self.is_logged_in(html_text) + logged_in = SiteUtils.is_logged_in(html_text) if not logged_in: self.err_msg = "未检测到已登陆，请检查cookies是否过期" logger.warn(f"{self.site_name} 未登录，跳过后续操作") @@ -330,31 +331,16 @@ class ISiteUserInfo(metaclass=ABCMeta): """ pass - @classmethod - def is_logged_in(cls, html_text: str) -> bool: + def to_dict(self): """ - 判断站点是否已经登陆 - :param html_text: - :return: + 转化为字典 """ - html = etree.HTML(html_text) - if not html: - return False - # 存在明显的密码输入框，说明未登录 - if html.xpath("//input[@type='password']"): - return False - # 是否存在登出和用户面板等链接 - xpaths = ['//a[contains(@href, "logout")' - ' or contains(@data-url, "logout")' - ' or contains(@href, "mybonus") ' - ' or contains(@onclick, "logout")' - ' or contains(@href, "usercp")]', - '//form[contains(@action, "logout")]'] - for xpath in xpaths: - if html.xpath(xpath): - return True - user_info_div = html.xpath('//div[@class="user-info-side"]') - if user_info_div: - return True - - return False + attributes = [ + attr for attr in dir(self) + if not callable(getattr(self, attr)) and not attr.startswith("_") + ] + return { + attr: getattr(self, attr).value + if isinstance(getattr(self, attr), SiteSchema) + else getattr(self, attr) for attr in attributes + } diff --git a/app/plugins/sitestatistics/siteuserinfo/discuz.py b/app/plugins/sitestatistic/siteuserinfo/discuz.py similarity index 98% rename from app/plugins/sitestatistics/siteuserinfo/discuz.py rename to app/plugins/sitestatistic/siteuserinfo/discuz.py index 9c67f78f..b636e805 100644 --- a/app/plugins/sitestatistics/siteuserinfo/discuz.py +++ b/app/plugins/sitestatistic/siteuserinfo/discuz.py @@ -4,7 +4,7 @@ from typing import Optional from lxml import etree -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/file_list.py b/app/plugins/sitestatistic/siteuserinfo/file_list.py similarity index 98% rename from app/plugins/sitestatistics/siteuserinfo/file_list.py rename to app/plugins/sitestatistic/siteuserinfo/file_list.py index 0c4e4d54..d2bd6062 100644 --- a/app/plugins/sitestatistics/siteuserinfo/file_list.py +++ b/app/plugins/sitestatistic/siteuserinfo/file_list.py @@ -4,7 +4,7 @@ from typing import Optional from lxml import etree -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/gazelle.py b/app/plugins/sitestatistic/siteuserinfo/gazelle.py similarity index 98% rename from app/plugins/sitestatistics/siteuserinfo/gazelle.py rename to app/plugins/sitestatistic/siteuserinfo/gazelle.py index cc53b0ba..2161f525 100644 --- a/app/plugins/sitestatistics/siteuserinfo/gazelle.py +++ b/app/plugins/sitestatistic/siteuserinfo/gazelle.py @@ -4,7 +4,7 @@ from typing import Optional from lxml import etree -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/ipt_project.py b/app/plugins/sitestatistic/siteuserinfo/ipt_project.py similarity index 97% rename from app/plugins/sitestatistics/siteuserinfo/ipt_project.py rename to app/plugins/sitestatistic/siteuserinfo/ipt_project.py index 26af3202..39399fc6 100644 --- a/app/plugins/sitestatistics/siteuserinfo/ipt_project.py +++ b/app/plugins/sitestatistic/siteuserinfo/ipt_project.py @@ -4,7 +4,7 @@ from typing import Optional from lxml import etree -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/nexus_php.py b/app/plugins/sitestatistic/siteuserinfo/nexus_php.py similarity index 99% rename from app/plugins/sitestatistics/siteuserinfo/nexus_php.py rename to app/plugins/sitestatistic/siteuserinfo/nexus_php.py index d9ade094..54817385 100644 --- a/app/plugins/sitestatistics/siteuserinfo/nexus_php.py +++ b/app/plugins/sitestatistic/siteuserinfo/nexus_php.py @@ -5,7 +5,7 @@ from typing import Optional from lxml import etree from app.log import logger -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/nexus_project.py b/app/plugins/sitestatistic/siteuserinfo/nexus_project.py similarity index 83% rename from app/plugins/sitestatistics/siteuserinfo/nexus_project.py rename to app/plugins/sitestatistic/siteuserinfo/nexus_project.py index 54c49fe5..049b193a 100644 --- a/app/plugins/sitestatistics/siteuserinfo/nexus_project.py +++ b/app/plugins/sitestatistic/siteuserinfo/nexus_project.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import re -from app.plugins.sitestatistics.siteuserinfo import SITE_BASE_ORDER -from app.plugins.sitestatistics.siteuserinfo.nexus_php import NexusPhpSiteUserInfo +from app.plugins.sitestatistic.siteuserinfo import SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo.nexus_php import NexusPhpSiteUserInfo from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/nexus_rabbit.py b/app/plugins/sitestatistic/siteuserinfo/nexus_rabbit.py similarity index 92% rename from app/plugins/sitestatistics/siteuserinfo/nexus_rabbit.py rename to app/plugins/sitestatistic/siteuserinfo/nexus_rabbit.py index 07f865ef..6de44904 100644 --- a/app/plugins/sitestatistics/siteuserinfo/nexus_rabbit.py +++ b/app/plugins/sitestatistic/siteuserinfo/nexus_rabbit.py @@ -5,8 +5,8 @@ from typing import Optional from lxml import etree from app.log import logger -from app.plugins.sitestatistics.siteuserinfo import SITE_BASE_ORDER -from app.plugins.sitestatistics.siteuserinfo.nexus_php import NexusPhpSiteUserInfo +from app.plugins.sitestatistic.siteuserinfo import SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo.nexus_php import NexusPhpSiteUserInfo from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/small_horse.py b/app/plugins/sitestatistic/siteuserinfo/small_horse.py similarity index 98% rename from app/plugins/sitestatistics/siteuserinfo/small_horse.py rename to app/plugins/sitestatistic/siteuserinfo/small_horse.py index 5a4cf8ff..147e0846 100644 --- a/app/plugins/sitestatistics/siteuserinfo/small_horse.py +++ b/app/plugins/sitestatistic/siteuserinfo/small_horse.py @@ -4,7 +4,7 @@ from typing import Optional from lxml import etree -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/tnode.py b/app/plugins/sitestatistic/siteuserinfo/tnode.py similarity index 97% rename from app/plugins/sitestatistics/siteuserinfo/tnode.py rename to app/plugins/sitestatistic/siteuserinfo/tnode.py index 3ca99e3f..71e65524 100644 --- a/app/plugins/sitestatistics/siteuserinfo/tnode.py +++ b/app/plugins/sitestatistic/siteuserinfo/tnode.py @@ -3,7 +3,7 @@ import json import re from typing import Optional -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/torrent_leech.py b/app/plugins/sitestatistic/siteuserinfo/torrent_leech.py similarity index 97% rename from app/plugins/sitestatistics/siteuserinfo/torrent_leech.py rename to app/plugins/sitestatistic/siteuserinfo/torrent_leech.py index 72431d13..de0d32cc 100644 --- a/app/plugins/sitestatistics/siteuserinfo/torrent_leech.py +++ b/app/plugins/sitestatistic/siteuserinfo/torrent_leech.py @@ -4,7 +4,7 @@ from typing import Optional from lxml import etree -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/plugins/sitestatistics/siteuserinfo/unit3d.py b/app/plugins/sitestatistic/siteuserinfo/unit3d.py similarity index 98% rename from app/plugins/sitestatistics/siteuserinfo/unit3d.py rename to app/plugins/sitestatistic/siteuserinfo/unit3d.py index a03430ac..4e1802b8 100644 --- a/app/plugins/sitestatistics/siteuserinfo/unit3d.py +++ b/app/plugins/sitestatistic/siteuserinfo/unit3d.py @@ -4,7 +4,7 @@ from typing import Optional from lxml import etree -from app.plugins.sitestatistics.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER from app.utils.string import StringUtils from app.utils.types import SiteSchema diff --git a/app/utils/site.py b/app/utils/site.py new file mode 100644 index 00000000..506e77e9 --- /dev/null +++ b/app/utils/site.py @@ -0,0 +1,33 @@ +from lxml import etree + + +class SiteUtils: + + @classmethod + def is_logged_in(cls, html_text: str) -> bool: + """ + 判断站点是否已经登陆 + :param html_text: + :return: + """ + html = etree.HTML(html_text) + if not html: + return False + # 存在明显的密码输入框，说明未登录 + if html.xpath("//input[@type='password']"): + return False + # 是否存在登出和用户面板等链接 + xpaths = ['//a[contains(@href, "logout")' + ' or contains(@data-url, "logout")' + ' or contains(@href, "mybonus") ' + ' or contains(@onclick, "logout")' + ' or contains(@href, "usercp")]', + '//form[contains(@action, "logout")]'] + for xpath in xpaths: + if html.xpath(xpath): + return True + user_info_div = html.xpath('//div[@class="user-info-side"]') + if user_info_div: + return True + + return False diff --git a/app/utils/timer.py b/app/utils/timer.py index cea7c8f2..1969f288 100644 --- a/app/utils/timer.py +++ b/app/utils/timer.py @@ -30,8 +30,8 @@ class TimerUtils: random_interval = datetime.timedelta(minutes=interval_minutes) # 更新当前时间为下一个任务的时间触发器 random_trigger += random_interval - # 达到结否时间时退出 - if now.hour > end_hour: + # 达到结束时间时退出 + if random_trigger.hour > end_hour: break # 添加到队列 trigger.append(random_trigger) diff --git a/tests/run.py b/tests/run.py index df3c053c..8ddc8438 100644 --- a/tests/run.py +++ b/tests/run.py @@ -15,9 +15,9 @@ if __name__ == '__main__': # 测试名称识别 suite.addTest(MetaInfoTest('test_metainfo')) # 测试媒体识别 - # suite.addTest(RecognizeTest('test_recognize')) + suite.addTest(RecognizeTest('test_recognize')) # 测试CookieCloud同步 - # suite.addTest(CookieCloudTest('test_cookiecloud')) + suite.addTest(CookieCloudTest('test_cookiecloud')) # 测试文件转移 # suite.addTest(TransferTest('test_transfer')) # 测试豆瓣同步