codfrm/MoviePilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix 站点数据统计插件增加全量/增量选项

Browse Source
This commit is contained in:
thsrite 2023-08-11 09:47:42 +08:00
parent f1ef102a8d
commit 06f513c830
1 changed files with 100 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
app/plugins/sitestatistic/__init__.py
View File

@ -4,6 +4,7 @@ from multiprocessing.dummy import Pool as ThreadPool
from threading import Lock from threading import Lock
from typing import Optional, Any, List, Dict, Tuple from typing import Optional, Any, List, Dict, Tuple
import pytz
import requests import requests
from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.cron import CronTrigger
@ -60,9 +61,11 @@ class SiteStatistic(_PluginBase):
# 配置属性 # 配置属性
_enabled: bool = False _enabled: bool = False
_onlyonce: bool = False
_cron: str = "" _cron: str = ""
_notify: bool = False _notify: bool = False
_queue_cnt: int = 5 _queue_cnt: int = 5
_statistic_type: str = None
_statistic_sites: list = [] _statistic_sites: list = []
def init_plugin(self, config: dict = None): def init_plugin(self, config: dict = None):
@ -73,12 +76,14 @@ class SiteStatistic(_PluginBase):
# 配置 # 配置
if config: if config:
self._enabled = config.get("enabled") self._enabled = config.get("enabled")
self._onlyonce = config.get("onlyonce")
self._cron = config.get("cron") self._cron = config.get("cron")
self._notify = config.get("notify") self._notify = config.get("notify")
self._queue_cnt = config.get("queue_cnt") self._queue_cnt = config.get("queue_cnt")
self._statistic_sites = config.get("statistic_sites") self._statistic_type = config.get("statistic_type") or "all"
self._statistic_sites = config.get("statistic_sites") or []
if self._enabled: if self._enabled or self._onlyonce:
# 加载模块 # 加载模块
self._site_schema = ModuleHelper.load('app.plugins.sitestatistic.siteuserinfo', self._site_schema = ModuleHelper.load('app.plugins.sitestatistic.siteuserinfo',
filter_func=lambda _, obj: hasattr(obj, 'schema')) filter_func=lambda _, obj: hasattr(obj, 'schema'))
@ -108,6 +113,17 @@ class SiteStatistic(_PluginBase):
self._scheduler.add_job(self.refresh_all_site_data, "cron", self._scheduler.add_job(self.refresh_all_site_data, "cron",
hour=trigger.hour, minute=trigger.minute, hour=trigger.hour, minute=trigger.minute,
name="站点数据统计") name="站点数据统计")
if self._onlyonce:
logger.info(f"站点数据统计服务启动，立即运行一次")
self._scheduler.add_job(self.refresh_all_site_data, 'date',
run_date=datetime.now(
tz=pytz.timezone(settings.TZ)) + timedelta(seconds=3)
)
# 关闭一次性开关
self._onlyonce = False
# 保存配置
self.__update_config()
# 启动任务 # 启动任务
if self._scheduler.get_jobs(): if self._scheduler.get_jobs():
@ -166,7 +182,7 @@ class SiteStatistic(_PluginBase):
'component': 'VCol', 'component': 'VCol',
'props': { 'props': {
'cols': 12, 'cols': 12,
'md': 6 'md': 4
}, },
'content': [ 'content': [
{ {
@ -182,7 +198,7 @@ class SiteStatistic(_PluginBase):
'component': 'VCol', 'component': 'VCol',
'props': { 'props': {
'cols': 12, 'cols': 12,
'md': 6 'md': 4
}, },
'content': [ 'content': [
{ {
@ -193,6 +209,22 @@ class SiteStatistic(_PluginBase):
} }
} }
] ]
},
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VSwitch',
'props': {
'model': 'onlyonce',
'label': '立即运行一次',
}
}
]
} }
] ]
}, },
@ -203,7 +235,7 @@ class SiteStatistic(_PluginBase):
'component': 'VCol', 'component': 'VCol',
'props': { 'props': {
'cols': 12, 'cols': 12,
'md': 6 'md': 4
}, },
'content': [ 'content': [
{ {
@ -220,7 +252,7 @@ class SiteStatistic(_PluginBase):
'component': 'VCol', 'component': 'VCol',
'props': { 'props': {
'cols': 12, 'cols': 12,
'md': 6 'md': 4
}, },
'content': [ 'content': [
{ {
@ -231,6 +263,26 @@ class SiteStatistic(_PluginBase):
} }
} }
] ]
},
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VSelect',
'props': {
'model': 'statistic_type',
'label': '统计类型',
'items': [
{'title': '全量', 'value': 'all'},
{'title': '增量', 'value': 'add'}
]
}
}
]
} }
] ]
}, },
@ -258,9 +310,11 @@ class SiteStatistic(_PluginBase):
} }
], { ], {
"enabled": False, "enabled": False,
"onlyonce": False,
"notify": True, "notify": True,
"cron": "5 1 * * *", "cron": "5 1 * * *",
"queue_cnt": 5, "queue_cnt": 5,
"statistic_type": "all",
"statistic_sites": [] "statistic_sites": []
} }
@ -660,35 +714,35 @@ class SiteStatistic(_PluginBase):
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '站点' 'text': '站点'
}, },
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '用户名' 'text': '用户名'
}, },
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '用户等级' 'text': '用户等级'
}, },
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '上传量' 'text': '上传量'
}, },
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '下载量' 'text': '下载量'
}, },
@ -702,21 +756,21 @@ class SiteStatistic(_PluginBase):
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '魔力值' 'text': '魔力值'
}, },
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '做种数' 'text': '做种数'
}, },
{ {
'component': 'th', 'component': 'th',
'props': { 'props': {
'class': 'text-start ps-4' 'class': 'text-start ps-4'
}, },
'text': '做种体积' 'text': '做种体积'
} }
@ -956,7 +1010,7 @@ class SiteStatistic(_PluginBase):
self.post_message(channel=event.event_data.get("channel"), self.post_message(channel=event.event_data.get("channel"),
title="站点数据刷新完成！", userid=event.event_data.get("user")) title="站点数据刷新完成！", userid=event.event_data.get("user"))
def refresh_all_site_data(self, force: bool = False, specify_sites: list = None): def refresh_all_site_data(self, force: bool = False):
""" """
多线程刷新站点下载上传量，默认间隔6小时 多线程刷新站点下载上传量，默认间隔6小时
""" """
@ -968,22 +1022,15 @@ class SiteStatistic(_PluginBase):
with lock: with lock:
if not force \ if not force \
and not specify_sites \ and not self._statistic_sites:
and self._last_update_time:
return return
if specify_sites \
and not isinstance(specify_sites, list):
specify_sites = [specify_sites]
# 没有指定站点，默认使用全部站点 # 没有指定站点，默认使用全部站点
if not specify_sites: if not self._statistic_sites:
refresh_sites = [site for site in self.sites.get_indexers() if not site.get("public")] refresh_sites = [site for site in self.sites.get_indexers() if not site.get("public")]
else: else:
refresh_sites = [site for site in self.sites.get_indexers() if refresh_sites = [site for site in self.sites.get_indexers() if
site.get("name") in specify_sites] site.get("id") in self._statistic_sites]
# 过滤掉未选中的站点
refresh_sites = [site for site in refresh_sites if site.get("id") in self._statistic_sites]
if not refresh_sites: if not refresh_sites:
return return
@ -996,16 +1043,25 @@ class SiteStatistic(_PluginBase):
key = datetime.now().strftime('%Y-%m-%d') key = datetime.now().strftime('%Y-%m-%d')
# 保存数据 # 保存数据
self.save_data(key, self._sites_data) self.save_data(key, self._sites_data)
# 更新时间
self._last_update_time = datetime.now()
# 通知刷新完成 # 通知刷新完成
if self._notify: if self._notify:
yesterday_sites_data = {}
# 增量数据
if self._statistic_type == "add":
last_update_time = self.get_data("last_update_time")
if last_update_time:
yesterday_sites_data = self.get_data(last_update_time) or {}
messages = [] messages = []
# 按照上传降序排序 # 按照上传降序排序
sites = self._sites_data.keys() sites = self._sites_data.keys()
uploads = [self._sites_data[site].get("upload") or 0 for site in sites] uploads = [self._sites_data[site].get("upload") or 0 if not yesterday_sites_data.get(site) else
downloads = [self._sites_data[site].get("download") or 0 for site in sites] (self._sites_data[site].get("upload") or 0) - (
yesterday_sites_data[site].get("upload") or 0) for site in sites]
downloads = [self._sites_data[site].get("download") or 0 if not yesterday_sites_data.get(site) else
(self._sites_data[site].get("download") or 0) - (
yesterday_sites_data[site].get("download") or 0) for site in sites]
data_list = sorted(list(zip(sites, uploads, downloads)), data_list = sorted(list(zip(sites, uploads, downloads)),
key=lambda x: x[1], key=lambda x: x[1],
reverse=True) reverse=True)
@ -1030,7 +1086,20 @@ class SiteStatistic(_PluginBase):
f"总上传：{StringUtils.str_filesize(incUploads)}\n" f"总上传：{StringUtils.str_filesize(incUploads)}\n"
f"总下载：{StringUtils.str_filesize(incDownloads)}\n" f"总下载：{StringUtils.str_filesize(incDownloads)}\n"
f"————————————") f"————————————")
self.post_message(mtype=NotificationType.SiteMessage, self.post_message(mtype=NotificationType.SiteMessage,
title="站点数据统计", text="\n".join(messages)) title="站点数据统计", text="\n".join(messages))
# 更新时间
self.save_data("last_update_time", key)
logger.info("站点数据刷新完成") logger.info("站点数据刷新完成")
def __update_config(self):
self.update_config({
"enable": self._enabled,
"onlyonce": self._onlyonce,
"cron": self._cron,
"notify": self._notify,
"queue_cnt": self._queue_cnt,
"statistic_type": self._statistic_type,
"statistic_sites": self._statistic_sites,
})