Merge pull request #688 from thsrite/main

feat 定时清理媒体库插件
This commit is contained in:
jxxghp 2023-09-28 15:46:13 +08:00 committed by GitHub
commit bd4c639761
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 642 additions and 7 deletions

View File

@ -0,0 +1,30 @@
"""1_0_9
Revision ID: a521fbc28b18
Revises: b2f011d3a8b7
Create Date: 2023-09-28 13:37:16.479360
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'a521fbc28b18'
down_revision = 'b2f011d3a8b7'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
try:
with op.batch_alter_table("downloadhistory") as batch_op:
batch_op.add_column(sa.Column('date', sa.String, nullable=True))
batch_op.add_column(sa.Column('channel', sa.String, nullable=True))
except Exception as e:
pass
# ### end Alembic commands ###
def downgrade() -> None:
pass

View File

@ -1,6 +1,7 @@
import base64
import json
import re
import time
from pathlib import Path
from typing import List, Optional, Tuple, Set, Dict, Union
@ -269,7 +270,9 @@ class DownloadChain(ChainBase):
torrent_name=_torrent.title,
torrent_description=_torrent.description,
torrent_site=_torrent.site_name,
userid=userid
userid=userid,
channel=channel.value if channel else None,
date=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
)
# 登记下载文件
@ -321,7 +324,7 @@ class DownloadChain(ChainBase):
contexts: List[Context],
no_exists: Dict[int, Dict[int, NotExistMediaInfo]] = None,
save_path: str = None,
channel: str = None,
channel: MessageChannel = None,
userid: str = None) -> Tuple[List[Context], Dict[int, Dict[int, NotExistMediaInfo]]]:
"""
根据缺失数据自动种子列表中组合择优下载

View File

@ -187,7 +187,7 @@ class MessageChain(ChainBase):
# 下载种子
context: Context = cache_list[int(text) - 1]
# 下载
self.downloadchain.download_single(context, userid=userid)
self.downloadchain.download_single(context, userid=userid, channel=channel)
elif text.lower() == "p":
# 上一页

View File

@ -108,3 +108,11 @@ class DownloadHistoryOper(DbOper):
season=season,
episode=episode,
tmdbid=tmdbid)
def list_by_user_date(self, date: str, userid: str = None) -> List[DownloadHistory]:
"""
查询某用户某时间之后的下载历史
"""
return DownloadHistory.list_by_user_date(db=self._db,
date=date,
userid=userid)

View File

@ -37,6 +37,10 @@ class DownloadHistory(Base):
torrent_site = Column(String)
# 下载用户
userid = Column(String)
# 下载渠道
channel = Column(String)
# 创建时间
date = Column(String)
# 附加信息
note = Column(String)
@ -92,6 +96,19 @@ class DownloadHistory(Base):
DownloadHistory.episodes == episode).order_by(
DownloadHistory.id.desc()).all()
@staticmethod
def list_by_user_date(db: Session, date: str, userid: str = None):
"""
查询某用户某时间之后的下载历史
"""
if userid:
return db.query(DownloadHistory).filter(DownloadHistory.date < date,
DownloadHistory.userid == userid).order_by(
DownloadHistory.id.desc()).all()
else:
return db.query(DownloadHistory).filter(DownloadHistory.date < date).order_by(
DownloadHistory.id.desc()).all()
class DownloadFiles(Base):
"""

View File

@ -65,6 +65,10 @@ class TransferHistory(Base):
def get_by_src(db: Session, src: str):
return db.query(TransferHistory).filter(TransferHistory.src == src).first()
@staticmethod
def list_by_hash(db: Session, download_hash: str):
return db.query(TransferHistory).filter(TransferHistory.download_hash == download_hash).all()
@staticmethod
def statistic(db: Session, days: int = 7):
"""

View File

@ -36,6 +36,13 @@ class TransferHistoryOper(DbOper):
"""
return TransferHistory.get_by_src(self._db, src)
def list_by_hash(self, download_hash: str) -> List[TransferHistory]:
"""
按种子hash查询转移记录
:param download_hash: 种子hash
"""
return TransferHistory.list_by_hash(self._db, download_hash)
def add(self, **kwargs) -> TransferHistory:
"""
新增转移历史

View File

@ -0,0 +1,551 @@
import time
from collections import defaultdict
from datetime import datetime, timedelta
from pathlib import Path
import pytz
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from app.chain.transfer import TransferChain
from app.core.config import settings
from app.core.event import eventmanager
from app.db.downloadhistory_oper import DownloadHistoryOper
from app.db.transferhistory_oper import TransferHistoryOper
from app.plugins import _PluginBase
from typing import Any, List, Dict, Tuple, Optional
from app.log import logger
from app.schemas import NotificationType, DownloadHistory
from app.schemas.types import EventType
class AutoClean(_PluginBase):
# 插件名称
plugin_name = "定时清理媒体库"
# 插件描述
plugin_desc = "定时清理用户下载的种子、源文件、媒体库文件。"
# 插件图标
plugin_icon = "clean.png"
# 主题色
plugin_color = "#3377ed"
# 插件版本
plugin_version = "1.0"
# 插件作者
plugin_author = "thsrite"
# 作者主页
author_url = "https://github.com/thsrite"
# 插件配置项ID前缀
plugin_config_prefix = "autoclean_"
# 加载顺序
plugin_order = 15
# 可使用的用户级别
auth_level = 2
# 私有属性
_enabled = False
# 任务执行间隔
_cron = None
_type = None
_onlyonce = False
_notify = False
_cleantype = None
_cleanuser = None
_cleandate = None
_downloadhis = None
_transferhis = None
# 定时器
_scheduler: Optional[BackgroundScheduler] = None
def init_plugin(self, config: dict = None):
# 停止现有任务
self.stop_service()
if config:
self._enabled = config.get("enabled")
self._cron = config.get("cron")
self._onlyonce = config.get("onlyonce")
self._notify = config.get("notify")
self._cleantype = config.get("cleantype")
self._cleanuser = config.get("cleanuser")
self._cleandate = config.get("cleandate")
# 加载模块
if self._enabled:
self._downloadhis = DownloadHistoryOper(self.db)
self._transferhis = TransferHistoryOper(self.db)
# 定时服务
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
if self._cron:
try:
self._scheduler.add_job(func=self.__clean,
trigger=CronTrigger.from_crontab(self._cron),
name="定时清理媒体库")
except Exception as err:
logger.error(f"定时任务配置错误:{err}")
if self._onlyonce:
logger.info(f"定时清理媒体库服务启动,立即运行一次")
self._scheduler.add_job(func=self.__clean, trigger='date',
run_date=datetime.now(tz=pytz.timezone(settings.TZ)) + timedelta(seconds=3),
name="定时清理媒体库")
# 关闭一次性开关
self._onlyonce = False
self.update_config({
"onlyonce": False,
"cron": self._cron,
"cleantype": self._cleantype,
"enabled": self._enabled,
"cleanuser": self._cleanuser,
"cleandate": self._cleandate,
"notify": self._notify,
})
# 启动任务
if self._scheduler.get_jobs():
self._scheduler.print_jobs()
self._scheduler.start()
def __clean(self):
"""
定时清理媒体库
"""
if not self._cleandate:
logger.error("未配置清理媒体库时间,停止运行")
return
# 清理日期
current_time = datetime.now()
days_ago = current_time - timedelta(days=int(self._cleandate))
clean_date = days_ago.strftime("%Y-%m-%d")
# 查询用户清理日期之后的下载历史
if not self._cleanuser:
downloadhis_list = self._downloadhis.list_by_user_date(date=clean_date)
logger.info(f'获取到日期 {clean_date} 之后的下载历史 {len(downloadhis_list)}')
self.__clean_history(date=clean_date, downloadhis_list=downloadhis_list)
else:
for userid in str(self._cleanuser).split(","):
downloadhis_list = self._downloadhis.list_by_user_date(date=clean_date,
userid=userid)
logger.info(
f'获取到用户 {userid} 日期 {clean_date} 之后的下载历史 {len(downloadhis_list)}')
self.__clean_history(date=clean_date, downloadhis_list=downloadhis_list, userid=userid)
def __clean_history(self, date: str, downloadhis_list: List[DownloadHistory], userid: str = None):
"""
清理下载历史转移记录
"""
if not downloadhis_list:
logger.warn(f"未获取到日期 {date} 之后的下载记录,停止运行")
return
# 读取历史记录
history = self.get_data('history') or []
# 创建一个字典来保存分组结果
downloadhis_grouped_dict = defaultdict(list)
# 遍历DownloadHistory对象列表
for downloadhis in downloadhis_list:
# 获取type和tmdbid的值
type = downloadhis.type
tmdbid = downloadhis.tmdbid
# 将DownloadHistory对象添加到对应分组的列表中
downloadhis_grouped_dict[(type, tmdbid)].append(downloadhis)
# 输出分组结果
for key, downloadhis_list in downloadhis_grouped_dict.items():
logger.info(f"开始清理 {key}")
del_transferhis_cnt = 0
del_media_name = downloadhis_list[0].title
del_media_user = downloadhis_list[0].userid
del_media_type = downloadhis_list[0].type
del_media_year = downloadhis_list[0].year
del_media_season = downloadhis_list[0].seasons
del_media_episode = downloadhis_list[0].episodes
del_image = downloadhis_list[0].image
for downloadhis in downloadhis_list:
if not downloadhis.download_hash:
logger.debug(f'下载历史 {downloadhis.id} {downloadhis.title} 未获取到download_hash跳过处理')
continue
# 根据hash获取转移记录
transferhis_list = self._transferhis.list_by_hash(download_hash=downloadhis.download_hash)
if not transferhis_list:
logger.warn(f"下载历史 {downloadhis.download_hash} 未查询到转移记录,跳过处理")
continue
for history in transferhis_list:
# 册除媒体库文件
if str(self._cleantype == "dest") or str(self._cleantype == "all"):
TransferChain(self.db).delete_files(Path(history.dest))
# 删除记录
self._transferhis.delete(history.id)
# 删除源文件
if str(self._cleantype == "src") or str(self._cleantype == "all"):
TransferChain(self.db).delete_files(Path(history.src))
# 发送事件
eventmanager.send_event(
EventType.DownloadFileDeleted,
{
"src": history.src
}
)
# 累加删除数量
del_transferhis_cnt += len(transferhis_list)
# 发送消息
if self._notify:
self.post_message(
mtype=NotificationType.MediaServer,
title="【定时清理媒体库任务完成】",
text=f"清理媒体名称 {del_media_name}\n"
f"下载媒体用户 {del_media_user}\n"
f"删除历史记录 {del_transferhis_cnt}",
userid=userid)
history.append({
"type": del_media_type,
"title": del_media_name,
"year": del_media_year,
"season": del_media_season,
"episode": del_media_episode,
"image": del_image,
"del_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
})
# 保存历史
self.save_data("history", history)
def get_state(self) -> bool:
return self._enabled
@staticmethod
def get_command() -> List[Dict[str, Any]]:
pass
def get_api(self) -> List[Dict[str, Any]]:
pass
def get_form(self) -> Tuple[List[dict], Dict[str, Any]]:
"""
拼装插件配置页面需要返回两块数据1页面配置2数据结构
"""
return [
{
'component': 'VForm',
'content': [
{
'component': 'VRow',
'content': [
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VSwitch',
'props': {
'model': 'enabled',
'label': '启用插件',
}
}
]
},
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VSwitch',
'props': {
'model': 'onlyonce',
'label': '立即运行一次',
}
}
]
},
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VSwitch',
'props': {
'model': 'notify',
'label': '开启通知',
}
}
]
}
]
},
{
'component': 'VRow',
'content': [
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VTextField',
'props': {
'model': 'cron',
'label': '执行周期',
'placeholder': '0 0 ? ? ?'
}
}
]
},
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VSelect',
'props': {
'model': 'cleantype',
'label': '清理方式',
'items': [
{'title': '媒体库文件', 'value': 'dest'},
{'title': '源文件', 'value': 'src'},
{'title': '所有文件', 'value': 'all'},
]
}
}
]
},
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 4
},
'content': [
{
'component': 'VTextField',
'props': {
'model': 'cleandate',
'label': '清理媒体日期',
'placeholder': '清理多少天之前的下载记录(天)'
}
}
]
}
]
},
{
'component': 'VRow',
'content': [
{
'component': 'VCol',
'props': {
'cols': 12,
},
'content': [
{
'component': 'VTextField',
'props': {
'model': 'cleanuser',
'label': '清理下载用户',
'placeholder': '多个用户,分割'
}
}
]
}
]
}
]
}
], {
"enabled": False,
"onlyonce": False,
"notify": False,
"cleantype": "dest",
"cron": "",
"cleanuser": "",
"cleandate": 30
}
def get_page(self) -> List[dict]:
"""
拼装插件详情页面需要返回页面配置同时附带数据
"""
# 查询同步详情
historys = self.get_data('history')
if not historys:
return [
{
'component': 'div',
'text': '暂无数据',
'props': {
'class': 'text-center',
}
}
]
# 数据按时间降序排序
historys = sorted(historys, key=lambda x: x.get('del_time'), reverse=True)
# 拼装页面
contents = []
for history in historys:
htype = history.get("type")
title = history.get("title")
year = history.get("year")
season = history.get("season")
episode = history.get("episode")
image = history.get("image")
del_time = history.get("del_time")
if season:
sub_contents = [
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'类型:{htype}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'标题:{title}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'年份:{year}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'季:{season}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'集:{episode}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'时间:{del_time}'
}
]
else:
sub_contents = [
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'类型:{htype}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'标题:{title}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'年份:{year}'
},
{
'component': 'VCardText',
'props': {
'class': 'pa-0 px-2'
},
'text': f'时间:{del_time}'
}
]
contents.append(
{
'component': 'VCard',
'content': [
{
'component': 'div',
'props': {
'class': 'd-flex justify-space-start flex-nowrap flex-row',
},
'content': [
{
'component': 'div',
'content': [
{
'component': 'VImg',
'props': {
'src': image,
'height': 120,
'width': 80,
'aspect-ratio': '2/3',
'class': 'object-cover shadow ring-gray-500',
'cover': True
}
}
]
},
{
'component': 'div',
'content': sub_contents
}
]
}
]
}
)
return [
{
'component': 'div',
'props': {
'class': 'grid gap-3 grid-info-card',
},
'content': contents
}
]
def stop_service(self):
"""
退出插件
"""
try:
if self._scheduler:
self._scheduler.remove_all_jobs()
if self._scheduler.running:
self._scheduler.shutdown()
self._scheduler = None
except Exception as e:
logger.error("退出插件失败:%s" % str(e))

View File

@ -8,7 +8,7 @@ from app.plugins import _PluginBase
from typing import Any, List, Dict, Tuple, Optional, Union
from app.log import logger
from app.schemas import NotificationType, TransferTorrent, DownloadingTorrent
from app.schemas.types import TorrentStatus
from app.schemas.types import TorrentStatus, MessageChannel
from app.utils.string import StringUtils
@ -117,7 +117,8 @@ class DownloadingMsg(_PluginBase):
if not userid:
continue
# 如果用户是管理员,无需重复推送
if self._adminuser and userid in str(self._adminuser).split(","):
if self._type == "admin" or self._type == "both" and self._adminuser and userid in str(
self._adminuser).split(","):
logger.debug("管理员已推送")
continue
@ -138,6 +139,7 @@ class DownloadingMsg(_PluginBase):
title = f"{len(torrents)} 个任务正在下载:"
messages = []
index = 1
channel_value = None
for torrent in torrents:
year = None
name = None
@ -150,6 +152,8 @@ class DownloadingMsg(_PluginBase):
year = downloadhis.year
se = downloadhis.seasons
ep = downloadhis.episodes
if not channel_value:
channel_value = downloadhis.channel
else:
try:
context = MediaChain(self.db).recognize_by_title(title=torrent.title)
@ -178,7 +182,15 @@ class DownloadingMsg(_PluginBase):
f"{StringUtils.str_filesize(torrent.size)} "
f"{round(torrent.progress, 1)}%")
index += 1
# 用户消息渠道
if channel_value:
channel = next(
(channel for channel in MessageChannel.__members__.values() if channel.value == channel_value), None)
else:
channel = None
self.post_message(mtype=NotificationType.Download,
channel=channel,
title=title,
text="\n".join(messages),
userid=userid)

View File

@ -214,6 +214,7 @@ class NAStoolSync(_PluginBase):
mtorrent = history[9]
mdesc = history[10]
msite = history[11]
mdate = history[12]
# 处理站点映射
if self._site:
@ -236,7 +237,8 @@ class NAStoolSync(_PluginBase):
torrent_name=mtorrent,
torrent_description=mdesc,
torrent_site=msite,
userid=settings.SUPERUSER
userid=settings.SUPERUSER,
date=mdate
)
cnt += 1
if cnt % 100 == 0:
@ -360,7 +362,8 @@ class NAStoolSync(_PluginBase):
DOWNLOAD_ID,
TORRENT,
DESC,
SITE
SITE,
DATE
FROM
DOWNLOAD_HISTORY
WHERE