Merge pull request #688 from thsrite/main

feat 定时清理媒体库插件

alembic/versions/a521fbc28b18_1_0_9.py

@@ -0,0 +1,30 @@
+"""1_0_9
+
+Revision ID: a521fbc28b18
+Revises: b2f011d3a8b7
+Create Date: 2023-09-28 13:37:16.479360
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a521fbc28b18'
+down_revision = 'b2f011d3a8b7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    try:
+        with op.batch_alter_table("downloadhistory") as batch_op:
+            batch_op.add_column(sa.Column('date', sa.String, nullable=True))
+            batch_op.add_column(sa.Column('channel', sa.String, nullable=True))
+    except Exception as e:
+        pass
+    # ### end Alembic commands ###
+
+def downgrade() -> None:
+    pass

app/chain/download.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import re
+import time
 from pathlib import Path
 from typing import List, Optional, Tuple, Set, Dict, Union
 
@@ -269,7 +270,9 @@ class DownloadChain(ChainBase):
                 torrent_name=_torrent.title,
                 torrent_description=_torrent.description,
                 torrent_site=_torrent.site_name,
-                userid=userid
+                userid=userid,
+                channel=channel.value if channel else None,
+                date=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
             )
 
             # 登记下载文件
@@ -321,7 +324,7 @@ class DownloadChain(ChainBase):
                        contexts: List[Context],
                        no_exists: Dict[int, Dict[int, NotExistMediaInfo]] = None,
                        save_path: str = None,
-                       channel: str = None,
+                       channel: MessageChannel = None,
                        userid: str = None) -> Tuple[List[Context], Dict[int, Dict[int, NotExistMediaInfo]]]:
         """
         根据缺失数据，自动种子列表中组合择优下载

app/chain/message.py

@@ -187,7 +187,7 @@ class MessageChain(ChainBase):
                     # 下载种子
                     context: Context = cache_list[int(text) - 1]
                     # 下载
-                    self.downloadchain.download_single(context, userid=userid)
+                    self.downloadchain.download_single(context, userid=userid, channel=channel)
 
         elif text.lower() == "p":
             # 上一页

app/db/downloadhistory_oper.py

@@ -108,3 +108,11 @@ class DownloadHistoryOper(DbOper):
                                            season=season,
                                            episode=episode,
                                            tmdbid=tmdbid)
+
+    def list_by_user_date(self, date: str, userid: str = None) -> List[DownloadHistory]:
+        """
+        查询某用户某时间之后的下载历史
+        """
+        return DownloadHistory.list_by_user_date(db=self._db,
+                                                 date=date,
+                                                 userid=userid)

app/db/models/downloadhistory.py

@@ -37,6 +37,10 @@ class DownloadHistory(Base):
     torrent_site = Column(String)
     # 下载用户
     userid = Column(String)
+    # 下载渠道
+    channel = Column(String)
+    # 创建时间
+    date = Column(String)
     # 附加信息
     note = Column(String)
 
@@ -92,6 +96,19 @@ class DownloadHistory(Base):
                                                     DownloadHistory.episodes == episode).order_by(
                 DownloadHistory.id.desc()).all()
 
+    @staticmethod
+    def list_by_user_date(db: Session, date: str, userid: str = None):
+        """
+        查询某用户某时间之后的下载历史
+        """
+        if userid:
+            return db.query(DownloadHistory).filter(DownloadHistory.date < date,
+                                                    DownloadHistory.userid == userid).order_by(
+                DownloadHistory.id.desc()).all()
+        else:
+            return db.query(DownloadHistory).filter(DownloadHistory.date < date).order_by(
+                DownloadHistory.id.desc()).all()
+
 
 class DownloadFiles(Base):
     """

app/db/models/transferhistory.py

@@ -65,6 +65,10 @@ class TransferHistory(Base):
     def get_by_src(db: Session, src: str):
         return db.query(TransferHistory).filter(TransferHistory.src == src).first()
 
+    @staticmethod
+    def list_by_hash(db: Session, download_hash: str):
+        return db.query(TransferHistory).filter(TransferHistory.download_hash == download_hash).all()
+
     @staticmethod
     def statistic(db: Session, days: int = 7):
         """

app/db/transferhistory_oper.py

@@ -36,6 +36,13 @@ class TransferHistoryOper(DbOper):
         """
         return TransferHistory.get_by_src(self._db, src)
 
+    def list_by_hash(self, download_hash: str) -> List[TransferHistory]:
+        """
+        按种子hash查询转移记录
+        :param download_hash: 种子hash
+        """
+        return TransferHistory.list_by_hash(self._db, download_hash)
+
     def add(self, **kwargs) -> TransferHistory:
         """
         新增转移历史

app/plugins/autoclean/__init__.py

@@ -0,0 +1,551 @@
+import time
+from collections import defaultdict
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pytz
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.cron import CronTrigger
+
+from app.chain.transfer import TransferChain
+from app.core.config import settings
+from app.core.event import eventmanager
+from app.db.downloadhistory_oper import DownloadHistoryOper
+from app.db.transferhistory_oper import TransferHistoryOper
+from app.plugins import _PluginBase
+from typing import Any, List, Dict, Tuple, Optional
+from app.log import logger
+from app.schemas import NotificationType, DownloadHistory
+from app.schemas.types import EventType
+
+
+class AutoClean(_PluginBase):
+    # 插件名称
+    plugin_name = "定时清理媒体库"
+    # 插件描述
+    plugin_desc = "定时清理用户下载的种子、源文件、媒体库文件。"
+    # 插件图标
+    plugin_icon = "clean.png"
+    # 主题色
+    plugin_color = "#3377ed"
+    # 插件版本
+    plugin_version = "1.0"
+    # 插件作者
+    plugin_author = "thsrite"
+    # 作者主页
+    author_url = "https://github.com/thsrite"
+    # 插件配置项ID前缀
+    plugin_config_prefix = "autoclean_"
+    # 加载顺序
+    plugin_order = 15
+    # 可使用的用户级别
+    auth_level = 2
+
+    # 私有属性
+    _enabled = False
+    # 任务执行间隔
+    _cron = None
+    _type = None
+    _onlyonce = False
+    _notify = False
+    _cleantype = None
+    _cleanuser = None
+    _cleandate = None
+    _downloadhis = None
+    _transferhis = None
+
+    # 定时器
+    _scheduler: Optional[BackgroundScheduler] = None
+
+    def init_plugin(self, config: dict = None):
+        # 停止现有任务
+        self.stop_service()
+
+        if config:
+            self._enabled = config.get("enabled")
+            self._cron = config.get("cron")
+            self._onlyonce = config.get("onlyonce")
+            self._notify = config.get("notify")
+            self._cleantype = config.get("cleantype")
+            self._cleanuser = config.get("cleanuser")
+            self._cleandate = config.get("cleandate")
+
+            # 加载模块
+        if self._enabled:
+            self._downloadhis = DownloadHistoryOper(self.db)
+            self._transferhis = TransferHistoryOper(self.db)
+            # 定时服务
+            self._scheduler = BackgroundScheduler(timezone=settings.TZ)
+
+            if self._cron:
+                try:
+                    self._scheduler.add_job(func=self.__clean,
+                                            trigger=CronTrigger.from_crontab(self._cron),
+                                            name="定时清理媒体库")
+                except Exception as err:
+                    logger.error(f"定时任务配置错误：{err}")
+
+            if self._onlyonce:
+                logger.info(f"定时清理媒体库服务启动，立即运行一次")
+                self._scheduler.add_job(func=self.__clean, trigger='date',
+                                        run_date=datetime.now(tz=pytz.timezone(settings.TZ)) + timedelta(seconds=3),
+                                        name="定时清理媒体库")
+                # 关闭一次性开关
+                self._onlyonce = False
+                self.update_config({
+                    "onlyonce": False,
+                    "cron": self._cron,
+                    "cleantype": self._cleantype,
+                    "enabled": self._enabled,
+                    "cleanuser": self._cleanuser,
+                    "cleandate": self._cleandate,
+                    "notify": self._notify,
+                })
+
+            # 启动任务
+            if self._scheduler.get_jobs():
+                self._scheduler.print_jobs()
+                self._scheduler.start()
+
+    def __clean(self):
+        """
+        定时清理媒体库
+        """
+        if not self._cleandate:
+            logger.error("未配置清理媒体库时间，停止运行")
+            return
+
+        # 清理日期
+        current_time = datetime.now()
+        days_ago = current_time - timedelta(days=int(self._cleandate))
+        clean_date = days_ago.strftime("%Y-%m-%d")
+
+        # 查询用户清理日期之后的下载历史
+        if not self._cleanuser:
+            downloadhis_list = self._downloadhis.list_by_user_date(date=clean_date)
+            logger.info(f'获取到日期 {clean_date} 之后的下载历史 {len(downloadhis_list)} 条')
+
+            self.__clean_history(date=clean_date, downloadhis_list=downloadhis_list)
+        else:
+            for userid in str(self._cleanuser).split(","):
+                downloadhis_list = self._downloadhis.list_by_user_date(date=clean_date,
+                                                                       userid=userid)
+                logger.info(
+                    f'获取到用户 {userid} 日期 {clean_date} 之后的下载历史 {len(downloadhis_list)} 条')
+                self.__clean_history(date=clean_date, downloadhis_list=downloadhis_list, userid=userid)
+
+    def __clean_history(self, date: str, downloadhis_list: List[DownloadHistory], userid: str = None):
+        """
+        清理下载历史、转移记录
+        """
+        if not downloadhis_list:
+            logger.warn(f"未获取到日期 {date} 之后的下载记录，停止运行")
+            return
+
+        # 读取历史记录
+        history = self.get_data('history') or []
+
+        # 创建一个字典来保存分组结果
+        downloadhis_grouped_dict = defaultdict(list)
+        # 遍历DownloadHistory对象列表
+        for downloadhis in downloadhis_list:
+            # 获取type和tmdbid的值
+            type = downloadhis.type
+            tmdbid = downloadhis.tmdbid
+
+            # 将DownloadHistory对象添加到对应分组的列表中
+            downloadhis_grouped_dict[(type, tmdbid)].append(downloadhis)
+
+        # 输出分组结果
+        for key, downloadhis_list in downloadhis_grouped_dict.items():
+            logger.info(f"开始清理 {key}")
+
+            del_transferhis_cnt = 0
+            del_media_name = downloadhis_list[0].title
+            del_media_user = downloadhis_list[0].userid
+            del_media_type = downloadhis_list[0].type
+            del_media_year = downloadhis_list[0].year
+            del_media_season = downloadhis_list[0].seasons
+            del_media_episode = downloadhis_list[0].episodes
+            del_image = downloadhis_list[0].image
+            for downloadhis in downloadhis_list:
+                if not downloadhis.download_hash:
+                    logger.debug(f'下载历史 {downloadhis.id} {downloadhis.title} 未获取到download_hash，跳过处理')
+                    continue
+                # 根据hash获取转移记录
+                transferhis_list = self._transferhis.list_by_hash(download_hash=downloadhis.download_hash)
+                if not transferhis_list:
+                    logger.warn(f"下载历史 {downloadhis.download_hash} 未查询到转移记录，跳过处理")
+                    continue
+
+                for history in transferhis_list:
+                    # 册除媒体库文件
+                    if str(self._cleantype == "dest") or str(self._cleantype == "all"):
+                        TransferChain(self.db).delete_files(Path(history.dest))
+                        # 删除记录
+                        self._transferhis.delete(history.id)
+                    # 删除源文件
+                    if str(self._cleantype == "src") or str(self._cleantype == "all"):
+                        TransferChain(self.db).delete_files(Path(history.src))
+                        # 发送事件
+                        eventmanager.send_event(
+                            EventType.DownloadFileDeleted,
+                            {
+                                "src": history.src
+                            }
+                        )
+
+                # 累加删除数量
+                del_transferhis_cnt += len(transferhis_list)
+
+            # 发送消息
+            if self._notify:
+                self.post_message(
+                    mtype=NotificationType.MediaServer,
+                    title="【定时清理媒体库任务完成】",
+                    text=f"清理媒体名称 {del_media_name}\n"
+                         f"下载媒体用户 {del_media_user}\n"
+                         f"删除历史记录 {del_transferhis_cnt}",
+                    userid=userid)
+
+            history.append({
+                "type": del_media_type,
+                "title": del_media_name,
+                "year": del_media_year,
+                "season": del_media_season,
+                "episode": del_media_episode,
+                "image": del_image,
+                "del_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
+            })
+
+        # 保存历史
+        self.save_data("history", history)
+
+    def get_state(self) -> bool:
+        return self._enabled
+
+    @staticmethod
+    def get_command() -> List[Dict[str, Any]]:
+        pass
+
+    def get_api(self) -> List[Dict[str, Any]]:
+        pass
+
+    def get_form(self) -> Tuple[List[dict], Dict[str, Any]]:
+        """
+        拼装插件配置页面，需要返回两块数据：1、页面配置；2、数据结构
+        """
+        return [
+                   {
+                       'component': 'VForm',
+                       'content': [
+                           {
+                               'component': 'VRow',
+                               'content': [
+                                   {
+                                       'component': 'VCol',
+                                       'props': {
+                                           'cols': 12,
+                                           'md': 4
+                                       },
+                                       'content': [
+                                           {
+                                               'component': 'VSwitch',
+                                               'props': {
+                                                   'model': 'enabled',
+                                                   'label': '启用插件',
+                                               }
+                                           }
+                                       ]
+                                   },
+                                   {
+                                       'component': 'VCol',
+                                       'props': {
+                                           'cols': 12,
+                                           'md': 4
+                                       },
+                                       'content': [
+                                           {
+                                               'component': 'VSwitch',
+                                               'props': {
+                                                   'model': 'onlyonce',
+                                                   'label': '立即运行一次',
+                                               }
+                                           }
+                                       ]
+                                   },
+                                   {
+                                       'component': 'VCol',
+                                       'props': {
+                                           'cols': 12,
+                                           'md': 4
+                                       },
+                                       'content': [
+                                           {
+                                               'component': 'VSwitch',
+                                               'props': {
+                                                   'model': 'notify',
+                                                   'label': '开启通知',
+                                               }
+                                           }
+                                       ]
+                                   }
+                               ]
+                           },
+                           {
+                               'component': 'VRow',
+                               'content': [
+                                   {
+                                       'component': 'VCol',
+                                       'props': {
+                                           'cols': 12,
+                                           'md': 4
+                                       },
+                                       'content': [
+                                           {
+                                               'component': 'VTextField',
+                                               'props': {
+                                                   'model': 'cron',
+                                                   'label': '执行周期',
+                                                   'placeholder': '0 0 ? ? ?'
+                                               }
+                                           }
+                                       ]
+                                   },
+                                   {
+                                       'component': 'VCol',
+                                       'props': {
+                                           'cols': 12,
+                                           'md': 4
+                                       },
+                                       'content': [
+                                           {
+                                               'component': 'VSelect',
+                                               'props': {
+                                                   'model': 'cleantype',
+                                                   'label': '清理方式',
+                                                   'items': [
+                                                       {'title': '媒体库文件', 'value': 'dest'},
+                                                       {'title': '源文件', 'value': 'src'},
+                                                       {'title': '所有文件', 'value': 'all'},
+                                                   ]
+                                               }
+                                           }
+                                       ]
+                                   },
+                                   {
+                                       'component': 'VCol',
+                                       'props': {
+                                           'cols': 12,
+                                           'md': 4
+                                       },
+                                       'content': [
+                                           {
+                                               'component': 'VTextField',
+                                               'props': {
+                                                   'model': 'cleandate',
+                                                   'label': '清理媒体日期',
+                                                   'placeholder': '清理多少天之前的下载记录（天）'
+                                               }
+                                           }
+                                       ]
+                                   }
+                               ]
+                           },
+                           {
+                               'component': 'VRow',
+                               'content': [
+                                   {
+                                       'component': 'VCol',
+                                       'props': {
+                                           'cols': 12,
+                                       },
+                                       'content': [
+                                           {
+                                               'component': 'VTextField',
+                                               'props': {
+                                                   'model': 'cleanuser',
+                                                   'label': '清理下载用户',
+                                                   'placeholder': '多个用户,分割'
+                                               }
+                                           }
+                                       ]
+                                   }
+                               ]
+                           }
+                       ]
+                   }
+               ], {
+                   "enabled": False,
+                   "onlyonce": False,
+                   "notify": False,
+                   "cleantype": "dest",
+                   "cron": "",
+                   "cleanuser": "",
+                   "cleandate": 30
+               }
+
+    def get_page(self) -> List[dict]:
+        """
+        拼装插件详情页面，需要返回页面配置，同时附带数据
+        """
+        # 查询同步详情
+        historys = self.get_data('history')
+        if not historys:
+            return [
+                {
+                    'component': 'div',
+                    'text': '暂无数据',
+                    'props': {
+                        'class': 'text-center',
+                    }
+                }
+            ]
+        # 数据按时间降序排序
+        historys = sorted(historys, key=lambda x: x.get('del_time'), reverse=True)
+        # 拼装页面
+        contents = []
+        for history in historys:
+            htype = history.get("type")
+            title = history.get("title")
+            year = history.get("year")
+            season = history.get("season")
+            episode = history.get("episode")
+            image = history.get("image")
+            del_time = history.get("del_time")
+
+            if season:
+                sub_contents = [
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'类型：{htype}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'标题：{title}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'年份：{year}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'季：{season}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'集：{episode}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'时间：{del_time}'
+                    }
+                ]
+            else:
+                sub_contents = [
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'类型：{htype}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'标题：{title}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'年份：{year}'
+                    },
+                    {
+                        'component': 'VCardText',
+                        'props': {
+                            'class': 'pa-0 px-2'
+                        },
+                        'text': f'时间：{del_time}'
+                    }
+                ]
+
+            contents.append(
+                {
+                    'component': 'VCard',
+                    'content': [
+                        {
+                            'component': 'div',
+                            'props': {
+                                'class': 'd-flex justify-space-start flex-nowrap flex-row',
+                            },
+                            'content': [
+                                {
+                                    'component': 'div',
+                                    'content': [
+                                        {
+                                            'component': 'VImg',
+                                            'props': {
+                                                'src': image,
+                                                'height': 120,
+                                                'width': 80,
+                                                'aspect-ratio': '2/3',
+                                                'class': 'object-cover shadow ring-gray-500',
+                                                'cover': True
+                                            }
+                                        }
+                                    ]
+                                },
+                                {
+                                    'component': 'div',
+                                    'content': sub_contents
+                                }
+                            ]
+                        }
+                    ]
+                }
+            )
+
+        return [
+            {
+                'component': 'div',
+                'props': {
+                    'class': 'grid gap-3 grid-info-card',
+                },
+                'content': contents
+            }
+        ]
+
+    def stop_service(self):
+        """
+        退出插件
+        """
+        try:
+            if self._scheduler:
+                self._scheduler.remove_all_jobs()
+                if self._scheduler.running:
+                    self._scheduler.shutdown()
+                self._scheduler = None
+        except Exception as e:
+            logger.error("退出插件失败：%s" % str(e))

app/plugins/downloadingmsg/__init__.py

@@ -8,7 +8,7 @@ from app.plugins import _PluginBase
 from typing import Any, List, Dict, Tuple, Optional, Union
 from app.log import logger
 from app.schemas import NotificationType, TransferTorrent, DownloadingTorrent
-from app.schemas.types import TorrentStatus
+from app.schemas.types import TorrentStatus, MessageChannel
 from app.utils.string import StringUtils
 
 
@@ -117,7 +117,8 @@ class DownloadingMsg(_PluginBase):
                 if not userid:
                     continue
                 # 如果用户是管理员，无需重复推送
-                if self._adminuser and userid in str(self._adminuser).split(","):
+                if self._type == "admin" or self._type == "both" and self._adminuser and userid in str(
+                        self._adminuser).split(","):
                     logger.debug("管理员已推送")
                     continue
 
@@ -138,6 +139,7 @@ class DownloadingMsg(_PluginBase):
         title = f"共 {len(torrents)} 个任务正在下载："
         messages = []
         index = 1
+        channel_value = None
         for torrent in torrents:
             year = None
             name = None
@@ -150,6 +152,8 @@ class DownloadingMsg(_PluginBase):
                 year = downloadhis.year
                 se = downloadhis.seasons
                 ep = downloadhis.episodes
+                if not channel_value:
+                    channel_value = downloadhis.channel
             else:
                 try:
                     context = MediaChain(self.db).recognize_by_title(title=torrent.title)
@@ -178,7 +182,15 @@ class DownloadingMsg(_PluginBase):
                             f"{StringUtils.str_filesize(torrent.size)} "
                             f"{round(torrent.progress, 1)}%")
             index += 1
+
+        # 用户消息渠道
+        if channel_value:
+            channel = next(
+                (channel for channel in MessageChannel.__members__.values() if channel.value == channel_value), None)
+        else:
+            channel = None
         self.post_message(mtype=NotificationType.Download,
+                          channel=channel,
                           title=title,
                           text="\n".join(messages),
                           userid=userid)

app/plugins/nastoolsync/__init__.py

@@ -214,6 +214,7 @@ class NAStoolSync(_PluginBase):
             mtorrent = history[9]
             mdesc = history[10]
             msite = history[11]
+            mdate = history[12]
 
             # 处理站点映射
             if self._site:
@@ -236,7 +237,8 @@ class NAStoolSync(_PluginBase):
                 torrent_name=mtorrent,
                 torrent_description=mdesc,
                 torrent_site=msite,
-                userid=settings.SUPERUSER
+                userid=settings.SUPERUSER,
+                date=mdate
             )
             cnt += 1
             if cnt % 100 == 0:
@@ -360,7 +362,8 @@ class NAStoolSync(_PluginBase):
             DOWNLOAD_ID,
             TORRENT,
             DESC,
-            SITE 
+            SITE,
+            DATE
         FROM
             DOWNLOAD_HISTORY 
         WHERE