fix DirMonitor

app/chain/__init__.py

@@ -94,16 +94,6 @@ class ChainBase(metaclass=ABCMeta):
                 logger.error(f"运行模块 {method} 出错：{module.__class__.__name__} - {err}\n{traceback.print_exc()}")
         return result
 
-    def prepare_recognize(self, title: str,
-                          subtitle: str = None) -> Tuple[str, str]:
-        """
-        处理各类特别命名，以便识别
-        :param title:     标题
-        :param subtitle:  副标题
-        :return: 处理后的标题、副标题，该方法可被多个模块同时处理
-        """
-        return self.run_module("prepare_recognize", title=title, subtitle=subtitle)
-
     def recognize_media(self, meta: MetaBase = None,
                         mtype: MediaType = None,
                         tmdbid: int = None) -> Optional[MediaInfo]:

app/chain/media.py

@@ -18,10 +18,6 @@ class MediaChain(ChainBase):
         根据主副标题识别媒体信息
         """
         logger.info(f'开始识别媒体信息，标题：{title}，副标题：{subtitle} ...')
-        # 识别前预处理
-        result: Optional[tuple] = self.prepare_recognize(title=title, subtitle=subtitle)
-        if result:
-            title, subtitle = result
         # 识别元数据
         metainfo = MetaInfo(title, subtitle)
         # 识别媒体信息

app/chain/search.py

@@ -157,14 +157,8 @@ class SearchChain(ChainBase):
                     logger.info(f'{mediainfo.title} 匹配到资源：{torrent.site_name} - {torrent.title}')
                     _match_torrents.append(torrent)
                     continue
-                # 识别前预处理
-                result: Optional[tuple] = self.prepare_recognize(title=torrent.title, subtitle=torrent.description)
-                if result:
-                    title, subtitle = result
-                else:
-                    title, subtitle = torrent.title, torrent.description
                 # 识别
-                torrent_meta = MetaInfo(title=title, subtitle=subtitle)
+                torrent_meta = MetaInfo(title=torrent.title, subtitle=torrent.description)
                 # 比对类型
                 if torrent_meta.type == MediaType.TV and mediainfo.type != MediaType.TV:
                     logger.warn(f'{torrent.site_name} - {torrent.title} 类型不匹配')

app/chain/subscribe.py

@@ -52,10 +52,6 @@ class SubscribeChain(ChainBase):
         识别媒体信息并添加订阅
         """
         logger.info(f'开始添加订阅，标题：{title} ...')
-        # 识别前预处理
-        result: Optional[tuple] = self.prepare_recognize(title=title)
-        if result:
-            title, _ = result
         # 识别元数据
         metainfo = MetaInfo(title)
         if year:
@@ -385,15 +381,8 @@ class SubscribeChain(ChainBase):
                     continue
                 for torrent in torrents:
                     logger.info(f'处理资源：{torrent.title} ...')
-                    # 识别前预处理
-                    result: Optional[tuple] = self.prepare_recognize(title=torrent.title,
-                                                                     subtitle=torrent.description)
-                    if result:
-                        title, subtitle = result
-                    else:
-                        title, subtitle = torrent.title, torrent.description
                     # 识别
-                    meta = MetaInfo(title=title, subtitle=subtitle)
+                    meta = MetaInfo(title=torrent.title, subtitle=torrent.description)
                     # 识别媒体信息
                     mediainfo: MediaInfo = self.recognize_media(meta=meta)
                     if not mediainfo:

app/chain/transfer.py

@@ -89,16 +89,10 @@ class TransferChain(ChainBase):
             self.progress.update(value=processed_num / total_num * 100,
                                  text=f"正在转移 {torrent.title} ...",
                                  key=ProgressKey.FileTransfer)
-            # 识别前预处理
-            result: Optional[tuple] = self.prepare_recognize(title=torrent.title)
-            if result:
-                title, subtitle = result
-            else:
-                title, subtitle = torrent.title, None
             # 识别元数据
-            meta: MetaBase = MetaInfo(title=title, subtitle=subtitle)
+            meta: MetaBase = MetaInfo(title=torrent.title)
             if not meta.name:
-                logger.error(f'未识别到元数据，标题：{title}')
+                logger.error(f'未识别到元数据，标题：{torrent.title}')
                 continue
             if not arg_mediainfo:
                 # 查询下载记录识别情况
@@ -110,7 +104,7 @@ class TransferChain(ChainBase):
                     if mtype == MediaType.TV \
                             and ((not meta.season_list and downloadhis.seasons)
                                  or (not meta.episode_list and downloadhis.episodes)):
-                        meta = MetaInfo(f"{title} {downloadhis.seasons} {downloadhis.episodes}")
+                        meta = MetaInfo(f"{torrent.title} {downloadhis.seasons} {downloadhis.episodes}")
                     # 按TMDBID识别
                     mediainfo = self.recognize_media(mtype=mtype,
                                                      tmdbid=downloadhis.tmdbid)
@@ -195,7 +189,7 @@ class TransferChain(ChainBase):
             )
             # 转移完成
             self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo)
-            # 刮剥
+            # 刮削元数据
             self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo)
             # 刷新媒体库
             self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path)

app/db/models/transferhistory.py

@@ -30,7 +30,7 @@ class TransferHistory(Base):
     tvdbid = Column(Integer)
     doubanid = Column(String)
     # Sxx
-    seasons = Column(Integer)
+    seasons = Column(String)
     # Exx
     episodes = Column(String)
     # 海报
@@ -59,6 +59,10 @@ class TransferHistory(Base):
     def get_by_hash(db: Session, download_hash: str):
         return db.query(TransferHistory).filter(TransferHistory.download_hash == download_hash).first()
 
+    @staticmethod
+    def get_by_src(db: Session, src: str):
+        return db.query(TransferHistory).filter(TransferHistory.src == src).first()
+
     @staticmethod
     def statistic(db: Session, days: int = 7):
         """

app/db/transferhistory_oper.py

@@ -17,6 +17,13 @@ class TransferHistoryOper(DbOper):
         """
         return TransferHistory.search_by_title(self._db, title)
 
+    def get_by_src(self, src: str) -> Any:
+        """
+        按源查询转移记录
+        :param src: 数据key
+        """
+        return TransferHistory.get_by_src(self._db, src)
+
     def add(self, **kwargs):
         """
         新增转移历史
@@ -36,7 +43,8 @@ class TransferHistoryOper(DbOper):
         """
         return TransferHistory.statistic(self._db, days)
 
-    def get_by(self, mtype: str, title: str, year: int, season=None, episode=None) -> Any:
+    def get_by(self, mtype: str, title: str, year: int,
+               season: str = None, episode: str = None) -> Any:
         """
         按类型、标题、年份、季集查询转移记录
         """

app/plugins/__init__.py

@@ -4,6 +4,7 @@ from typing import Any, List, Dict, Tuple
 
 from app.chain import ChainBase
 from app.core.config import settings
+from app.core.event import EventManager
 from app.db.models import Base
 from app.db.plugindata_oper import PluginDataOper
 from app.db.systemconfig_oper import SystemConfigOper
@@ -44,6 +45,8 @@ class _PluginBase(metaclass=ABCMeta):
         self.systemconfig = SystemConfigOper()
         # 系统消息
         self.systemmessage = MessageHelper()
+        # 事件管理器
+        self.eventmanager = EventManager()
 
     @abstractmethod
     def init_plugin(self, config: dict = None):

app/plugins/dirmonitor/__init__.py

@@ -1,6 +1,39 @@
+import threading
+import traceback
+from pathlib import Path
 from typing import List, Tuple, Dict, Any
 
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
+from watchdog.observers.polling import PollingObserver
+
+from app.core.config import settings
+from app.core.context import MediaInfo
+from app.core.metainfo import MetaInfo
+from app.db.transferhistory_oper import TransferHistoryOper
+from app.log import logger
 from app.plugins import _PluginBase
+from app.schemas import MediaType, Notification, NotificationType, TransferInfo
+from app.schemas.types import EventType
+
+lock = threading.Lock()
+
+
+class FileMonitorHandler(FileSystemEventHandler):
+    """
+    目录监控响应类
+    """
+
+    def __init__(self, monpath: str, sync: Any, **kwargs):
+        super(FileMonitorHandler, self).__init__(**kwargs)
+        self._watch_path = monpath
+        self.sync = sync
+
+    def on_created(self, event):
+        self.sync.file_change_handler(event, "创建", event.src_path)
+
+    def on_moved(self, event):
+        self.sync.file_change_handler(event, "移动", event.dest_path)
 
 
 class DirMonitor(_PluginBase):
@@ -25,19 +58,205 @@ class DirMonitor(_PluginBase):
     # 可使用的用户级别
     user_level = 1
 
+    # 已处理的文件清单
+    _synced_files = []
+
     # 私有属性
-    _monitor = None
+    transferhis = None
+    _observer = []
     _enabled = False
+    # 模式 compatibility/fast
+    _mode = "fast"
+    # 转移方式
+    _transfer_type = settings.TRANSFER_TYPE
+    _monitor_dirs = ""
+    _exclude_keywords = ""
 
     def init_plugin(self, config: dict = None):
+        self.transferhis = TransferHistoryOper()
+
         # 读取配置
         if config:
             self._enabled = config.get("enabled")
+            self._mode = config.get("mode")
+            self._transfer_type = config.get("transfer_type")
+            self._monitor_dirs = config.get("monitor_dirs") or ""
+            self._exclude_keywords = config.get("exclude_keywords") or ""
 
         # 停止现有任务
         self.stop_service()
 
-        # TODO 启动任务
+        # 启动任务
+        monitor_dirs = self._monitor_dirs.split("\n")
+        if not monitor_dirs:
+            return
+        for mon_path in monitor_dirs:
+            if not mon_path:
+                continue
+            # 检查目录是不是媒体库目录的子目录
+            if Path(mon_path).is_relative_to(settings.LIBRARY_PATH):
+                logger.warn(f"{mon_path} 是媒体库目录的子目录，无法监控")
+                self.systemmessage.put(f"{mon_path} 是媒体库目录的子目录，无法监控")
+                continue
+
+            try:
+                if self._mode == "compatibility":
+                    # 兼容模式，目录同步性能降低且NAS不能休眠，但可以兼容挂载的远程共享目录如SMB
+                    observer = PollingObserver(timeout=10)
+                else:
+                    # 内部处理系统操作类型选择最优解
+                    observer = Observer(timeout=10)
+                self._observer.append(observer)
+                observer.schedule(FileMonitorHandler(mon_path, self), path=mon_path, recursive=True)
+                observer.daemon = True
+                observer.start()
+                logger.info(f"{mon_path} 的目录监控服务启动")
+            except Exception as e:
+                err_msg = str(e)
+                if "inotify" in err_msg and "reached" in err_msg:
+                    logger.warn(f"目录监控服务启动出现异常：{err_msg}，请在宿主机上（不是docker容器内）执行以下命令并重启："
+                                + """
+                             echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf
+                             echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf
+                             sudo sysctl -p
+                             """)
+                else:
+                    logger.error(f"{mon_path} 启动目录监控失败：{err_msg}")
+                self.systemmessage.put(f"{mon_path} 启动目录监控失败：{err_msg}")
+
+    def file_change_handler(self, event, text: str, event_path: str):
+        """
+        处理文件变化
+        :param event: 事件
+        :param text: 事件描述
+        :param event_path: 事件文件路径
+        """
+        if not event.is_directory:
+            # 文件发生变化
+            file_path = Path(event_path)
+            try:
+                if not file_path.exists():
+                    return
+
+                logger.debug("文件%s：%s" % (text, event_path))
+
+                # 全程加锁
+                with lock:
+                    if event_path not in self._synced_files:
+                        self._synced_files.append(event_path)
+                    else:
+                        logger.debug("文件已处理过：%s" % event_path)
+                        return
+
+                    # 命中过滤关键字不处理
+                    if self._exclude_keywords:
+                        for keyword in self._exclude_keywords.split("\n"):
+                            if keyword and keyword in event_path:
+                                logger.debug(f"{event_path} 命中过滤关键字 {keyword}")
+                                return
+
+                    # 回收站及隐藏的文件不处理
+                    if event_path.find('/@Recycle/') != -1 \
+                            or event_path.find('/#recycle/') != -1 \
+                            or event_path.find('/.') != -1 \
+                            or event_path.find('/@eaDir') != -1:
+                        logger.debug(f"{event_path} 是回收站或隐藏的文件")
+                        return
+
+                    # 不是媒体文件不处理
+                    if file_path.suffix not in settings.RMT_MEDIAEXT:
+                        logger.debug(f"{event_path} 不是媒体文件")
+                        return
+
+                    # 查询历史记录，已转移的不处理
+                    if self.transferhis.get_by_src(event_path):
+                        logger.info(f"{event_path} 已整理过")
+                        return
+
+                    # 文件元数据
+                    file_meta = MetaInfo(title=file_path.name)
+                    # 上级目录元数据
+                    dir_meta = MetaInfo(title=file_path.parent.name)
+                    # 整合元数据
+                    if not file_meta.cn_name and dir_meta.cn_name:
+                        file_meta.cn_name = dir_meta.cn_name
+                    if not file_meta.en_name and dir_meta.en_name:
+                        file_meta.en_name = dir_meta.en_name
+                    if file_meta.type != MediaType.TV and dir_meta.type == MediaType.TV:
+                        file_meta.type = MediaType.TV
+                    if not file_meta.year and dir_meta.year:
+                        file_meta.year = dir_meta.year
+                    if not file_meta.begin_season and dir_meta.begin_season:
+                        file_meta.begin_season = dir_meta.begin_season
+                    if not file_meta.episode_list and dir_meta.episode_list:
+                        file_meta.begin_episode = dir_meta.begin_episode
+                        file_meta.end_episode = dir_meta.end_episode
+
+                    if not file_meta.name:
+                        logger.warn(f"{file_path.name} 无法识别有效信息")
+                        return
+
+                    # 识别媒体信息
+                    mediainfo: MediaInfo = self.chain.recognize_media(meta=file_meta)
+                    if not mediainfo:
+                        logger.warn(f'未识别到媒体信息，标题：{file_meta.name}')
+                        self.chain.post_message(Notification(
+                            mtype=NotificationType.Manual,
+                            title=f"{file_path.name} 未识别到媒体信息，无法入库！"
+                        ))
+                        return
+                    logger.info(f"{file_path.name} 识别为：{mediainfo.type.value} {mediainfo.title_year}")
+
+                    # 更新媒体图片
+                    self.chain.obtain_images(mediainfo=mediainfo)
+
+                    # 转移
+                    transferinfo: TransferInfo = self.chain.transfer(mediainfo=mediainfo, path=file_path)
+
+                    if not transferinfo or not transferinfo.target_path:
+                        # 转移失败
+                        logger.warn(f"{file_path.name} 入库失败")
+                        self.chain.post_message(Notification(
+                            title=f"{mediainfo.title_year}{file_meta.season_episode} 入库失败！",
+                            text=f"原因：{transferinfo.message if transferinfo else '未知'}",
+                            image=mediainfo.get_message_image()
+                        ))
+                        return
+
+                    # 新增转移成功历史记录
+                    self.transferhis.add(
+                        src=event_path,
+                        dest=str(transferinfo.target_path) if transferinfo else None,
+                        mode=settings.TRANSFER_TYPE,
+                        type=mediainfo.type.value,
+                        category=mediainfo.category,
+                        title=mediainfo.title,
+                        year=mediainfo.year,
+                        tmdbid=mediainfo.tmdb_id,
+                        imdbid=mediainfo.imdb_id,
+                        tvdbid=mediainfo.tvdb_id,
+                        doubanid=mediainfo.douban_id,
+                        seasons=file_meta.season,
+                        episodes=file_meta.episode,
+                        image=mediainfo.get_poster_image(),
+                        status=1
+                    )
+
+                    # 刮削元数据
+                    self.chain.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo)
+                    # 刷新媒体库
+                    self.chain.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path)
+                    # 发送通知
+                    self.chain.__send_transfer_message(meta=file_meta, mediainfo=mediainfo, transferinfo=transferinfo)
+                    # 广播事件
+                    self.eventmanager.send_event(EventType.TransferComplete, {
+                        'meta': file_meta,
+                        'mediainfo': mediainfo,
+                        'transferinfo': transferinfo
+                    })
+
+            except Exception as e:
+                logger.error("目录监控发生错误：%s - %s" % (str(e), traceback.format_exc()))
 
     def get_state(self) -> bool:
         return self._enabled
@@ -50,7 +269,131 @@ class DirMonitor(_PluginBase):
         pass
 
     def get_form(self) -> Tuple[List[dict], Dict[str, Any]]:
-        pass
+        return [
+            {
+                'component': 'VForm',
+                'content': [
+                    {
+                        'component': 'VRow',
+                        'content': [
+                            {
+                                'component': 'VCol',
+                                'props': {
+                                    'cols': 12,
+                                    'md': 6
+                                },
+                                'content': [
+                                    {
+                                        'component': 'VSwitch',
+                                        'props': {
+                                            'model': 'enabled',
+                                            'label': '启用插件',
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        'component': 'VRow',
+                        'content': [
+                            {
+                                'component': 'VCol',
+                                'props': {
+                                    'cols': 12,
+                                    'md': 6
+                                },
+                                'content': [
+                                    {
+                                        'component': 'VSelect',
+                                        'props': {
+                                            'model': 'mode',
+                                            'label': '监控模式',
+                                            'items': [
+                                                {'title': '兼容模式', 'value': 'compatibility'},
+                                                {'title': '性能模式', 'value': 'fast'}
+                                            ]
+                                        }
+                                    }
+                                ]
+                            },
+                            {
+                                'component': 'VCol',
+                                'props': {
+                                    'cols': 12,
+                                    'md': 6
+                                },
+                                'content': [
+                                    {
+                                        'component': 'VSelect',
+                                        'props': {
+                                            'model': 'transfer_type',
+                                            'label': '转移方式',
+                                            'items': [
+                                                {'title': '移动', 'value': 'move'},
+                                                {'title': '复制', 'value': 'copy'},
+                                                {'title': '硬链接', 'value': 'link'},
+                                                {'title': '软链接', 'value': 'softlink'}
+                                            ]
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        'component': 'VRow',
+                        'content': [
+                            {
+                                'component': 'VCol',
+                                'props': {
+                                    'cols': 12
+                                },
+                                'content': [
+                                    {
+                                        'component': 'VTextarea',
+                                        'props': {
+                                            'model': 'monitor_dirs',
+                                            'label': '监控目录',
+                                            'rows': 5,
+                                            'placeholder': '每一行一个目录'
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        'component': 'VRow',
+                        'content': [
+                            {
+                                'component': 'VCol',
+                                'props': {
+                                    'cols': 12
+                                },
+                                'content': [
+                                    {
+                                        'component': 'VTextarea',
+                                        'props': {
+                                            'model': 'exclude_keywords',
+                                            'label': '排除关键词',
+                                            'rows': 2,
+                                            'placeholder': '每一行一个关键词'
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                ]
+            }
+        ], {
+            "enabled": False,
+            "mode": "fast",
+            "transfer_type": settings.TRANSFER_TYPE,
+            "monitor_dirs": "",
+            "exclude_keywords": ""
+        }
 
     def get_page(self) -> List[dict]:
         pass
@@ -59,4 +402,11 @@ class DirMonitor(_PluginBase):
         """
         退出插件
         """
-        pass
+        if self._observer:
+            for observer in self._observer:
+                try:
+                    observer.stop()
+                    observer.join()
+                except Exception as e:
+                    print(str(e))
+        self._observer = []

app/plugins/libraryscraper/__init__.py

@@ -54,8 +54,8 @@ class LibraryScraper(_PluginBase):
         if config:
             self._enabled = config.get("enabled")
             self._cron = config.get("cron")
-            self._scraper_paths = config.get("scraper_paths")
+            self._scraper_paths = config.get("scraper_paths") or ""
-            self._exclude_paths = config.get("exclude_paths")
+            self._exclude_paths = config.get("exclude_paths") or ""
 
         # 停止现有任务
         self.stop_service()

requirements.txt

@@ -44,3 +44,5 @@ Cython~=0.29.35
 tvdb_api~=3.1
 psutil~=5.9.4
 python_hosts~=1.0.3
+tmdbv3api~=1.7.7
+watchdog~=3.0.0