diff --git a/app/chain/__init__.py b/app/chain/__init__.py index 118ec35c..a0cfb82b 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -94,16 +94,6 @@ class ChainBase(metaclass=ABCMeta): logger.error(f"运行模块 {method} 出错：{module.__class__.__name__} - {err}\n{traceback.print_exc()}") return result - def prepare_recognize(self, title: str, - subtitle: str = None) -> Tuple[str, str]: - """ - 处理各类特别命名，以便识别 - :param title: 标题 - :param subtitle: 副标题 - :return: 处理后的标题、副标题，该方法可被多个模块同时处理 - """ - return self.run_module("prepare_recognize", title=title, subtitle=subtitle) - def recognize_media(self, meta: MetaBase = None, mtype: MediaType = None, tmdbid: int = None) -> Optional[MediaInfo]: diff --git a/app/chain/media.py b/app/chain/media.py index d85775b3..baa5aa5e 100644 --- a/app/chain/media.py +++ b/app/chain/media.py @@ -18,10 +18,6 @@ class MediaChain(ChainBase): 根据主副标题识别媒体信息 """ logger.info(f'开始识别媒体信息，标题：{title}，副标题：{subtitle} ...') - # 识别前预处理 - result: Optional[tuple] = self.prepare_recognize(title=title, subtitle=subtitle) - if result: - title, subtitle = result # 识别元数据 metainfo = MetaInfo(title, subtitle) # 识别媒体信息 diff --git a/app/chain/search.py b/app/chain/search.py index 50d97236..819b14ed 100644 --- a/app/chain/search.py +++ b/app/chain/search.py @@ -157,14 +157,8 @@ class SearchChain(ChainBase): logger.info(f'{mediainfo.title} 匹配到资源：{torrent.site_name} - {torrent.title}') _match_torrents.append(torrent) continue - # 识别前预处理 - result: Optional[tuple] = self.prepare_recognize(title=torrent.title, subtitle=torrent.description) - if result: - title, subtitle = result - else: - title, subtitle = torrent.title, torrent.description # 识别 - torrent_meta = MetaInfo(title=title, subtitle=subtitle) + torrent_meta = MetaInfo(title=torrent.title, subtitle=torrent.description) # 比对类型 if torrent_meta.type == MediaType.TV and mediainfo.type != MediaType.TV: logger.warn(f'{torrent.site_name} - {torrent.title} 类型不匹配') diff --git a/app/chain/subscribe.py b/app/chain/subscribe.py index e79a5af2..d30b7784 100644 --- a/app/chain/subscribe.py +++ b/app/chain/subscribe.py @@ -52,10 +52,6 @@ class SubscribeChain(ChainBase): 识别媒体信息并添加订阅 """ logger.info(f'开始添加订阅，标题：{title} ...') - # 识别前预处理 - result: Optional[tuple] = self.prepare_recognize(title=title) - if result: - title, _ = result # 识别元数据 metainfo = MetaInfo(title) if year: @@ -385,15 +381,8 @@ class SubscribeChain(ChainBase): continue for torrent in torrents: logger.info(f'处理资源：{torrent.title} ...') - # 识别前预处理 - result: Optional[tuple] = self.prepare_recognize(title=torrent.title, - subtitle=torrent.description) - if result: - title, subtitle = result - else: - title, subtitle = torrent.title, torrent.description # 识别 - meta = MetaInfo(title=title, subtitle=subtitle) + meta = MetaInfo(title=torrent.title, subtitle=torrent.description) # 识别媒体信息 mediainfo: MediaInfo = self.recognize_media(meta=meta) if not mediainfo: diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 8be95baf..89e10157 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -89,16 +89,10 @@ class TransferChain(ChainBase): self.progress.update(value=processed_num / total_num * 100, text=f"正在转移 {torrent.title} ...", key=ProgressKey.FileTransfer) - # 识别前预处理 - result: Optional[tuple] = self.prepare_recognize(title=torrent.title) - if result: - title, subtitle = result - else: - title, subtitle = torrent.title, None # 识别元数据 - meta: MetaBase = MetaInfo(title=title, subtitle=subtitle) + meta: MetaBase = MetaInfo(title=torrent.title) if not meta.name: - logger.error(f'未识别到元数据，标题：{title}') + logger.error(f'未识别到元数据，标题：{torrent.title}') continue if not arg_mediainfo: # 查询下载记录识别情况 @@ -110,7 +104,7 @@ class TransferChain(ChainBase): if mtype == MediaType.TV \ and ((not meta.season_list and downloadhis.seasons) or (not meta.episode_list and downloadhis.episodes)): - meta = MetaInfo(f"{title} {downloadhis.seasons} {downloadhis.episodes}") + meta = MetaInfo(f"{torrent.title} {downloadhis.seasons} {downloadhis.episodes}") # 按TMDBID识别 mediainfo = self.recognize_media(mtype=mtype, tmdbid=downloadhis.tmdbid) @@ -195,7 +189,7 @@ class TransferChain(ChainBase): ) # 转移完成 self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo) - # 刮剥 + # 刮削元数据 self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) # 刷新媒体库 self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path) diff --git a/app/db/models/transferhistory.py b/app/db/models/transferhistory.py index 7b93320c..6cde6304 100644 --- a/app/db/models/transferhistory.py +++ b/app/db/models/transferhistory.py @@ -30,7 +30,7 @@ class TransferHistory(Base): tvdbid = Column(Integer) doubanid = Column(String) # Sxx - seasons = Column(Integer) + seasons = Column(String) # Exx episodes = Column(String) # 海报 @@ -59,6 +59,10 @@ class TransferHistory(Base): def get_by_hash(db: Session, download_hash: str): return db.query(TransferHistory).filter(TransferHistory.download_hash == download_hash).first() + @staticmethod + def get_by_src(db: Session, src: str): + return db.query(TransferHistory).filter(TransferHistory.src == src).first() + @staticmethod def statistic(db: Session, days: int = 7): """ diff --git a/app/db/transferhistory_oper.py b/app/db/transferhistory_oper.py index 400694d8..b02bd154 100644 --- a/app/db/transferhistory_oper.py +++ b/app/db/transferhistory_oper.py @@ -17,6 +17,13 @@ class TransferHistoryOper(DbOper): """ return TransferHistory.search_by_title(self._db, title) + def get_by_src(self, src: str) -> Any: + """ + 按源查询转移记录 + :param src: 数据key + """ + return TransferHistory.get_by_src(self._db, src) + def add(self, **kwargs): """ 新增转移历史 @@ -36,7 +43,8 @@ class TransferHistoryOper(DbOper): """ return TransferHistory.statistic(self._db, days) - def get_by(self, mtype: str, title: str, year: int, season=None, episode=None) -> Any: + def get_by(self, mtype: str, title: str, year: int, + season: str = None, episode: str = None) -> Any: """ 按类型、标题、年份、季集查询转移记录 """ diff --git a/app/plugins/__init__.py b/app/plugins/__init__.py index e6ad9391..caeda644 100644 --- a/app/plugins/__init__.py +++ b/app/plugins/__init__.py @@ -4,6 +4,7 @@ from typing import Any, List, Dict, Tuple from app.chain import ChainBase from app.core.config import settings +from app.core.event import EventManager from app.db.models import Base from app.db.plugindata_oper import PluginDataOper from app.db.systemconfig_oper import SystemConfigOper @@ -44,6 +45,8 @@ class _PluginBase(metaclass=ABCMeta): self.systemconfig = SystemConfigOper() # 系统消息 self.systemmessage = MessageHelper() + # 事件管理器 + self.eventmanager = EventManager() @abstractmethod def init_plugin(self, config: dict = None): diff --git a/app/plugins/dirmonitor/__init__.py b/app/plugins/dirmonitor/__init__.py index a0a91861..0fd0ce0b 100644 --- a/app/plugins/dirmonitor/__init__.py +++ b/app/plugins/dirmonitor/__init__.py @@ -1,6 +1,39 @@ +import threading +import traceback +from pathlib import Path from typing import List, Tuple, Dict, Any +from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer +from watchdog.observers.polling import PollingObserver + +from app.core.config import settings +from app.core.context import MediaInfo +from app.core.metainfo import MetaInfo +from app.db.transferhistory_oper import TransferHistoryOper +from app.log import logger from app.plugins import _PluginBase +from app.schemas import MediaType, Notification, NotificationType, TransferInfo +from app.schemas.types import EventType + +lock = threading.Lock() + + +class FileMonitorHandler(FileSystemEventHandler): + """ + 目录监控响应类 + """ + + def __init__(self, monpath: str, sync: Any, **kwargs): + super(FileMonitorHandler, self).__init__(**kwargs) + self._watch_path = monpath + self.sync = sync + + def on_created(self, event): + self.sync.file_change_handler(event, "创建", event.src_path) + + def on_moved(self, event): + self.sync.file_change_handler(event, "移动", event.dest_path) class DirMonitor(_PluginBase): @@ -25,19 +58,205 @@ class DirMonitor(_PluginBase): # 可使用的用户级别 user_level = 1 + # 已处理的文件清单 + _synced_files = [] + # 私有属性 - _monitor = None + transferhis = None + _observer = [] _enabled = False + # 模式 compatibility/fast + _mode = "fast" + # 转移方式 + _transfer_type = settings.TRANSFER_TYPE + _monitor_dirs = "" + _exclude_keywords = "" def init_plugin(self, config: dict = None): + self.transferhis = TransferHistoryOper() + # 读取配置 if config: self._enabled = config.get("enabled") + self._mode = config.get("mode") + self._transfer_type = config.get("transfer_type") + self._monitor_dirs = config.get("monitor_dirs") or "" + self._exclude_keywords = config.get("exclude_keywords") or "" # 停止现有任务 self.stop_service() - # TODO 启动任务 + # 启动任务 + monitor_dirs = self._monitor_dirs.split("\n") + if not monitor_dirs: + return + for mon_path in monitor_dirs: + if not mon_path: + continue + # 检查目录是不是媒体库目录的子目录 + if Path(mon_path).is_relative_to(settings.LIBRARY_PATH): + logger.warn(f"{mon_path} 是媒体库目录的子目录，无法监控") + self.systemmessage.put(f"{mon_path} 是媒体库目录的子目录，无法监控") + continue + + try: + if self._mode == "compatibility": + # 兼容模式，目录同步性能降低且NAS不能休眠，但可以兼容挂载的远程共享目录如SMB + observer = PollingObserver(timeout=10) + else: + # 内部处理系统操作类型选择最优解 + observer = Observer(timeout=10) + self._observer.append(observer) + observer.schedule(FileMonitorHandler(mon_path, self), path=mon_path, recursive=True) + observer.daemon = True + observer.start() + logger.info(f"{mon_path} 的目录监控服务启动") + except Exception as e: + err_msg = str(e) + if "inotify" in err_msg and "reached" in err_msg: + logger.warn(f"目录监控服务启动出现异常：{err_msg}，请在宿主机上（不是docker容器内）执行以下命令并重启：" + + """ + echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf + echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf + sudo sysctl -p + """) + else: + logger.error(f"{mon_path} 启动目录监控失败：{err_msg}") + self.systemmessage.put(f"{mon_path} 启动目录监控失败：{err_msg}") + + def file_change_handler(self, event, text: str, event_path: str): + """ + 处理文件变化 + :param event: 事件 + :param text: 事件描述 + :param event_path: 事件文件路径 + """ + if not event.is_directory: + # 文件发生变化 + file_path = Path(event_path) + try: + if not file_path.exists(): + return + + logger.debug("文件%s：%s" % (text, event_path)) + + # 全程加锁 + with lock: + if event_path not in self._synced_files: + self._synced_files.append(event_path) + else: + logger.debug("文件已处理过：%s" % event_path) + return + + # 命中过滤关键字不处理 + if self._exclude_keywords: + for keyword in self._exclude_keywords.split("\n"): + if keyword and keyword in event_path: + logger.debug(f"{event_path} 命中过滤关键字 {keyword}") + return + + # 回收站及隐藏的文件不处理 + if event_path.find('/@Recycle/') != -1 \ + or event_path.find('/#recycle/') != -1 \ + or event_path.find('/.') != -1 \ + or event_path.find('/@eaDir') != -1: + logger.debug(f"{event_path} 是回收站或隐藏的文件") + return + + # 不是媒体文件不处理 + if file_path.suffix not in settings.RMT_MEDIAEXT: + logger.debug(f"{event_path} 不是媒体文件") + return + + # 查询历史记录，已转移的不处理 + if self.transferhis.get_by_src(event_path): + logger.info(f"{event_path} 已整理过") + return + + # 文件元数据 + file_meta = MetaInfo(title=file_path.name) + # 上级目录元数据 + dir_meta = MetaInfo(title=file_path.parent.name) + # 整合元数据 + if not file_meta.cn_name and dir_meta.cn_name: + file_meta.cn_name = dir_meta.cn_name + if not file_meta.en_name and dir_meta.en_name: + file_meta.en_name = dir_meta.en_name + if file_meta.type != MediaType.TV and dir_meta.type == MediaType.TV: + file_meta.type = MediaType.TV + if not file_meta.year and dir_meta.year: + file_meta.year = dir_meta.year + if not file_meta.begin_season and dir_meta.begin_season: + file_meta.begin_season = dir_meta.begin_season + if not file_meta.episode_list and dir_meta.episode_list: + file_meta.begin_episode = dir_meta.begin_episode + file_meta.end_episode = dir_meta.end_episode + + if not file_meta.name: + logger.warn(f"{file_path.name} 无法识别有效信息") + return + + # 识别媒体信息 + mediainfo: MediaInfo = self.chain.recognize_media(meta=file_meta) + if not mediainfo: + logger.warn(f'未识别到媒体信息，标题：{file_meta.name}') + self.chain.post_message(Notification( + mtype=NotificationType.Manual, + title=f"{file_path.name} 未识别到媒体信息，无法入库！" + )) + return + logger.info(f"{file_path.name} 识别为：{mediainfo.type.value} {mediainfo.title_year}") + + # 更新媒体图片 + self.chain.obtain_images(mediainfo=mediainfo) + + # 转移 + transferinfo: TransferInfo = self.chain.transfer(mediainfo=mediainfo, path=file_path) + + if not transferinfo or not transferinfo.target_path: + # 转移失败 + logger.warn(f"{file_path.name} 入库失败") + self.chain.post_message(Notification( + title=f"{mediainfo.title_year}{file_meta.season_episode} 入库失败！", + text=f"原因：{transferinfo.message if transferinfo else '未知'}", + image=mediainfo.get_message_image() + )) + return + + # 新增转移成功历史记录 + self.transferhis.add( + src=event_path, + dest=str(transferinfo.target_path) if transferinfo else None, + mode=settings.TRANSFER_TYPE, + type=mediainfo.type.value, + category=mediainfo.category, + title=mediainfo.title, + year=mediainfo.year, + tmdbid=mediainfo.tmdb_id, + imdbid=mediainfo.imdb_id, + tvdbid=mediainfo.tvdb_id, + doubanid=mediainfo.douban_id, + seasons=file_meta.season, + episodes=file_meta.episode, + image=mediainfo.get_poster_image(), + status=1 + ) + + # 刮削元数据 + self.chain.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) + # 刷新媒体库 + self.chain.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path) + # 发送通知 + self.chain.__send_transfer_message(meta=file_meta, mediainfo=mediainfo, transferinfo=transferinfo) + # 广播事件 + self.eventmanager.send_event(EventType.TransferComplete, { + 'meta': file_meta, + 'mediainfo': mediainfo, + 'transferinfo': transferinfo + }) + + except Exception as e: + logger.error("目录监控发生错误：%s - %s" % (str(e), traceback.format_exc())) def get_state(self) -> bool: return self._enabled @@ -50,7 +269,131 @@ class DirMonitor(_PluginBase): pass def get_form(self) -> Tuple[List[dict], Dict[str, Any]]: - pass + return [ + { + 'component': 'VForm', + 'content': [ + { + 'component': 'VRow', + 'content': [ + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 6 + }, + 'content': [ + { + 'component': 'VSwitch', + 'props': { + 'model': 'enabled', + 'label': '启用插件', + } + } + ] + } + ] + }, + { + 'component': 'VRow', + 'content': [ + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 6 + }, + 'content': [ + { + 'component': 'VSelect', + 'props': { + 'model': 'mode', + 'label': '监控模式', + 'items': [ + {'title': '兼容模式', 'value': 'compatibility'}, + {'title': '性能模式', 'value': 'fast'} + ] + } + } + ] + }, + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 6 + }, + 'content': [ + { + 'component': 'VSelect', + 'props': { + 'model': 'transfer_type', + 'label': '转移方式', + 'items': [ + {'title': '移动', 'value': 'move'}, + {'title': '复制', 'value': 'copy'}, + {'title': '硬链接', 'value': 'link'}, + {'title': '软链接', 'value': 'softlink'} + ] + } + } + ] + } + ] + }, + { + 'component': 'VRow', + 'content': [ + { + 'component': 'VCol', + 'props': { + 'cols': 12 + }, + 'content': [ + { + 'component': 'VTextarea', + 'props': { + 'model': 'monitor_dirs', + 'label': '监控目录', + 'rows': 5, + 'placeholder': '每一行一个目录' + } + } + ] + } + ] + }, + { + 'component': 'VRow', + 'content': [ + { + 'component': 'VCol', + 'props': { + 'cols': 12 + }, + 'content': [ + { + 'component': 'VTextarea', + 'props': { + 'model': 'exclude_keywords', + 'label': '排除关键词', + 'rows': 2, + 'placeholder': '每一行一个关键词' + } + } + ] + } + ] + } + ] + } + ], { + "enabled": False, + "mode": "fast", + "transfer_type": settings.TRANSFER_TYPE, + "monitor_dirs": "", + "exclude_keywords": "" + } def get_page(self) -> List[dict]: pass @@ -59,4 +402,11 @@ class DirMonitor(_PluginBase): """ 退出插件 """ - pass + if self._observer: + for observer in self._observer: + try: + observer.stop() + observer.join() + except Exception as e: + print(str(e)) + self._observer = [] diff --git a/app/plugins/libraryscraper/__init__.py b/app/plugins/libraryscraper/__init__.py index 7c960bfc..ef3ae102 100644 --- a/app/plugins/libraryscraper/__init__.py +++ b/app/plugins/libraryscraper/__init__.py @@ -54,8 +54,8 @@ class LibraryScraper(_PluginBase): if config: self._enabled = config.get("enabled") self._cron = config.get("cron") - self._scraper_paths = config.get("scraper_paths") - self._exclude_paths = config.get("exclude_paths") + self._scraper_paths = config.get("scraper_paths") or "" + self._exclude_paths = config.get("exclude_paths") or "" # 停止现有任务 self.stop_service() diff --git a/requirements.txt b/requirements.txt index ab2774be..4e8941c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,4 +43,6 @@ PyVirtualDisplay~=3.0 Cython~=0.29.35 tvdb_api~=3.1 psutil~=5.9.4 -python_hosts~=1.0.3 \ No newline at end of file +python_hosts~=1.0.3 +tmdbv3api~=1.7.7 +watchdog~=3.0.0 \ No newline at end of file