From 66a1f25465b10f937ac5b58e4dd4004d80b4e009 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Wed, 23 Aug 2023 08:47:03 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=E4=B8=8B=E8=BD=BD=E5=99=A8=E7=9B=91?= =?UTF-8?q?=E6=8E=A7=E6=94=AF=E6=8C=81=E8=BD=AC=E7=A7=BB=E5=90=88=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/chain/transfer.py | 162 +++++++++++++++---------- app/modules/douban/__init__.py | 2 +- app/modules/filetransfer/__init__.py | 6 +- app/modules/qbittorrent/__init__.py | 2 +- app/modules/subtitle/__init__.py | 2 +- app/modules/themoviedb/__init__.py | 2 +- app/modules/transmission/__init__.py | 2 +- app/plugins/dirmonitor/__init__.py | 2 +- app/plugins/libraryscraper/__init__.py | 2 +- app/utils/system.py | 48 +++++++- 10 files changed, 155 insertions(+), 75 deletions(-) diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 6fc7dfb6..5c3f99e3 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -60,6 +60,7 @@ class TransferChain(ChainBase): self.progress.update(value=0, text=f"开始转移下载任务文件，共 {total_num} 个任务 ...", key=ProgressKey.FileTransfer) + for torrent in torrents: # 更新进度 self.progress.update(value=processed_num / total_num * 100, @@ -105,59 +106,67 @@ class TransferChain(ChainBase): continue logger.info(f"{torrent.title} 识别为：{mediainfo.type.value} {mediainfo.title_year}") + # 更新媒体图片 self.obtain_images(mediainfo=mediainfo) - # 转移 - transferinfo: TransferInfo = self.transfer(mediainfo=mediainfo, - path=torrent.path, - transfer_type=settings.TRANSFER_TYPE) - if not transferinfo: - logger.error("文件转移模块运行失败") + # 获取待转移路径清单 + trans_paths = self.__get_trans_paths(torrent.path) + if not trans_paths: + logger.warn(f"{torrent.title} 对应目录没有找到媒体文件") continue - if not transferinfo.target_path: - # 转移失败 - logger.warn(f"{torrent.title} 入库失败：{transferinfo.message}") - # 新增转移失败历史记录 - self.__insert_fail_history( - src_path=torrent.path, + + # 转移所有文件 + for trans_path in trans_paths: + transferinfo: TransferInfo = self.transfer(mediainfo=mediainfo, + path=trans_path, + transfer_type=settings.TRANSFER_TYPE) + if not transferinfo: + logger.error("文件转移模块运行失败") + continue + if not transferinfo.target_path: + # 转移失败 + logger.warn(f"{torrent.title} 入库失败：{transferinfo.message}") + # 新增转移失败历史记录 + self.__insert_fail_history( + src_path=trans_path, + download_hash=torrent.hash, + meta=meta, + mediainfo=mediainfo, + transferinfo=transferinfo + ) + # 发送消息 + self.post_message(Notification( + title=f"{mediainfo.title_year} {meta.season_episode} 入库失败！", + text=f"原因：{transferinfo.message or '未知'}", + image=mediainfo.get_message_image() + )) + continue + + # 新增转移成功历史记录 + self.__insert_sucess_history( + src_path=trans_path, download_hash=torrent.hash, meta=meta, mediainfo=mediainfo, transferinfo=transferinfo ) - # 发送消息 - self.post_message(Notification( - title=f"{mediainfo.title_year} {meta.season_episode} 入库失败！", - text=f"原因：{transferinfo.message or '未知'}", - image=mediainfo.get_message_image() - )) - # 设置种子状态，避免一直报错 - self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo) - continue + # 刮削元数据 + self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) + # 刷新媒体库 + self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path) + # 发送通知 + self.send_transfer_message(meta=meta, mediainfo=mediainfo, transferinfo=transferinfo) + # 广播事件 + self.eventmanager.send_event(EventType.TransferComplete, { + 'meta': meta, + 'mediainfo': mediainfo, + 'transferinfo': transferinfo + }) - # 新增转移成功历史记录 - self.__insert_sucess_history( - src_path=torrent.path, - download_hash=torrent.hash, - meta=meta, - mediainfo=mediainfo, - transferinfo=transferinfo - ) # 转移完成 self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo) - # 刮削元数据 - self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) - # 刷新媒体库 - self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path) - # 发送通知 - self.send_transfer_message(meta=meta, mediainfo=mediainfo, transferinfo=transferinfo) - # 广播事件 - self.eventmanager.send_event(EventType.TransferComplete, { - 'meta': meta, - 'mediainfo': mediainfo, - 'transferinfo': transferinfo - }) + # 计数 processed_num += 1 # 更新进度 @@ -169,6 +178,38 @@ class TransferChain(ChainBase): logger.info("下载器文件转移执行完成") return True + @staticmethod + def __get_trans_paths(directory: Path): + """ + 获取转移目录列表 + """ + + if not directory.exists(): + logger.warn(f"目录不存在：{directory}") + return [] + + if directory.is_file(): + return [directory] + + # 需要转移的路径列表 + trans_paths = [] + + # 先检查当前目录的下级目录，以支持合集的情况 + for sub_dir in SystemUtils.list_sub_directory(directory): + # 没有媒体文件的目录跳过 + if not SystemUtils.list_files(sub_dir, extensions=settings.RMT_MEDIAEXT): + continue + trans_paths.append(sub_dir) + + if not trans_paths: + # 没有有效子目录，直接转移当前目录 + trans_paths.append(directory) + else: + # 有子目录时，把当前目录的文件添加到转移任务中 + trans_paths.extend( + SystemUtils.list_sub_files(directory, extensions=settings.RMT_MEDIAEXT) + ) + def remote_transfer(self, arg_str: str, channel: MessageChannel, userid: Union[str, int] = None): """ 远程重新转移，参数 历史记录ID TMDBID|类型 @@ -210,7 +251,7 @@ class TransferChain(ChainBase): def re_transfer(self, logid: int, mtype: MediaType, tmdbid: int) -> Tuple[bool, str]: """ - 根据历史记录，重新识别转移 + 根据历史记录，重新识别转移，只处理对应的src目录 :param logid: 历史记录ID :param mtype: 媒体类型 :param tmdbid: TMDB ID @@ -220,18 +261,10 @@ class TransferChain(ChainBase): if not history: logger.error(f"历史记录不存在，ID：{logid}") return False, "历史记录不存在" - if history.download_hash: - # 有下载记录，按下载记录重新转移 - torrents: Optional[List[TransferTorrent]] = self.list_torrents(hashs=history.download_hash) - if not torrents: - return False, f"没有获取到种子，hash：{history.download_hash}" - # 源目录 - src_path = Path(torrents[0].path) - else: - # 没有下载记录，按源目录路径重新转移 - src_path = Path(history.src) - if not src_path.exists(): - return False, f"源目录不存在：{src_path}" + # 没有下载记录，按源目录路径重新转移 + src_path = Path(history.src) + if not src_path.exists(): + return False, f"源目录不存在：{src_path}" # 识别元数据 meta = MetaInfo(title=src_path.stem) if not meta.name: @@ -252,9 +285,13 @@ class TransferChain(ChainBase): if not transferinfo: logger.error("文件转移模块运行失败") return False, "文件转移模块运行失败" + + # 删除旧历史记录 + self.transferhis.delete(logid) + if not transferinfo.target_path: # 转移失败 - logger.warn(f"{src_path.name} 入库失败：{transferinfo.message}") + logger.warn(f"{src_path} 入库失败：{transferinfo.message}") # 新增转移失败历史记录 self.__insert_fail_history( src_path=src_path, @@ -273,8 +310,6 @@ class TransferChain(ChainBase): mediainfo=mediainfo, transferinfo=transferinfo ) - # 删除旧历史记录 - self.transferhis.delete(logid) # 刮削元数据 self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) # 刷新媒体库 @@ -287,6 +322,7 @@ class TransferChain(ChainBase): 'mediainfo': mediainfo, 'transferinfo': transferinfo }) + return True, "" def __insert_sucess_history(self, src_path: Path, download_hash: str, meta: MetaBase, @@ -294,7 +330,7 @@ class TransferChain(ChainBase): """ 新增转移成功历史记录 """ - self.transferhis.add( + self.transferhis.add_force( src=str(src_path), dest=str(transferinfo.target_path), mode=settings.TRANSFER_TYPE, @@ -317,10 +353,10 @@ class TransferChain(ChainBase): def __insert_fail_history(self, src_path: Path, download_hash: str, meta: MetaBase, transferinfo: TransferInfo = None, mediainfo: MediaInfo = None): """ - 新增转移失败历史记录 + 新增转移失败历史记录，不能按download_hash判重 """ if mediainfo and transferinfo: - his = self.transferhis.add( + his = self.transferhis.add_force( src=str(src_path), dest=str(transferinfo.target_path), mode=settings.TRANSFER_TYPE, @@ -341,7 +377,7 @@ class TransferChain(ChainBase): files=json.dumps(transferinfo.file_list) ) else: - his = self.transferhis.add( + his = self.transferhis.add_force( src=str(src_path), mode=settings.TRANSFER_TYPE, seasons=meta.season, @@ -389,8 +425,8 @@ class TransferChain(ChainBase): logger.warn(f"文件 {path} 已删除") # 判断目录是否为空, 为空则删除 if str(path.parent.parent) != str(path.root): - # 父父目录非根目录，才删除父目录 - files = SystemUtils.list_files_with_extensions(path.parent, settings.RMT_MEDIAEXT) + # 父目录非根目录，才删除父目录 + files = SystemUtils.list_files(path.parent, settings.RMT_MEDIAEXT) if not files: shutil.rmtree(path.parent) logger.warn(f"目录 {path.parent} 已删除") diff --git a/app/modules/douban/__init__.py b/app/modules/douban/__init__.py index 7d330309..6b100914 100644 --- a/app/modules/douban/__init__.py +++ b/app/modules/douban/__init__.py @@ -167,7 +167,7 @@ class DoubanModule(_ModuleBase): if settings.SCRAP_SOURCE != "douban": return None # 目录下的所有文件 - for file in SystemUtils.list_files_with_extensions(path, settings.RMT_MEDIAEXT): + for file in SystemUtils.list_files(path, settings.RMT_MEDIAEXT): if not file: continue logger.info(f"开始刮削媒体库文件：{file} ...") diff --git a/app/modules/filetransfer/__init__.py b/app/modules/filetransfer/__init__.py index e416d4c0..fa3a1da3 100644 --- a/app/modules/filetransfer/__init__.py +++ b/app/modules/filetransfer/__init__.py @@ -121,7 +121,7 @@ class FileTransferModule(_ModuleBase): # 比对文件名并转移字幕 org_dir: Path = org_path.parent - file_list: List[Path] = SystemUtils.list_files_with_extensions(org_dir, settings.RMT_SUBEXT) + file_list: List[Path] = SystemUtils.list_files(org_dir, settings.RMT_SUBEXT) if len(file_list) == 0: logger.debug(f"{org_dir} 目录下没有找到字幕文件...") else: @@ -207,7 +207,7 @@ class FileTransferModule(_ModuleBase): """ dir_name = org_path.parent file_name = org_path.name - file_list: List[Path] = SystemUtils.list_files_with_extensions(dir_name, ['.mka']) + file_list: List[Path] = SystemUtils.list_files(dir_name, ['.mka']) pending_file_list: List[Path] = [file for file in file_list if org_path.stem == file.stem] if len(pending_file_list) == 0: logger.debug(f"{dir_name} 目录下没有找到匹配的音轨文件") @@ -409,7 +409,7 @@ class FileTransferModule(_ModuleBase): file_list_new=[]) else: # 获取文件清单 - transfer_files: List[Path] = SystemUtils.list_files_with_extensions(in_path, settings.RMT_MEDIAEXT) + transfer_files: List[Path] = SystemUtils.list_files(in_path, settings.RMT_MEDIAEXT) if len(transfer_files) == 0: return TransferInfo(message=f"{in_path} 目录下没有找到可转移的文件") if not in_meta: diff --git a/app/modules/qbittorrent/__init__.py b/app/modules/qbittorrent/__init__.py index fbca0e6f..12785915 100644 --- a/app/modules/qbittorrent/__init__.py +++ b/app/modules/qbittorrent/__init__.py @@ -168,7 +168,7 @@ class QbittorrentModule(_ModuleBase): logger.info(f"移动模式删除种子成功：{hashs} ") # 删除残留文件 if transinfo.path and transinfo.path.exists(): - files = SystemUtils.list_files_with_extensions(transinfo.path, settings.RMT_MEDIAEXT) + files = SystemUtils.list_files(transinfo.path, settings.RMT_MEDIAEXT) if not files: logger.warn(f"删除残留文件夹：{transinfo.path}") shutil.rmtree(transinfo.path, ignore_errors=True) diff --git a/app/modules/subtitle/__init__.py b/app/modules/subtitle/__init__.py index 4b14799c..18afbbf3 100644 --- a/app/modules/subtitle/__init__.py +++ b/app/modules/subtitle/__init__.py @@ -108,7 +108,7 @@ class SubtitleModule(_ModuleBase): # 解压文件 shutil.unpack_archive(zip_file, zip_path, format='zip') # 遍历转移文件 - for sub_file in SystemUtils.list_files_with_extensions(zip_path, settings.RMT_SUBEXT): + for sub_file in SystemUtils.list_files(zip_path, settings.RMT_SUBEXT): target_sub_file = download_dir / sub_file.name if target_sub_file.exists(): logger.info(f"字幕文件已存在：{target_sub_file}") diff --git a/app/modules/themoviedb/__init__.py b/app/modules/themoviedb/__init__.py index a7170700..7c7bac7c 100644 --- a/app/modules/themoviedb/__init__.py +++ b/app/modules/themoviedb/__init__.py @@ -190,7 +190,7 @@ class TheMovieDbModule(_ModuleBase): if settings.SCRAP_SOURCE != "themoviedb": return None # 目录下的所有文件 - for file in SystemUtils.list_files_with_extensions(path, settings.RMT_MEDIAEXT): + for file in SystemUtils.list_files(path, settings.RMT_MEDIAEXT): if not file: continue logger.info(f"开始刮削媒体库文件：{file} ...") diff --git a/app/modules/transmission/__init__.py b/app/modules/transmission/__init__.py index 91bf062e..6ab78133 100644 --- a/app/modules/transmission/__init__.py +++ b/app/modules/transmission/__init__.py @@ -152,7 +152,7 @@ class TransmissionModule(_ModuleBase): logger.info(f"移动模式删除种子成功：{hashs} ") # 删除残留文件 if transinfo.path and transinfo.path.exists(): - files = SystemUtils.list_files_with_extensions(transinfo.path, settings.RMT_MEDIAEXT) + files = SystemUtils.list_files(transinfo.path, settings.RMT_MEDIAEXT) if not files: logger.warn(f"删除残留文件夹：{transinfo.path}") shutil.rmtree(transinfo.path, ignore_errors=True) diff --git a/app/plugins/dirmonitor/__init__.py b/app/plugins/dirmonitor/__init__.py index 1bb81f45..b3bfa96e 100644 --- a/app/plugins/dirmonitor/__init__.py +++ b/app/plugins/dirmonitor/__init__.py @@ -309,7 +309,7 @@ class DirMonitor(_PluginBase): if len(str(file_dir)) <= len(str(Path(mon_path))): # 重要，删除到监控目录为止 break - files = SystemUtils.list_files_with_extensions(file_dir, settings.RMT_MEDIAEXT) + files = SystemUtils.list_files(file_dir, settings.RMT_MEDIAEXT) if not files: logger.warn(f"移动模式，删除空目录：{file_dir}") shutil.rmtree(file_dir, ignore_errors=True) diff --git a/app/plugins/libraryscraper/__init__.py b/app/plugins/libraryscraper/__init__.py index 24b76705..b55fa6ee 100644 --- a/app/plugins/libraryscraper/__init__.py +++ b/app/plugins/libraryscraper/__init__.py @@ -258,7 +258,7 @@ class LibraryScraper(_PluginBase): """ exclude_paths = self._exclude_paths.split("\n") # 查找目录下所有的文件 - files = SystemUtils.list_files_with_extensions(path, settings.RMT_MEDIAEXT) + files = SystemUtils.list_files(path, settings.RMT_MEDIAEXT) for file in files: if self._event.is_set(): logger.info(f"媒体库刮削服务停止") diff --git a/app/utils/system.py b/app/utils/system.py index 5c548db5..9956bacd 100644 --- a/app/utils/system.py +++ b/app/utils/system.py @@ -91,10 +91,13 @@ class SystemUtils: return -1, str(err) @staticmethod - def list_files_with_extensions(directory: Path, extensions: list) -> List[Path]: + def list_files(directory: Path, extensions: list) -> List[Path]: """ - 获取目录下所有指定扩展名的文件 + 获取目录下所有指定扩展名的文件（包括子目录） """ + if not directory.exists(): + return [] + if directory.is_file(): return [directory] @@ -108,6 +111,47 @@ class SystemUtils: return files + @staticmethod + def list_sub_files(directory: Path, extensions: list) -> List[Path]: + """ + 列出当前目录下的所有指定扩展名的文件(不包括子目录) + """ + if not directory.exists(): + return [] + + if directory.is_file(): + return [directory] + + files = [] + pattern = r".*(" + "|".join(extensions) + ")$" + + # 遍历目录 + for path in directory.iterdir(): + if path.is_file() and re.match(pattern, path.name, re.IGNORECASE): + files.append(path) + + return files + + @staticmethod + def list_sub_directory(directory: Path) -> List[Path]: + """ + 列出当前目录下的所有子目录（不递归） + """ + if not directory.exists(): + return [] + + if directory.is_file(): + return [] + + dirs = [] + + # 遍历目录 + for path in directory.iterdir(): + if path.is_dir(): + dirs.append(path) + + return dirs + @staticmethod def get_directory_size(path: Path) -> float: """