From 9c0c7e849781f78ce78c134ccbae77290c1d7110 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sun, 13 Aug 2023 08:42:36 +0800 Subject: [PATCH] fix #96 --- app/chain/__init__.py | 5 +- app/chain/download.py | 11 - app/chain/transfer.py | 269 +++++++++++++------------ app/modules/qbittorrent/__init__.py | 5 +- app/modules/qbittorrent/qbittorrent.py | 5 +- app/modules/transmission/__init__.py | 3 +- app/plugins/mediasyncdel/__init__.py | 4 +- 7 files changed, 149 insertions(+), 153 deletions(-) diff --git a/app/chain/__init__.py b/app/chain/__init__.py index c934fe75..9e14230a 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -2,11 +2,12 @@ import gc import pickle import traceback from abc import ABCMeta -from msilib.schema import File from pathlib import Path from typing import Optional, Any, Tuple, List, Set, Union, Dict +from qbittorrentapi import TorrentFilesList from ruamel.yaml import CommentedMap +from transmission_rpc import File from app.core.config import settings from app.core.context import Context @@ -308,7 +309,7 @@ class ChainBase(metaclass=ABCMeta): """ return self.run_module("stop_torrents", hashs=hashs) - def torrent_files(self, tid: str) -> Optional[List[File]]: + def torrent_files(self, tid: str) -> Optional[Union[TorrentFilesList, List[File]]]: """ 根据种子文件，选择并添加下载任务 :param tid: 种子Hash diff --git a/app/chain/download.py b/app/chain/download.py index 4a007af0..417998c1 100644 --- a/app/chain/download.py +++ b/app/chain/download.py @@ -1,5 +1,4 @@ import re -from msilib.schema import File from pathlib import Path from typing import List, Optional, Tuple, Set, Dict, Union @@ -624,13 +623,3 @@ class DownloadChain(ChainBase): 删除下载任务 """ return self.remove_torrents(hashs=[hash_str]) - - def get_files(self, tid: str) -> Optional[List[File]]: - """ - 获取种子文件清单 - """ - try: - return self.torrent_files(tid=tid) - except Exception as err: - logger.error(f"获取种子文件列表出错：{err}") - return None diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 83e0a203..e03c7788 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -1,6 +1,7 @@ import json import re import shutil +import threading from pathlib import Path from typing import List, Optional, Union @@ -19,6 +20,8 @@ from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey, from app.utils.string import StringUtils from app.utils.system import SystemUtils +lock = threading.Lock() + class TransferChain(ChainBase): """ @@ -52,105 +55,129 @@ class TransferChain(ChainBase): else: return None, None - if arg_str: - logger.info(f"开始转移下载器文件，参数：{arg_str}") - # 解析中种子hash，TMDB ID - torrent_hash, tmdbid = extract_hash_and_number(arg_str) - if not hash or not tmdbid: - logger.error(f"参数错误，参数：{arg_str}") - return False - # 获取种子 - torrents: Optional[List[TransferTorrent]] = self.list_torrents(hashs=torrent_hash) - if not torrents: - logger.error(f"没有获取到种子，参数：{arg_str}") - return False - # 查询媒体信息 - arg_mediainfo = self.recognize_media(tmdbid=tmdbid) - else: - arg_mediainfo = None - logger.info("开始执行下载器文件转移 ...") - # 从下载器获取种子列表 - torrents: Optional[List[TransferTorrent]] = self.list_torrents(status=TorrentStatus.TRANSFER) - if not torrents: - logger.info("没有获取到已完成的下载任务") - return False + # 全局锁，避免重复处理 + with lock: + if arg_str: + logger.info(f"开始转移下载器文件，参数：{arg_str}") + # 解析中种子hash，TMDB ID + torrent_hash, tmdbid = extract_hash_and_number(arg_str) + if not hash or not tmdbid: + logger.error(f"参数错误，参数：{arg_str}") + return False + # 获取种子 + torrents: Optional[List[TransferTorrent]] = self.list_torrents(hashs=torrent_hash) + if not torrents: + logger.error(f"没有获取到种子，参数：{arg_str}") + return False + # 查询媒体信息 + arg_mediainfo = self.recognize_media(tmdbid=tmdbid) + else: + arg_mediainfo = None + logger.info("开始执行下载器文件转移 ...") + # 从下载器获取种子列表 + torrents: Optional[List[TransferTorrent]] = self.list_torrents(status=TorrentStatus.TRANSFER) + if not torrents: + logger.info("没有获取到已完成的下载任务") + return False - logger.info(f"获取到 {len(torrents)} 个已完成的下载任务") - # 开始进度 - self.progress.start(ProgressKey.FileTransfer) - # 总数 - total_num = len(torrents) - # 已处理数量 - processed_num = 0 - self.progress.update(value=0, - text=f"开始转移下载任务文件，共 {total_num} 个任务 ...", - key=ProgressKey.FileTransfer) - for torrent in torrents: - # 更新进度 - self.progress.update(value=processed_num / total_num * 100, - text=f"正在转移 {torrent.title} ...", + logger.info(f"获取到 {len(torrents)} 个已完成的下载任务") + # 开始进度 + self.progress.start(ProgressKey.FileTransfer) + # 总数 + total_num = len(torrents) + # 已处理数量 + processed_num = 0 + self.progress.update(value=0, + text=f"开始转移下载任务文件，共 {total_num} 个任务 ...", key=ProgressKey.FileTransfer) - # 识别元数据 - meta: MetaBase = MetaInfo(title=torrent.title) - if not meta.name: - logger.error(f'未识别到元数据，标题：{torrent.title}') - continue - if not arg_mediainfo: - # 查询下载记录识别情况 - downloadhis: DownloadHistory = self.downloadhis.get_by_hash(torrent.hash) - if downloadhis: - # 类型 - mtype = MediaType(downloadhis.type) - # 补充剧集信息 - if mtype == MediaType.TV \ - and ((not meta.season_list and downloadhis.seasons) - or (not meta.episode_list and downloadhis.episodes)): - meta = MetaInfo(f"{torrent.title} {downloadhis.seasons} {downloadhis.episodes}") - # 按TMDBID识别 - mediainfo = self.recognize_media(mtype=mtype, - tmdbid=downloadhis.tmdbid) + for torrent in torrents: + # 更新进度 + self.progress.update(value=processed_num / total_num * 100, + text=f"正在转移 {torrent.title} ...", + key=ProgressKey.FileTransfer) + # 识别元数据 + meta: MetaBase = MetaInfo(title=torrent.title) + if not meta.name: + logger.error(f'未识别到元数据，标题：{torrent.title}') + continue + if not arg_mediainfo: + # 查询下载记录识别情况 + downloadhis: DownloadHistory = self.downloadhis.get_by_hash(torrent.hash) + if downloadhis: + # 类型 + mtype = MediaType(downloadhis.type) + # 补充剧集信息 + if mtype == MediaType.TV \ + and ((not meta.season_list and downloadhis.seasons) + or (not meta.episode_list and downloadhis.episodes)): + meta = MetaInfo(f"{torrent.title} {downloadhis.seasons} {downloadhis.episodes}") + # 按TMDBID识别 + mediainfo = self.recognize_media(mtype=mtype, + tmdbid=downloadhis.tmdbid) + else: + # 使用标题识别媒体信息 + mediainfo: MediaInfo = self.recognize_media(meta=meta) + if not mediainfo: + logger.warn(f'未识别到媒体信息，标题：{torrent.title}') + self.post_message(Notification( + channel=channel, + mtype=NotificationType.Manual, + title=f"{torrent.title} 未识别到媒体信息，无法入库！\n" + f"回复：```\n/transfer {torrent.hash} [tmdbid]\n``` 手动识别转移。", + userid=userid)) + # 新增转移失败历史记录 + self.transferhis.add( + src=str(torrent.path), + mode=settings.TRANSFER_TYPE, + seasons=meta.season, + episodes=meta.episode, + download_hash=torrent.hash, + status=0, + errmsg="未识别到媒体信息" + ) + continue else: - # 使用标题识别媒体信息 - mediainfo: MediaInfo = self.recognize_media(meta=meta) - if not mediainfo: - logger.warn(f'未识别到媒体信息，标题：{torrent.title}') + mediainfo = arg_mediainfo + logger.info(f"{torrent.title} 识别为：{mediainfo.type.value} {mediainfo.title_year}") + # 更新媒体图片 + self.obtain_images(mediainfo=mediainfo) + # 转移 + transferinfo: TransferInfo = self.transfer(mediainfo=mediainfo, + path=torrent.path, + transfer_type=settings.TRANSFER_TYPE) + if not transferinfo or not transferinfo.target_path: + # 转移失败 + logger.warn(f"{torrent.title} 入库失败") self.post_message(Notification( channel=channel, - mtype=NotificationType.Manual, - title=f"{torrent.title} 未识别到媒体信息，无法入库！\n" - f"回复：```\n/transfer {torrent.hash} [tmdbid]\n``` 手动识别转移。", - userid=userid)) + title=f"{mediainfo.title_year}{meta.season_episode} 入库失败！", + text=f"原因：{transferinfo.message if transferinfo else '未知'}", + image=mediainfo.get_message_image(), + userid=userid + )) # 新增转移失败历史记录 self.transferhis.add( src=str(torrent.path), + dest=str(transferinfo.target_path) if transferinfo else None, mode=settings.TRANSFER_TYPE, + type=mediainfo.type.value, + category=mediainfo.category, + title=mediainfo.title, + year=mediainfo.year, + tmdbid=mediainfo.tmdb_id, + imdbid=mediainfo.imdb_id, + tvdbid=mediainfo.tvdb_id, + doubanid=mediainfo.douban_id, seasons=meta.season, episodes=meta.episode, + image=mediainfo.get_poster_image(), download_hash=torrent.hash, status=0, - errmsg="未识别到媒体信息" + errmsg=transferinfo.message if transferinfo else '未知错误', + files=json.dumps(transferinfo.file_list) if transferinfo else None ) continue - else: - mediainfo = arg_mediainfo - logger.info(f"{torrent.title} 识别为：{mediainfo.type.value} {mediainfo.title_year}") - # 更新媒体图片 - self.obtain_images(mediainfo=mediainfo) - # 转移 - transferinfo: TransferInfo = self.transfer(mediainfo=mediainfo, - path=torrent.path, - transfer_type=settings.TRANSFER_TYPE) - if not transferinfo or not transferinfo.target_path: - # 转移失败 - logger.warn(f"{torrent.title} 入库失败") - self.post_message(Notification( - channel=channel, - title=f"{mediainfo.title_year}{meta.season_episode} 入库失败！", - text=f"原因：{transferinfo.message if transferinfo else '未知'}", - image=mediainfo.get_message_image(), - userid=userid - )) - # 新增转移失败历史记录 + # 新增转移成功历史记录 self.transferhis.add( src=str(torrent.path), dest=str(transferinfo.target_path) if transferinfo else None, @@ -167,55 +194,33 @@ class TransferChain(ChainBase): episodes=meta.episode, image=mediainfo.get_poster_image(), download_hash=torrent.hash, - status=0, - errmsg=transferinfo.message if transferinfo else '未知错误', - files=json.dumps(transferinfo.file_list) if transferinfo else None + status=1, + files=json.dumps(transferinfo.file_list) ) - continue - # 新增转移成功历史记录 - self.transferhis.add( - src=str(torrent.path), - dest=str(transferinfo.target_path) if transferinfo else None, - mode=settings.TRANSFER_TYPE, - type=mediainfo.type.value, - category=mediainfo.category, - title=mediainfo.title, - year=mediainfo.year, - tmdbid=mediainfo.tmdb_id, - imdbid=mediainfo.imdb_id, - tvdbid=mediainfo.tvdb_id, - doubanid=mediainfo.douban_id, - seasons=meta.season, - episodes=meta.episode, - image=mediainfo.get_poster_image(), - download_hash=torrent.hash, - status=1, - files=json.dumps(transferinfo.file_list) - ) - # 转移完成 - self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo) - # 刮削元数据 - self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) - # 刷新媒体库 - self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path) - # 发送通知 - self.send_transfer_message(meta=meta, mediainfo=mediainfo, transferinfo=transferinfo) - # 广播事件 - self.eventmanager.send_event(EventType.TransferComplete, { - 'meta': meta, - 'mediainfo': mediainfo, - 'transferinfo': transferinfo - }) - # 计数 - processed_num += 1 - # 更新进度 - self.progress.update(value=processed_num / total_num * 100, - text=f"{torrent.title} 转移完成", - key=ProgressKey.FileTransfer) - # 结束进度 - self.progress.end(ProgressKey.FileTransfer) - logger.info("下载器文件转移执行完成") - return True + # 转移完成 + self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo) + # 刮削元数据 + self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) + # 刷新媒体库 + self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path) + # 发送通知 + self.send_transfer_message(meta=meta, mediainfo=mediainfo, transferinfo=transferinfo) + # 广播事件 + self.eventmanager.send_event(EventType.TransferComplete, { + 'meta': meta, + 'mediainfo': mediainfo, + 'transferinfo': transferinfo + }) + # 计数 + processed_num += 1 + # 更新进度 + self.progress.update(value=processed_num / total_num * 100, + text=f"{torrent.title} 转移完成", + key=ProgressKey.FileTransfer) + # 结束进度 + self.progress.end(ProgressKey.FileTransfer) + logger.info("下载器文件转移执行完成") + return True def send_transfer_message(self, meta: MetaBase, mediainfo: MediaInfo, transferinfo: TransferInfo): """ diff --git a/app/modules/qbittorrent/__init__.py b/app/modules/qbittorrent/__init__.py index 6663d8c8..4c4ab689 100644 --- a/app/modules/qbittorrent/__init__.py +++ b/app/modules/qbittorrent/__init__.py @@ -1,7 +1,8 @@ -from msilib.schema import File from pathlib import Path from typing import Set, Tuple, Optional, Union, List +from qbittorrentapi import TorrentFilesList + from app import schemas from app.core.config import settings from app.core.metainfo import MetaInfo @@ -188,7 +189,7 @@ class QbittorrentModule(_ModuleBase): """ return self.qbittorrent.start_torrents(ids=hashs) - def torrent_files(self, tid: str) -> Optional[List[File]]: + def torrent_files(self, tid: str) -> Optional[TorrentFilesList]: """ 获取种子文件列表 """ diff --git a/app/modules/qbittorrent/qbittorrent.py b/app/modules/qbittorrent/qbittorrent.py index 09c7d67f..03e86ae2 100644 --- a/app/modules/qbittorrent/qbittorrent.py +++ b/app/modules/qbittorrent/qbittorrent.py @@ -1,9 +1,8 @@ import time -from msilib.schema import File from typing import Optional, Union, Tuple, List import qbittorrentapi -from qbittorrentapi import TorrentDictionary +from qbittorrentapi import TorrentDictionary, TorrentFilesList from qbittorrentapi.client import Client from qbittorrentapi.transfer import TransferInfoDictionary @@ -266,7 +265,7 @@ class Qbittorrent(metaclass=Singleton): logger.error(f"删除种子出错：{err}") return False - def get_files(self, tid: str) -> Optional[List[File]]: + def get_files(self, tid: str) -> Optional[TorrentFilesList]: """ 获取种子文件清单 """ diff --git a/app/modules/transmission/__init__.py b/app/modules/transmission/__init__.py index 1a8b17a4..01aa7ebd 100644 --- a/app/modules/transmission/__init__.py +++ b/app/modules/transmission/__init__.py @@ -1,7 +1,8 @@ -from msilib.schema import File from pathlib import Path from typing import Set, Tuple, Optional, Union, List +from transmission_rpc import File + from app import schemas from app.core.config import settings from app.core.metainfo import MetaInfo diff --git a/app/plugins/mediasyncdel/__init__.py b/app/plugins/mediasyncdel/__init__.py index 02b98fb4..5e3c2ab5 100644 --- a/app/plugins/mediasyncdel/__init__.py +++ b/app/plugins/mediasyncdel/__init__.py @@ -715,7 +715,7 @@ class MediaSyncDel(_PluginBase): logger.info(f"{history_key} 转种时未删除源下载任务，开始删除源下载任务…") try: - dl_files = self.chain.get_files(tid=torrent_hash) + dl_files = self.chain.torrent_files(tid=torrent_hash) if not dl_files: logger.info(f"未获取到 {settings.DOWNLOADER} - {torrent_hash} 种子文件，种子已被删除") else: @@ -736,7 +736,7 @@ class MediaSyncDel(_PluginBase): # 如果是False则说明种子文件没有完全被删除，暂停种子，暂不处理 if delete_flag: try: - dl_files = self.chain.get_files(tid=download_id) + dl_files = self.chain.torrent_files(tid=download_id) if not dl_files: logger.info(f"未获取到 {download} - {download_id} 种子文件，种子已被删除") else: