codfrm/MoviePilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat 下载器监控支持转移合集

Browse Source
This commit is contained in:
jxxghp 2023-08-23 08:47:03 +08:00
parent e5e33d4486
commit 66a1f25465
10 changed files with 155 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
app/chain/transfer.py
View File

@ -60,6 +60,7 @@ class TransferChain(ChainBase):
self.progress.update(value=0,
text=f"开始转移下载任务文件，共 {total_num} 个任务 ...",
key=ProgressKey.FileTransfer)
for torrent in torrents:
# 更新进度
self.progress.update(value=processed_num / total_num * 100,
@ -105,12 +106,20 @@ class TransferChain(ChainBase):
continue
logger.info(f"{torrent.title} 识别为：{mediainfo.type.value} {mediainfo.title_year}")
# 更新媒体图片
self.obtain_images(mediainfo=mediainfo)
# 转移
# 获取待转移路径清单
trans_paths = self.__get_trans_paths(torrent.path)
if not trans_paths:
logger.warn(f"{torrent.title} 对应目录没有找到媒体文件")
continue
# 转移所有文件
for trans_path in trans_paths:
transferinfo: TransferInfo = self.transfer(mediainfo=mediainfo,
path=torrent.path,
path=trans_path,
transfer_type=settings.TRANSFER_TYPE)
if not transferinfo:
logger.error("文件转移模块运行失败")
@ -120,7 +129,7 @@ class TransferChain(ChainBase):
logger.warn(f"{torrent.title} 入库失败：{transferinfo.message}")
# 新增转移失败历史记录
self.__insert_fail_history(
src_path=torrent.path,
src_path=trans_path,
download_hash=torrent.hash,
meta=meta,
mediainfo=mediainfo,
@ -132,20 +141,16 @@ class TransferChain(ChainBase):
text=f"原因：{transferinfo.message or '未知'}",
image=mediainfo.get_message_image()
))
# 设置种子状态，避免一直报错
self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo)
continue
# 新增转移成功历史记录
self.__insert_sucess_history(
src_path=torrent.path,
src_path=trans_path,
download_hash=torrent.hash,
meta=meta,
mediainfo=mediainfo,
transferinfo=transferinfo
)
# 转移完成
self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo)
# 刮削元数据
self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo)
# 刷新媒体库
@ -158,6 +163,10 @@ class TransferChain(ChainBase):
'mediainfo': mediainfo,
'transferinfo': transferinfo
})
# 转移完成
self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo)
# 计数
processed_num += 1
# 更新进度
@ -169,6 +178,38 @@ class TransferChain(ChainBase):
logger.info("下载器文件转移执行完成")
return True
@staticmethod
def __get_trans_paths(directory: Path):
"""
获取转移目录列表
"""
if not directory.exists():
logger.warn(f"目录不存在：{directory}")
return []
if directory.is_file():
return [directory]
# 需要转移的路径列表
trans_paths = []
# 先检查当前目录的下级目录，以支持合集的情况
for sub_dir in SystemUtils.list_sub_directory(directory):
# 没有媒体文件的目录跳过
if not SystemUtils.list_files(sub_dir, extensions=settings.RMT_MEDIAEXT):
continue
trans_paths.append(sub_dir)
if not trans_paths:
# 没有有效子目录，直接转移当前目录
trans_paths.append(directory)
else:
# 有子目录时，把当前目录的文件添加到转移任务中
trans_paths.extend(
SystemUtils.list_sub_files(directory, extensions=settings.RMT_MEDIAEXT)
)
def remote_transfer(self, arg_str: str, channel: MessageChannel, userid: Union[str, int] = None):
"""
远程重新转移，参数 历史记录ID TMDBID|类型
@ -210,7 +251,7 @@ class TransferChain(ChainBase):
def re_transfer(self, logid: int, mtype: MediaType, tmdbid: int) -> Tuple[bool, str]:
"""
根据历史记录，重新识别转移
根据历史记录，重新识别转移，只处理对应的src目录
:param logid: 历史记录ID
:param mtype: 媒体类型
:param tmdbid: TMDB ID
@ -220,14 +261,6 @@ class TransferChain(ChainBase):
if not history:
logger.error(f"历史记录不存在，ID：{logid}")
return False, "历史记录不存在"
if history.download_hash:
# 有下载记录，按下载记录重新转移
torrents: Optional[List[TransferTorrent]] = self.list_torrents(hashs=history.download_hash)
if not torrents:
return False, f"没有获取到种子，hash：{history.download_hash}"
# 源目录
src_path = Path(torrents[0].path)
else:
# 没有下载记录，按源目录路径重新转移
src_path = Path(history.src)
if not src_path.exists():
@ -252,9 +285,13 @@ class TransferChain(ChainBase):
if not transferinfo:
logger.error("文件转移模块运行失败")
return False, "文件转移模块运行失败"
# 删除旧历史记录
self.transferhis.delete(logid)
if not transferinfo.target_path:
# 转移失败
logger.warn(f"{src_path.name} 入库失败：{transferinfo.message}")
logger.warn(f"{src_path} 入库失败：{transferinfo.message}")
# 新增转移失败历史记录
self.__insert_fail_history(
src_path=src_path,
@ -273,8 +310,6 @@ class TransferChain(ChainBase):
mediainfo=mediainfo,
transferinfo=transferinfo
)
# 删除旧历史记录
self.transferhis.delete(logid)
# 刮削元数据
self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo)
# 刷新媒体库
@ -287,6 +322,7 @@ class TransferChain(ChainBase):
'mediainfo': mediainfo,
'transferinfo': transferinfo
})
return True, ""
def __insert_sucess_history(self, src_path: Path, download_hash: str, meta: MetaBase,
@ -294,7 +330,7 @@ class TransferChain(ChainBase):
"""
新增转移成功历史记录
"""
self.transferhis.add(
self.transferhis.add_force(
src=str(src_path),
dest=str(transferinfo.target_path),
mode=settings.TRANSFER_TYPE,
@ -317,10 +353,10 @@ class TransferChain(ChainBase):
def __insert_fail_history(self, src_path: Path, download_hash: str, meta: MetaBase,
transferinfo: TransferInfo = None, mediainfo: MediaInfo = None):
"""
新增转移失败历史记录
新增转移失败历史记录，不能按download_hash判重
"""
if mediainfo and transferinfo:
his = self.transferhis.add(
his = self.transferhis.add_force(
src=str(src_path),
dest=str(transferinfo.target_path),
mode=settings.TRANSFER_TYPE,
@ -341,7 +377,7 @@ class TransferChain(ChainBase):
files=json.dumps(transferinfo.file_list)
)
else:
his = self.transferhis.add(
his = self.transferhis.add_force(
src=str(src_path),
mode=settings.TRANSFER_TYPE,
seasons=meta.season,
@ -389,8 +425,8 @@ class TransferChain(ChainBase):
logger.warn(f"文件 {path} 已删除")
# 判断目录是否为空, 为空则删除
if str(path.parent.parent) != str(path.root):
# 父父目录非根目录，才删除父目录
files = SystemUtils.list_files_with_extensions(path.parent, settings.RMT_MEDIAEXT)
# 父目录非根目录，才删除父目录
files = SystemUtils.list_files(path.parent, settings.RMT_MEDIAEXT)
if not files:
shutil.rmtree(path.parent)
logger.warn(f"目录 {path.parent} 已删除")

2
app/modules/douban/__init__.py
View File

@ -167,7 +167,7 @@ class DoubanModule(_ModuleBase):
if settings.SCRAP_SOURCE != "douban":
return None
# 目录下的所有文件
for file in SystemUtils.list_files_with_extensions(path, settings.RMT_MEDIAEXT):
for file in SystemUtils.list_files(path, settings.RMT_MEDIAEXT):
if not file:
continue
logger.info(f"开始刮削媒体库文件：{file} ...")

6
app/modules/filetransfer/__init__.py
View File

@ -121,7 +121,7 @@ class FileTransferModule(_ModuleBase):
# 比对文件名并转移字幕
org_dir: Path = org_path.parent
file_list: List[Path] = SystemUtils.list_files_with_extensions(org_dir, settings.RMT_SUBEXT)
file_list: List[Path] = SystemUtils.list_files(org_dir, settings.RMT_SUBEXT)
if len(file_list) == 0:
logger.debug(f"{org_dir} 目录下没有找到字幕文件...")
else:
@ -207,7 +207,7 @@ class FileTransferModule(_ModuleBase):
"""
dir_name = org_path.parent
file_name = org_path.name
file_list: List[Path] = SystemUtils.list_files_with_extensions(dir_name, ['.mka'])
file_list: List[Path] = SystemUtils.list_files(dir_name, ['.mka'])
pending_file_list: List[Path] = [file for file in file_list if org_path.stem == file.stem]
if len(pending_file_list) == 0:
logger.debug(f"{dir_name} 目录下没有找到匹配的音轨文件")
@ -409,7 +409,7 @@ class FileTransferModule(_ModuleBase):
file_list_new=[])
else:
# 获取文件清单
transfer_files: List[Path] = SystemUtils.list_files_with_extensions(in_path, settings.RMT_MEDIAEXT)
transfer_files: List[Path] = SystemUtils.list_files(in_path, settings.RMT_MEDIAEXT)
if len(transfer_files) == 0:
return TransferInfo(message=f"{in_path} 目录下没有找到可转移的文件")
if not in_meta:

2
app/modules/qbittorrent/__init__.py
View File

@ -168,7 +168,7 @@ class QbittorrentModule(_ModuleBase):
logger.info(f"移动模式删除种子成功：{hashs} ")
# 删除残留文件
if transinfo.path and transinfo.path.exists():
files = SystemUtils.list_files_with_extensions(transinfo.path, settings.RMT_MEDIAEXT)
files = SystemUtils.list_files(transinfo.path, settings.RMT_MEDIAEXT)
if not files:
logger.warn(f"删除残留文件夹：{transinfo.path}")
shutil.rmtree(transinfo.path, ignore_errors=True)

2
app/modules/subtitle/__init__.py
View File

@ -108,7 +108,7 @@ class SubtitleModule(_ModuleBase):
# 解压文件
shutil.unpack_archive(zip_file, zip_path, format='zip')
# 遍历转移文件
for sub_file in SystemUtils.list_files_with_extensions(zip_path, settings.RMT_SUBEXT):
for sub_file in SystemUtils.list_files(zip_path, settings.RMT_SUBEXT):
target_sub_file = download_dir / sub_file.name
if target_sub_file.exists():
logger.info(f"字幕文件已存在：{target_sub_file}")

2
app/modules/themoviedb/__init__.py
View File

@ -190,7 +190,7 @@ class TheMovieDbModule(_ModuleBase):
if settings.SCRAP_SOURCE != "themoviedb":
return None
# 目录下的所有文件
for file in SystemUtils.list_files_with_extensions(path, settings.RMT_MEDIAEXT):
for file in SystemUtils.list_files(path, settings.RMT_MEDIAEXT):
if not file:
continue
logger.info(f"开始刮削媒体库文件：{file} ...")

2
app/modules/transmission/__init__.py
View File

@ -152,7 +152,7 @@ class TransmissionModule(_ModuleBase):
logger.info(f"移动模式删除种子成功：{hashs} ")
# 删除残留文件
if transinfo.path and transinfo.path.exists():
files = SystemUtils.list_files_with_extensions(transinfo.path, settings.RMT_MEDIAEXT)
files = SystemUtils.list_files(transinfo.path, settings.RMT_MEDIAEXT)
if not files:
logger.warn(f"删除残留文件夹：{transinfo.path}")
shutil.rmtree(transinfo.path, ignore_errors=True)

2
app/plugins/dirmonitor/__init__.py
View File

@ -309,7 +309,7 @@ class DirMonitor(_PluginBase):
if len(str(file_dir)) <= len(str(Path(mon_path))):
# 重要，删除到监控目录为止
break
files = SystemUtils.list_files_with_extensions(file_dir, settings.RMT_MEDIAEXT)
files = SystemUtils.list_files(file_dir, settings.RMT_MEDIAEXT)
if not files:
logger.warn(f"移动模式，删除空目录：{file_dir}")
shutil.rmtree(file_dir, ignore_errors=True)

2
app/plugins/libraryscraper/__init__.py
View File

@ -258,7 +258,7 @@ class LibraryScraper(_PluginBase):
"""
exclude_paths = self._exclude_paths.split("\n")
# 查找目录下所有的文件
files = SystemUtils.list_files_with_extensions(path, settings.RMT_MEDIAEXT)
files = SystemUtils.list_files(path, settings.RMT_MEDIAEXT)
for file in files:
if self._event.is_set():
logger.info(f"媒体库刮削服务停止")

48
app/utils/system.py
View File

@ -91,10 +91,13 @@ class SystemUtils:
return -1, str(err)
@staticmethod
def list_files_with_extensions(directory: Path, extensions: list) -> List[Path]:
def list_files(directory: Path, extensions: list) -> List[Path]:
"""
获取目录下所有指定扩展名的文件
获取目录下所有指定扩展名的文件（包括子目录）
"""
if not directory.exists():
return []
if directory.is_file():
return [directory]
@ -108,6 +111,47 @@ class SystemUtils:
return files
@staticmethod
def list_sub_files(directory: Path, extensions: list) -> List[Path]:
"""
列出当前目录下的所有指定扩展名的文件(不包括子目录)
"""
if not directory.exists():
return []
if directory.is_file():
return [directory]
files = []
pattern = r".*(" + "|".join(extensions) + ")$"
# 遍历目录
for path in directory.iterdir():
if path.is_file() and re.match(pattern, path.name, re.IGNORECASE):
files.append(path)
return files
@staticmethod
def list_sub_directory(directory: Path) -> List[Path]:
"""
列出当前目录下的所有子目录（不递归）
"""
if not directory.exists():
return []
if directory.is_file():
return []
dirs = []
# 遍历目录
for path in directory.iterdir():
if path.is_dir():
dirs.append(path)
return dirs
@staticmethod
def get_directory_size(path: Path) -> float:
"""