From fc2312a0454bae8304af12d54f20896b01f435f3 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sat, 26 Aug 2023 22:47:41 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=E6=89=8B=E5=8A=A8=E6=95=B4=E7=90=86API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/apiv1.py | 5 +- app/api/endpoints/transfer.py | 89 ++++++++++++++++++ app/chain/__init__.py | 11 ++- app/chain/transfer.py | 61 +++++++++++- app/modules/filetransfer/__init__.py | 48 +++++++++- app/modules/filetransfer/format_parser.py | 108 ++++++++++++++++++++++ app/schemas/transfer.py | 10 ++ app/utils/system.py | 6 +- requirements.txt | 1 + 9 files changed, 324 insertions(+), 15 deletions(-) create mode 100644 app/api/endpoints/transfer.py create mode 100644 app/modules/filetransfer/format_parser.py diff --git a/app/api/apiv1.py b/app/api/apiv1.py index 04a21128..3183c102 100644 --- a/app/api/apiv1.py +++ b/app/api/apiv1.py @@ -1,7 +1,7 @@ from fastapi import APIRouter from app.api.endpoints import login, user, site, message, webhook, subscribe, \ - media, douban, search, plugin, tmdb, history, system, download, dashboard, rss, filebrowser + media, douban, search, plugin, tmdb, history, system, download, dashboard, rss, filebrowser, transfer api_router = APIRouter() api_router.include_router(login.router, prefix="/login", tags=["login"]) @@ -20,4 +20,5 @@ api_router.include_router(plugin.router, prefix="/plugin", tags=["plugin"]) api_router.include_router(download.router, prefix="/download", tags=["download"]) api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"]) api_router.include_router(rss.router, prefix="/rss", tags=["rss"]) -api_router.include_router(filebrowser.router, prefix="/filebrowser", tags=["filebrowser"]) \ No newline at end of file +api_router.include_router(filebrowser.router, prefix="/filebrowser", tags=["filebrowser"]) +api_router.include_router(transfer.router, prefix="/transfer", tags=["transfer"]) diff --git a/app/api/endpoints/transfer.py b/app/api/endpoints/transfer.py new file mode 100644 index 00000000..3d9529d1 --- /dev/null +++ b/app/api/endpoints/transfer.py @@ -0,0 +1,89 @@ +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app import schemas +from app.chain.media import MediaChain +from app.chain.transfer import TransferChain +from app.core.config import settings +from app.core.context import MediaInfo +from app.core.metainfo import MetaInfo +from app.core.security import verify_token +from app.db import get_db +from app.schemas import MediaType + +router = APIRouter() + + +@router.post("/manual", summary="手动转移", response_model=schemas.Response) +def manual_transfer(path: str, + tmdbid: int, + type_name: str, + target: str = None, + season: int = None, + transfer_type: str = settings.TRANSFER_TYPE, + episode_format: str = None, + episode_detail: str = None, + episode_part: str = None, + episode_offset: int = None, + min_filesize: int = 0, + db: Session = Depends(get_db), + _: schemas.TokenPayload = Depends(verify_token)) -> Any: + """ + 手动转移，支持自定义剧集识别格式 + :param path: 转移路径或文件 + :param target: 目标路径 + :param type_name: 媒体类型、电影/电视剧 + :param tmdbid: tmdbid + :param season: 剧集季号 + :param transfer_type: 转移类型，move/copy + :param episode_format: 剧集识别格式 + :param episode_detail: 剧集识别详细信息 + :param episode_part: 剧集识别分集信息 + :param episode_offset: 剧集识别偏移量 + :param min_filesize: 最小文件大小(MB) + :param db: 数据库 + :param _: Token校验 + """ + in_path = Path(path) + if target: + target = Path(target) + if not target.exists(): + return schemas.Response(success=False, message=f"目标路径不存在") + # 识别元数据 + meta = MetaInfo(in_path.stem) + mtype = MediaType(type_name) + # 整合数据 + meta.type = mtype + if season: + meta.begin_season = season + # 识别媒体信息 + mediainfo: MediaInfo = MediaChain(db).recognize_media(tmdbid=tmdbid, mtype=mtype) + if not mediainfo: + return schemas.Response(success=False, message=f"媒体信息识别失败，tmdbid: {tmdbid}") + # 自定义格式 + epformat = None + if episode_offset or episode_part or episode_detail or episode_format: + epformat = schemas.EpisodeFormat( + format=episode_format, + detail=episode_detail, + part=episode_part, + offset=episode_offset, + ) + # 开始转移 + state, errormsg = TransferChain(db).manual_transfer( + in_path=in_path, + mediainfo=mediainfo, + transfer_type=transfer_type, + target=target, + meta=meta, + epformat=epformat, + min_filesize=min_filesize + ) + # 失败 + if not state: + return schemas.Response(success=False, message=errormsg) + # 成功 + return schemas.Response(success=True) diff --git a/app/chain/__init__.py b/app/chain/__init__.py index 8a4e9192..cd350739 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -18,7 +18,7 @@ from app.core.meta import MetaBase from app.core.module import ModuleManager from app.log import logger from app.schemas import TransferInfo, TransferTorrent, ExistMediaInfo, DownloadingTorrent, CommingMessage, Notification, \ - WebhookEventInfo + WebhookEventInfo, EpisodeFormat from app.schemas.types import TorrentStatus, MediaType, MediaImageType, EventType from app.utils.object import ObjectUtils @@ -274,7 +274,9 @@ class ChainBase(metaclass=ABCMeta): def transfer(self, path: Path, mediainfo: MediaInfo, transfer_type: str, target: Path = None, - meta: MetaBase = None) -> Optional[TransferInfo]: + meta: MetaBase = None, + epformat: EpisodeFormat = None, + min_filesize: int = 0) -> Optional[TransferInfo]: """ 文件转移 :param path: 文件路径 @@ -282,10 +284,13 @@ class ChainBase(metaclass=ABCMeta): :param transfer_type: 转移模式 :param target: 转移目标路径 :param meta: 预识别的元数据，仅单文件转移时传递 + :param epformat: 自定义剧集识别格式 + :param min_filesize: 最小文件大小 :return: {path, target_path, message} """ return self.run_module("transfer", path=path, mediainfo=mediainfo, - transfer_type=transfer_type, target=target, meta=meta) + transfer_type=transfer_type, target=target, meta=meta, + epformat=epformat, min_filesize=min_filesize) def transfer_completed(self, hashs: Union[str, list], transinfo: TransferInfo = None) -> None: """ diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 39339705..6d22dce5 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -17,7 +17,7 @@ from app.db.models.transferhistory import TransferHistory from app.db.transferhistory_oper import TransferHistoryOper from app.helper.progress import ProgressHelper from app.log import logger -from app.schemas import TransferInfo, TransferTorrent, Notification +from app.schemas import TransferInfo, TransferTorrent, Notification, EpisodeFormat from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey, NotificationType, MessageChannel from app.utils.string import StringUtils from app.utils.system import SystemUtils @@ -220,6 +220,7 @@ class TransferChain(ChainBase): """ 远程重新转移，参数 历史记录ID TMDBID|类型 """ + def args_error(): self.post_message(Notification(channel=channel, title="请输入正确的命令格式：/redo [id] [tmdbid]|[类型]，" @@ -331,8 +332,62 @@ class TransferChain(ChainBase): return True, "" - def __insert_sucess_history(self, src_path: Path, download_hash: str, meta: MetaBase, - mediainfo: MediaInfo, transferinfo: TransferInfo): + def manual_transfer(self, in_path: Path, + mediainfo: MediaInfo, + transfer_type: str = settings.TRANSFER_TYPE, + target: Path = None, + meta: MetaBase = None, + epformat: EpisodeFormat = None, + min_filesize: int = 0) -> Tuple[bool, str]: + """ + 手动转移 + :param in_path: 源文件路径 + :param mediainfo: 媒体信息 + :param transfer_type: 转移类型 + :param target: 目标路径 + :param meta: 元数据 + :param epformat: 剧集格式 + :param min_filesize: 最小文件大小(MB) + """ + # 开始转移 + transferinfo: TransferInfo = self.transfer( + path=in_path, + mediainfo=mediainfo, + transfer_type=transfer_type, + target=target, + meta=meta, + epformat=epformat, + min_filesize=min_filesize + ) + if not transferinfo: + return False, "文件转移模块运行失败" + if not transferinfo.target_path: + return False, transferinfo.message + + # 新增转移成功历史记录 + self.__insert_sucess_history( + src_path=in_path, + meta=meta, + mediainfo=mediainfo, + transferinfo=transferinfo + ) + # 刮削元数据 + self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo) + # 刷新媒体库 + self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path) + # 发送通知 + self.send_transfer_message(meta=meta, mediainfo=mediainfo, transferinfo=transferinfo) + # 广播事件 + self.eventmanager.send_event(EventType.TransferComplete, { + 'meta': meta, + 'mediainfo': mediainfo, + 'transferinfo': transferinfo + }) + return True, "" + + def __insert_sucess_history(self, src_path: Path, meta: MetaBase, + mediainfo: MediaInfo, transferinfo: TransferInfo, + download_hash: str = None): """ 新增转移成功历史记录 """ diff --git a/app/modules/filetransfer/__init__.py b/app/modules/filetransfer/__init__.py index b9321c27..ab22864a 100644 --- a/app/modules/filetransfer/__init__.py +++ b/app/modules/filetransfer/__init__.py @@ -11,7 +11,8 @@ from app.core.config import settings from app.core.meta import MetaBase from app.log import logger from app.modules import _ModuleBase -from app.schemas import TransferInfo +from app.modules.filetransfer.format_parser import FormatParser +from app.schemas import TransferInfo, EpisodeFormat from app.utils.system import SystemUtils from app.schemas.types import MediaType @@ -30,7 +31,10 @@ class FileTransferModule(_ModuleBase): pass def transfer(self, path: Path, mediainfo: MediaInfo, - transfer_type: str, target: Path = None, meta: MetaBase = None) -> TransferInfo: + transfer_type: str, target: Path = None, + meta: MetaBase = None, + epformat: EpisodeFormat = None, + min_filesize: int = 0) -> TransferInfo: """ 文件转移 :param path: 文件路径 @@ -38,6 +42,8 @@ class FileTransferModule(_ModuleBase): :param transfer_type: 转移方式 :param target: 目标路径 :param meta: 预识别的元数据，仅单文件转移时传递 + :param epformat: 集识别格式 + :param min_filesize: 最小文件大小(MB) :return: {path, target_path, message} """ # 获取目标路径 @@ -51,7 +57,9 @@ class FileTransferModule(_ModuleBase): mediainfo=mediainfo, transfer_type=transfer_type, target_dir=target, - in_meta=meta) + in_meta=meta, + epformat=epformat, + min_filesize=min_filesize) @staticmethod def __transfer_command(file_item: Path, target_file: Path, transfer_type: str) -> int: @@ -316,7 +324,9 @@ class FileTransferModule(_ModuleBase): mediainfo: MediaInfo, transfer_type: str, target_dir: Path = None, - in_meta: MetaBase = None + in_meta: MetaBase = None, + epformat: EpisodeFormat = None, + min_filesize: int = 0 ) -> TransferInfo: """ 识别并转移一个文件、多个文件或者目录 @@ -325,6 +335,8 @@ class FileTransferModule(_ModuleBase): :param target_dir: 目的文件夹，非空的转移到该文件夹，为空时则按类型转移到配置文件中的媒体库文件夹 :param transfer_type: 文件转移方式 :param in_meta：预识别元数，为空则重新识别 + :param epformat: 识别的剧集格式 + :param min_filesize: 最小文件大小（MB），小于该值的文件不转移 :return: TransferInfo、错误信息 """ # 检查目录路径 @@ -397,9 +409,24 @@ class FileTransferModule(_ModuleBase): file_list_new=[]) else: # 获取文件清单 - transfer_files: List[Path] = SystemUtils.list_files(in_path, settings.RMT_MEDIAEXT) + transfer_files: List[Path] = SystemUtils.list_files( + directory=in_path, + extensions=settings.RMT_MEDIAEXT, + min_filesize=min_filesize + ) if len(transfer_files) == 0: return TransferInfo(message=f"{in_path} 目录下没有找到可转移的文件") + # 有集自定义格式 + formaterHandler = FormatParser(eformat=epformat.format, + details=epformat.detail, + part=epformat.part, + offset=epformat.offset) if epformat else None + # 过滤出符合自定义剧集格式的文件 + if formaterHandler: + transfer_files = [x for x in transfer_files if formaterHandler.match(x.name)] + if len(transfer_files) == 0: + return TransferInfo(message=f"{in_path} 目录下没有找到符合自定义剧集格式的文件") + if not in_meta: # 识别目录名称，不包括后缀 meta = MetaInfo(in_path.stem) @@ -431,6 +458,16 @@ class FileTransferModule(_ModuleBase): file_meta.total_episode = 1 file_meta.end_episode = None + # 自定义识别 + if formaterHandler: + # 开始集、结束集、PART + begin_ep, end_ep, part = formaterHandler.split_episode(transfer_file.stem) + if begin_ep is not None: + file_meta.begin_episode = begin_ep + file_meta.part = part + if end_ep is not None: + file_meta.end_episode = end_ep + # 目的文件名 new_file = self.get_rename_path( path=target_dir, @@ -446,6 +483,7 @@ class FileTransferModule(_ModuleBase): if new_file.stat().st_size < transfer_file.stat().st_size: logger.info(f"目标文件已存在，但文件大小更小，将覆盖：{new_file}") overflag = True + # 转移文件 retcode = self.__transfer_file(file_item=transfer_file, new_file=new_file, diff --git a/app/modules/filetransfer/format_parser.py b/app/modules/filetransfer/format_parser.py new file mode 100644 index 00000000..64a843c1 --- /dev/null +++ b/app/modules/filetransfer/format_parser.py @@ -0,0 +1,108 @@ +import re +from typing import Tuple, Optional + +import parse + + +class FormatParser(object): + _key = "" + _split_chars = r"\.|\s+|\(|\)|\[|]|-|\+|【|】|/|～|;|&|\||#|_|「|」|~" + + def __init__(self, eformat: str, details: str = None, part: str = None, + offset: int = None, key: str = "ep"): + """ + :params eformat: 格式化字符串 + :params details: 格式化详情 + :params part: 分集 + :params offset: 偏移量 + :prams key: EP关键字 + """ + self._format = eformat + self._start_ep = None + self._end_ep = None + self._part = None + if part: + self._part = part + if details: + if re.compile("\\d{1,4}-\\d{1,4}").match(details): + self._start_ep = details + self._end_ep = details + else: + tmp = details.split(",") + if len(tmp) > 1: + self._start_ep = int(tmp[0]) + self._end_ep = int(tmp[0]) if int(tmp[0]) > int(tmp[1]) else int(tmp[1]) + else: + self._start_ep = self._end_ep = int(tmp[0]) + self.__offset = int(offset) if offset else 0 + self._key = key + + @property + def format(self): + return self._format + + @property + def start_ep(self): + return self._start_ep + + @property + def end_ep(self): + return self._end_ep + + @property + def part(self): + return self._part + + @property + def offset(self): + return self.__offset + + def match(self, file: str) -> bool: + if not self._format: + return True + s, e = self.__handle_single(file) + if not s: + return False + if self._start_ep is None: + return True + if self._start_ep <= s <= self._end_ep: + return True + return False + + def split_episode(self, file_name: str) -> Tuple[Optional[int], Optional[int], Optional[str]]: + """ + 拆分集数，返回开始集数，结束集数，Part信息 + """ + # 指定的具体集数，直接返回 + if self._start_ep is not None and self._start_ep == self._end_ep: + if isinstance(self._start_ep, str): + s, e = self._start_ep.split("-") + if int(s) == int(e): + return int(s) + self.__offset, None, self.part + return int(s) + self.__offset, int(e) + self.__offset, self.part + return self._start_ep + self.__offset, None, self.part + if not self._format: + return None, None, None + s, e = self.__handle_single(file_name) + return s + self.__offset if s is not None else None, \ + e + self.__offset if e is not None else None, self.part + + def __handle_single(self, file: str) -> Tuple[Optional[int], Optional[int]]: + """ + 处理单集，返回单集的开始和结束集数 + """ + if not self._format: + return None, None + ret = parse.parse(self._format, file) + if not ret or not ret.__contains__(self._key): + return None, None + episodes = ret.__getitem__(self._key) + if not re.compile(r"^(EP)?(\d{1,4})(-(EP)?(\d{1,4}))?$", re.IGNORECASE).match(episodes): + return None, None + episode_splits = list(filter(lambda x: re.compile(r'[a-zA-Z]*\d{1,4}', re.IGNORECASE).match(x), + re.split(r'%s' % self._split_chars, episodes))) + if len(episode_splits) == 1: + return int(re.compile(r'[a-zA-Z]*', re.IGNORECASE).sub("", episode_splits[0])), None + else: + return int(re.compile(r'[a-zA-Z]*', re.IGNORECASE).sub("", episode_splits[0])), int( + re.compile(r'[a-zA-Z]*', re.IGNORECASE).sub("", episode_splits[1])) diff --git a/app/schemas/transfer.py b/app/schemas/transfer.py index 26cbaa94..e98cf013 100644 --- a/app/schemas/transfer.py +++ b/app/schemas/transfer.py @@ -62,3 +62,13 @@ class TransferInfo(BaseModel): dicts["path"] = str(self.path) if self.path else None dicts["target_path"] = str(self.target_path) if self.target_path else None return dicts + + +class EpisodeFormat(BaseModel): + """ + 剧集自定义识别格式 + """ + format: Optional[str] = None + detail: Optional[str] = None + part: Optional[str] = None + offset: Optional[int] = None diff --git a/app/utils/system.py b/app/utils/system.py index d809f548..fa2dd46c 100644 --- a/app/utils/system.py +++ b/app/utils/system.py @@ -91,7 +91,7 @@ class SystemUtils: return -1, str(err) @staticmethod - def list_files(directory: Path, extensions: list) -> List[Path]: + def list_files(directory: Path, extensions: list, min_filesize: int = 0) -> List[Path]: """ 获取目录下所有指定扩展名的文件（包括子目录） """ @@ -106,7 +106,9 @@ class SystemUtils: # 遍历目录及子目录 for path in directory.rglob('**/*'): - if path.is_file() and re.match(pattern, path.name, re.IGNORECASE): + if path.is_file() \ + and re.match(pattern, path.name, re.IGNORECASE) \ + and path.stat().st_size >= min_filesize * 1024 * 1024: files.append(path) return files diff --git a/requirements.txt b/requirements.txt index 129ededc..8ec1530e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,3 +49,4 @@ openai~=0.27.2 cacheout~=0.14.1 click~=8.1.6 requests_cache~=0.5.2 +parse==1.19.0 \ No newline at end of file