diff --git a/app/api/endpoints/media.py b/app/api/endpoints/media.py index 343c17b4..048de65e 100644 --- a/app/api/endpoints/media.py +++ b/app/api/endpoints/media.py @@ -116,14 +116,12 @@ def scrape(fileitem: schemas.FileItem, if storage == "local": if not scrape_path.exists(): return schemas.Response(success=False, message="刮削路径不存在") - # 刮削本地 - chain.scrape_metadata(path=scrape_path, mediainfo=mediainfo, transfer_type=settings.TRANSFER_TYPE) else: if not fileitem.fileid: return schemas.Response(success=False, message="刮削文件ID无效") - # 刮削在线 - chain.scrape_metadata_online(storage=storage, fileitem=fileitem, meta=meta, mediainfo=mediainfo) - return schemas.Response(success=True, message="刮削完成") + # 手动刮削 + chain.manual_scrape(storage=storage, fileitem=fileitem, meta=meta, mediainfo=mediainfo) + return schemas.Response(success=True, message=f"{fileitem.path} 刮削完成") @router.get("/category", summary="查询自动分类配置", response_model=dict) diff --git a/app/chain/media.py b/app/chain/media.py index 054facea..8b713c18 100644 --- a/app/chain/media.py +++ b/app/chain/media.py @@ -2,7 +2,7 @@ import copy import time from pathlib import Path from threading import Lock -from typing import Optional, List, Tuple +from typing import Optional, List, Tuple, Union from app import schemas from app.chain import ChainBase @@ -18,6 +18,7 @@ from app.schemas.types import EventType, MediaType from app.utils.http import RequestUtils from app.utils.singleton import Singleton from app.utils.string import StringUtils +from app.utils.system import SystemUtils recognize_lock = Lock() @@ -31,8 +32,8 @@ class MediaChain(ChainBase, metaclass=Singleton): # 临时识别结果 {title, name, year, season, episode} recognize_temp: Optional[dict] = None - def meta_nfo(self, meta: MetaBase, mediainfo: MediaInfo, - season: int = None, episode: int = None) -> Optional[str]: + def metadata_nfo(self, meta: MetaBase, mediainfo: MediaInfo, + season: int = None, episode: int = None) -> Optional[str]: """ 获取NFO文件内容文本 :param meta: 元数据 @@ -40,7 +41,7 @@ class MediaChain(ChainBase, metaclass=Singleton): :param season: 季号 :param episode: 集号 """ - return self.run_module("meta_nfo", meta=meta, mediainfo=mediainfo, season=season, episode=episode) + return self.run_module("metadata_nfo", meta=meta, mediainfo=mediainfo, season=season, episode=episode) def recognize_by_meta(self, metainfo: MetaBase) -> Optional[MediaInfo]: """ @@ -332,42 +333,66 @@ class MediaChain(ChainBase, metaclass=Singleton): ) return None - def scrape_metadata_online(self, storage: str, fileitem: schemas.FileItem, - meta: MetaBase, mediainfo: MediaInfo, init_folder: bool = True): + def manual_scrape(self, storage: str, fileitem: schemas.FileItem, + meta: MetaBase, mediainfo: MediaInfo, init_folder: bool = True): """ - 远程刮削媒体信息(网盘等) + 手动刮削媒体信息 """ def __list_files(_storage: str, _fileid: str, _path: str = None, _drive_id: str = None): + """ + 列出下级文件 + """ if _storage == "aliyun": return AliyunHelper().list(drive_id=_drive_id, parent_file_id=_fileid, path=_path) - if _storage == "u115": + elif _storage == "u115": return U115Helper().list(parent_file_id=_fileid, path=_path) - return [] + else: + items = SystemUtils.list_sub_all(Path(_path)) + return [schemas.FileItem( + type="file" if item.is_file() else "dir", + path=str(item), + name=item.name, + basename=item.stem, + extension=item.suffix[1:], + size=item.stat().st_size, + modify_time=item.stat().st_mtime + ) for item in items] - def __upload_file(_storage: str, _fileid: str, _path: Path): - if _storage == "aliyun": - return AliyunHelper().upload(parent_file_id=_fileid, file_path=_path) - if _storage == "u115": - return U115Helper().upload(parent_file_id=_fileid, file_path=_path) + def __save_file(_storage: str, _drive_id: str, _fileid: str, _path: Path, _content: Union[bytes, str]): + """ + 保存或上传文件 + """ + if _storage != "local": + # 写入到临时目录 + temp_path = settings.TEMP_PATH / _path.name + temp_path.write_bytes(_content) + # 上传文件 + logger.info(f"正在上传 {_path.name} ...") + if _storage == "aliyun": + AliyunHelper().upload(drive_id=_drive_id, parent_file_id=_fileid, file_path=temp_path) + elif _storage == "u115": + U115Helper().upload(parent_file_id=_fileid, file_path=temp_path) + logger.info(f"{_path.name} 上传完成") + else: + # 保存到本地 + logger.info(f"正在保存 {_path.name} ...") + _path.write_bytes(_content) + logger.info(f"{_path} 已保存") - def __save_image(u: str, f: Path): + def __save_image(_url: str) -> Optional[bytes]: """ 下载图片并保存 """ try: - logger.info(f"正在下载{f.stem}图片:{u} ...") - r = RequestUtils(proxies=settings.PROXY).get_res(url=u) + logger.info(f"正在下载图片:{_url} ...") + r = RequestUtils(proxies=settings.PROXY).get_res(url=_url) if r: - f.write_bytes(r.content) + return r.content else: - logger.info(f"{f.stem}图片下载失败,请检查网络连通性!") + logger.info(f"{_url} 图片下载失败,请检查网络连通性!") except Exception as err: - logger.error(f"{f.stem}图片下载失败:{str(err)}!") - - if storage not in ["aliyun", "u115"]: - logger.warn(f"不支持的存储类型:{storage}") - return + logger.error(f"{_url} 图片下载失败:{str(err)}!") # 当前文件路径 filepath = Path(fileitem.path) @@ -380,27 +405,24 @@ class MediaChain(ChainBase, metaclass=Singleton): if fileitem.type == "file": # 电影文件 logger.info(f"正在生成电影nfo:{mediainfo.title_year} - {filepath.name}") - movie_nfo = self.meta_nfo(meta=meta, mediainfo=mediainfo) + movie_nfo = self.metadata_nfo(meta=meta, mediainfo=mediainfo) if not movie_nfo: logger.warn(f"{filepath.name} nfo文件生成失败!") return - # 写入到临时目录 - nfo_path = settings.TEMP_PATH / f"{filepath.stem}.nfo" - nfo_path.write_bytes(movie_nfo) - # 上传NFO文件 - logger.info(f"上传NFO文件:{nfo_path.name} ...") - __upload_file(storage, fileitem.parent_fileid, nfo_path) - logger.info(f"{nfo_path.name} 上传成功") + # 保存或上传nfo文件 + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.parent_fileid, + _path=filepath.with_suffix(".nfo"), _content=movie_nfo) else: # 电影目录 files = __list_files(_storage=storage, _fileid=fileitem.fileid, _drive_id=fileitem.drive_id, _path=fileitem.path) for file in files: - self.scrape_metadata_online(storage=storage, fileitem=file, - meta=meta, mediainfo=mediainfo, - init_folder=False) - # 生成图片文件和上传 + self.manual_scrape(storage=storage, fileitem=file, + meta=meta, mediainfo=mediainfo, + init_folder=False) + # 生成目录内图片文件 if init_folder: + # 图片 for attr_name, attr_value in vars(mediainfo).items(): if attr_value \ and attr_name.endswith("_path") \ @@ -408,13 +430,12 @@ class MediaChain(ChainBase, metaclass=Singleton): and isinstance(attr_value, str) \ and attr_value.startswith("http"): image_name = attr_name.replace("_path", "") + Path(attr_value).suffix + image_path = filepath / image_name + # 下载图片 + content = __save_image(_url=attr_value) # 写入nfo到根目录 - image_path = settings.TEMP_PATH / image_name - __save_image(attr_value, image_path) - # 上传图片文件到当前目录 - logger.info(f"上传图片文件:{image_path.name} ...") - __upload_file(storage, fileitem.fileid, image_path) - logger.info(f"{image_path.name} 上传成功") + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.fileid, + _path=image_path, _content=content) else: # 电视剧 if fileitem.type == "file": @@ -428,94 +449,83 @@ class MediaChain(ChainBase, metaclass=Singleton): logger.warn(f"{filepath.name} 无法识别文件媒体信息!") return # 获取集的nfo文件 - episode_nfo = self.meta_nfo(meta=file_meta, mediainfo=file_mediainfo, - season=file_meta.begin_season, episode=file_meta.begin_episode) + episode_nfo = self.metadata_nfo(meta=file_meta, mediainfo=file_mediainfo, + season=file_meta.begin_season, episode=file_meta.begin_episode) if not episode_nfo: logger.warn(f"{filepath.name} nfo生成失败!") return - # 写入到临时目录 - nfo_path = settings.TEMP_PATH / f"{filepath.stem}.nfo" - nfo_path.write_bytes(episode_nfo) - # 上传NFO文件,到文件当前目录下 - logger.info(f"上传NFO文件:{nfo_path.name} ...") - __upload_file(storage, fileitem.parent_fileid, nfo_path) - logger.info(f"{nfo_path.name} 上传成功") + # 保存或上传nfo文件 + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.parent_fileid, + _path=filepath.with_suffix(".nfo"), _content=episode_nfo) elif meta.begin_season: # 当前为季的目录,处理目录内的文件 files = __list_files(_storage=storage, _fileid=fileitem.fileid, _drive_id=fileitem.drive_id, _path=fileitem.path) for file in files: - self.scrape_metadata_online(storage=storage, fileitem=file, - meta=meta, mediainfo=mediainfo, - init_folder=False) + self.manual_scrape(storage=storage, fileitem=file, + meta=meta, mediainfo=mediainfo, + init_folder=False) # 生成季的nfo和图片 if init_folder: # 季nfo - season_nfo = self.meta_nfo(meta=meta, mediainfo=mediainfo, season=meta.begin_season) + season_nfo = self.metadata_nfo(meta=meta, mediainfo=mediainfo, season=meta.begin_season) if not season_nfo: logger.warn(f"无法生成电视剧季nfo文件:{meta.name}") return # 写入nfo到根目录 - nfo_path = settings.TEMP_PATH / "season.nfo" - nfo_path.write_bytes(season_nfo) - # 上传NFO文件 - logger.info(f"上传NFO文件:{nfo_path.name} ...") - __upload_file(storage, fileitem.fileid, nfo_path) - logger.info(f"{nfo_path.name} 上传成功") + nfo_path = filepath / "season.nfo" + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.fileid, + _path=nfo_path, _content=season_nfo) # TMDB季poster图片 - sea_seq = str(meta.begin_season).rjust(2, '0') - # 查询季剧详情 - seasoninfo = self.tmdb_info(tmdbid=mediainfo.tmdb_id, mtype=MediaType.TV, - season=meta.begin_season) - if not seasoninfo: - logger.warn(f"无法获取 {mediainfo.title_year} 第{meta.begin_season}季 的媒体信息!") - return - if seasoninfo.get("poster_path"): - # 下载图片 - ext = Path(seasoninfo.get('poster_path')).suffix - url = f"https://{settings.TMDB_IMAGE_DOMAIN}/t/p/original{seasoninfo.get('poster_path')}" - image_path = filepath.parent.with_name(f"season{sea_seq}-poster{ext}") - __save_image(url, image_path) - # 上传图片文件到当前目录 - logger.info(f"上传图片文件:{image_path.name} ...") - __upload_file(storage, fileitem.fileid, image_path) - logger.info(f"{image_path.name} 上传成功") - # 季的其它图片 - for attr_name, attr_value in vars(mediainfo).items(): - if attr_value \ - and attr_name.startswith("season") \ - and not attr_name.endswith("poster_path") \ - and attr_value \ - and isinstance(attr_value, str) \ - and attr_value.startswith("http"): - image_name = attr_name.replace("_path", "") + Path(attr_value).suffix - image_path = filepath.parent.with_name(image_name) - __save_image(attr_value, image_path) - # 上传图片文件到当前目录 - logger.info(f"上传图片文件:{image_path.name} ...") - __upload_file(storage, fileitem.fileid, image_path) - logger.info(f"{image_path.name} 上传成功") + if settings.SCRAP_SOURCE == "themoviedb": + sea_seq = str(meta.begin_season).rjust(2, '0') + # 查询季剧详情 + seasoninfo = self.tmdb_info(tmdbid=mediainfo.tmdb_id, mtype=MediaType.TV, + season=meta.begin_season) + if not seasoninfo: + logger.warn(f"无法获取 {mediainfo.title_year} 第{meta.begin_season}季 的媒体信息!") + return + if seasoninfo.get("poster_path"): + # 下载图片 + content = __save_image(f"https://{settings.TMDB_IMAGE_DOMAIN}/t/p/original" + f"{seasoninfo.get('poster_path')}") + image_path = filepath.with_name(f"season{sea_seq}" + f"-poster{Path(seasoninfo.get('poster_path')).suffix}") + # 保存图片文件到当前目录 + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.fileid, + _path=image_path, _content=content) + # 季的其它图片 + for attr_name, attr_value in vars(mediainfo).items(): + if attr_value \ + and attr_name.startswith("season") \ + and not attr_name.endswith("poster_path") \ + and attr_value \ + and isinstance(attr_value, str) \ + and attr_value.startswith("http"): + image_name = attr_name.replace("_path", "") + Path(attr_value).suffix + image_path = filepath.parent.with_name(image_name) + content = __save_image(attr_value) + # 保存图片文件到当前目录 + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.fileid, + _path=image_path, _content=content) else: # 当前为根目录,处理目录内的文件 files = __list_files(_storage=storage, _fileid=fileitem.fileid, _drive_id=fileitem.drive_id, _path=fileitem.path) for file in files: - self.scrape_metadata_online(storage=storage, fileitem=file, - meta=meta, mediainfo=mediainfo, - init_folder=False) + self.manual_scrape(storage=storage, fileitem=file, + meta=meta, mediainfo=mediainfo, + init_folder=False) # 生成根目录的nfo和图片 if init_folder: - tv_nfo = self.meta_nfo(meta=meta, mediainfo=mediainfo) + tv_nfo = self.metadata_nfo(meta=meta, mediainfo=mediainfo) if not tv_nfo: logger.warn(f"无法生成电视剧nfo文件:{meta.name}") return # 写入nfo到根目录 - nfo_path = settings.TEMP_PATH / "tvshow.nfo" - nfo_path.write_bytes(tv_nfo) - # 上传NFO文件 - logger.info(f"上传NFO文件:{nfo_path.name} ...") - __upload_file(storage, fileitem.fileid, nfo_path) - logger.info(f"{nfo_path.name} 上传成功") + nfo_path = filepath / "tvshow.nfo" + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.fileid, + _path=nfo_path, _content=tv_nfo) # 生成根目录图片 for attr_name, attr_value in vars(mediainfo).items(): if attr_name \ @@ -526,10 +536,9 @@ class MediaChain(ChainBase, metaclass=Singleton): and attr_value.startswith("http"): image_name = attr_name.replace("_path", "") + Path(attr_value).suffix image_path = filepath.parent.with_name(image_name) - __save_image(attr_value, image_path) - # 上传图片文件到当前目录 - logger.info(f"上传图片文件:{image_path.name} ...") - __upload_file(storage, fileitem.fileid, image_path) - logger.info(f"{image_path.name} 上传成功") + content = __save_image(attr_value) + # 保存图片文件到当前目录 + __save_file(_storage=storage, _drive_id=fileitem.drive_id, _fileid=fileitem.fileid, + _path=image_path, _content=content) logger.info(f"{filepath.name} 刮削完成") diff --git a/app/helper/aliyun.py b/app/helper/aliyun.py index 1e822f28..6b865550 100644 --- a/app/helper/aliyun.py +++ b/app/helper/aliyun.py @@ -546,7 +546,7 @@ class AliyunHelper: self.__handle_error(res, "移动文件") return False - def upload(self, parent_file_id: str, file_path: Path) -> Optional[dict]: + def upload(self, drive_id: str, parent_file_id: str, file_path: Path) -> Optional[schemas.FileItem]: """ 上传文件,并标记完成 """ @@ -555,7 +555,7 @@ class AliyunHelper: return None headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.create_file_url, json={ - "drive_id": params.get("resourceDriveId"), + "drive_id": drive_id, "parent_file_id": parent_file_id, "name": file_path.name, "type": "file", @@ -566,7 +566,6 @@ class AliyunHelper: return None # 获取上传参数 result = res.json() - drive_id = result.get("drive_id") file_id = result.get("file_id") upload_id = result.get("upload_id") part_info_list = result.get("part_info_list") @@ -587,10 +586,15 @@ class AliyunHelper: if not res: self.__handle_error(res, "标记上传状态") return None - return { - "drive_id": drive_id, - "file_id": file_id - } + result = res.json() + return schemas.FileItem( + fileid=result.get("file_id"), + drive_id=result.get("drive_id"), + parent_fileid=result.get("parent_file_id"), + type="file", + name=result.get("name"), + path=f"{file_path.parent}/{result.get('name')}", + ) else: logger.warn("上传文件失败:无法获取上传地址!") return None diff --git a/app/helper/u115.py b/app/helper/u115.py index 62c03c9b..97fa15b6 100644 --- a/app/helper/u115.py +++ b/app/helper/u115.py @@ -233,14 +233,14 @@ class U115Helper(metaclass=Singleton): logger.error(f"移动115文件失败:{str(e)}") return False - def upload(self, parent_file_id: str, file_path: Path) -> Optional[dict]: + def upload(self, parent_file_id: str, file_path: Path) -> Optional[schemas.FileItem]: """ 上传文件 """ if not self.__init_cloud(): return None try: - ticket = self.cloud.storage().request_upload(dir_id=parent_file_id, file_path=file_path) + ticket = self.cloud.storage().request_upload(dir_id=parent_file_id, file_path=str(file_path)) if ticket is None: logger.warn(f"115请求上传出错") return None @@ -256,13 +256,23 @@ class U115Helper(metaclass=Singleton): ) por = bucket.put_object_from_file( key=ticket.object_key, - filename=file_path, + filename=str(file_path), headers=ticket.headers, ) result = por.resp.response.json() if result: - logger.info(f"115上传文件成功:{result}") - return result + fileitem = result.get('data') + logger.info(f"115上传文件成功:{fileitem}") + return schemas.FileItem( + fileid=fileitem.get('file_id'), + parent_fileid=parent_file_id, + type="file", + name=fileitem.get('file_name'), + path=f"{file_path / fileitem.get('file_name')}", + size=fileitem.get('file_size'), + extension=Path(fileitem.get('file_name')).suffix[1:], + pickcode=fileitem.get('pickcode') + ) else: logger.warn(f"115上传文件失败:{por.resp.response.text}") return None diff --git a/app/modules/douban/__init__.py b/app/modules/douban/__init__.py index d2be9f51..dbe09a89 100644 --- a/app/modules/douban/__init__.py +++ b/app/modules/douban/__init__.py @@ -765,7 +765,7 @@ class DoubanModule(_ModuleBase): logger.error(f"刮削文件 {file} 失败,原因:{str(e)}") logger.info(f"{path} 刮削完成") - def meta_nfo(self, mediainfo: MediaInfo, season: int = None, **kwargs) -> Optional[str]: + def metadata_nfo(self, mediainfo: MediaInfo, season: int = None, **kwargs) -> Optional[str]: """ 获取NFO文件内容文本 :param mediainfo: 媒体信息 diff --git a/app/modules/themoviedb/__init__.py b/app/modules/themoviedb/__init__.py index 409e3bdc..4ccaab2c 100644 --- a/app/modules/themoviedb/__init__.py +++ b/app/modules/themoviedb/__init__.py @@ -336,8 +336,8 @@ class TheMovieDbModule(_ModuleBase): force_img=force_img) logger.info(f"{path} 刮削完成") - def meta_nfo(self, meta: MetaBase, mediainfo: MediaInfo, - season: int = None, episode: int = None) -> Optional[str]: + def metadata_nfo(self, meta: MetaBase, mediainfo: MediaInfo, + season: int = None, episode: int = None) -> Optional[str]: """ 获取NFO文件内容文本 :param meta: 元数据 diff --git a/app/modules/themoviedb/scraper.py b/app/modules/themoviedb/scraper.py index bdc321c4..7984bd69 100644 --- a/app/modules/themoviedb/scraper.py +++ b/app/modules/themoviedb/scraper.py @@ -247,7 +247,8 @@ class TmdbScraper: :param file_path: 电影文件路径 """ # 开始生成XML - logger.info(f"正在生成电影NFO文件:{file_path.name}") + if file_path: + logger.info(f"正在生成电影NFO文件:{file_path.name}") doc = minidom.Document() root = DomUtils.add_node(doc, doc, "movie") # 公共部分 diff --git a/app/utils/system.py b/app/utils/system.py index c1ab0412..3266a79b 100644 --- a/app/utils/system.py +++ b/app/utils/system.py @@ -293,6 +293,25 @@ class SystemUtils: return dirs + @staticmethod + def list_sub_all(directory: Path) -> List[Path]: + """ + 列出当前目录下的所有子目录和文件(不递归) + """ + if not directory.exists(): + return [] + + if directory.is_file(): + return [] + + items = [] + + # 遍历目录 + for path in directory.iterdir(): + items.append(path) + + return items + @staticmethod def get_directory_size(path: Path) -> float: """