codfrm/MoviePilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source
This commit is contained in:
jxxghp
2023-11-18 21:54:10 +08:00
parent b868cdb25e 04339539d1
commit c336e62885
19 changed files with 253 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/api/endpoints/system.py
View File

@ -163,7 +163,8 @@ def latest_version(_: schemas.TokenPayload = Depends(verify_token)):
"""
查询Github所有Release版本
"""
version_res = RequestUtils().get_res(f"https://api.github.com/repos/jxxghp/MoviePilot/releases")
version_res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res(
f"https://api.github.com/repos/jxxghp/MoviePilot/releases")
if version_res:
ver_json = version_res.json()
if ver_json:

2
app/chain/search.py
View File

@ -213,7 +213,7 @@ class SearchChain(ChainBase):
continue
# 在副标题中判断是否存在标题与原语种标题
if torrent.description:
subtitle = torrent.description.split()
subtitle = re.split(r'[\s/|]+', torrent.description)
if (StringUtils.is_chinese(mediainfo.title)
and str(mediainfo.title) in subtitle) \
or (StringUtils.is_chinese(mediainfo.original_title)

2
app/chain/system.py
View File

@ -87,7 +87,7 @@ class SystemChain(ChainBase, metaclass=Singleton):
"""
获取最新版本
"""
version_res = RequestUtils(proxies=settings.PROXY).get_res(
version_res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res(
"https://api.github.com/repos/jxxghp/MoviePilot/releases/latest")
if version_res:
ver_json = version_res.json()

4
app/chain/torrents.py
View File

@ -60,7 +60,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
else:
return self.load_cache(self._rss_file) or {}
@cached(cache=TTLCache(maxsize=128, ttl=600))
@cached(cache=TTLCache(maxsize=128, ttl=595))
def browse(self, domain: str) -> List[TorrentInfo]:
"""
浏览站点首页内容，返回种子清单，TTL缓存10分钟
@ -73,7 +73,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
return []
return self.refresh_torrents(site=site)
@cached(cache=TTLCache(maxsize=128, ttl=300))
@cached(cache=TTLCache(maxsize=128, ttl=295))
def rss(self, domain: str) -> List[TorrentInfo]:
"""
获取站点RSS内容，返回种子清单，TTL缓存5分钟

15
app/core/config.py
View File

@ -212,6 +212,10 @@ class Settings(BaseSettings):
BIG_MEMORY_MODE: bool = False
# 插件市场仓库地址，多个地址使用,分隔，地址以/结尾
PLUGIN_MARKET: str = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Plugins/main/"
# Github token，提高请求api限流阈值 ghp_****
GITHUB_TOKEN: str = None
# 自动检查和更新站点资源包（站点索引、认证等）
AUTO_UPDATE_RESOURCE: bool = True
@property
def INNER_CONFIG_PATH(self):
@ -321,6 +325,17 @@ class Settings(BaseSettings):
return Path(self.DOWNLOAD_ANIME_PATH)
return self.SAVE_TV_PATH
@property
def GITHUB_HEADERS(self):
"""
Github请求头
"""
if self.GITHUB_TOKEN:
return {
"Authorization": f"Bearer {self.GITHUB_TOKEN}"
}
return {}
def __init__(self, **kwargs):
super().__init__(**kwargs)
with self.CONFIG_PATH as p:

6
app/core/meta/metavideo.py
View File

@ -34,6 +34,7 @@ class MetaVideo(MetaBase):
_name_no_begin_re = r"^\[.+?]"
_name_no_chinese_re = r".*版|.*字幕"
_name_se_words = ['共', '第', '季', '集', '话', '話', '期']
_name_movie_words = ['剧场版', '劇場版', '电影版', '電影版']
_name_nostring_re = r"^PTS|^JADE|^AOD|^CHC|^[A-Z]{1,4}TV[\-0-9UVHDK]*" \
r"|HBO$|\s+HBO|\d{1,2}th|\d{1,2}bit|NETFLIX|AMAZON|IMAX|^3D|\s+3D|^BBC\s+|\s+BBC|BBC$|DISNEY\+?|XXX|\s+DC$" \
r"|[第\s共]+[0-9一二三四五六七八九十\-\s]+季" \
@ -182,8 +183,9 @@ class MetaVideo(MetaBase):
if not self.cn_name:
self.cn_name = token
elif not self._stop_cnname_flag:
if not re.search("%s" % self._name_no_chinese_re, token, flags=re.IGNORECASE) \
and not re.search("%s" % self._name_se_words, token, flags=re.IGNORECASE):
if re.search("%s" % self._name_movie_words, token, flags=re.IGNORECASE) \
or (not re.search("%s" % self._name_no_chinese_re, token, flags=re.IGNORECASE)
and not re.search("%s" % self._name_se_words, token, flags=re.IGNORECASE)):
self.cn_name = "%s %s" % (self.cn_name, token)
self._stop_cnname_flag = True
else:

3
app/core/plugin.py
View File

@ -12,6 +12,7 @@ from app.schemas.types import SystemConfigKey
from app.utils.object import ObjectUtils
from app.utils.singleton import Singleton
from app.utils.string import StringUtils
from app.utils.system import SystemUtils
class PluginManager(metaclass=Singleton):
@ -105,6 +106,8 @@ class PluginManager(metaclass=Singleton):
"""
安装本地不存在的在线插件
"""
if SystemUtils.is_frozen():
return
logger.info("开始安装在线插件...")
# 已安装插件
install_plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins) or []

8
app/helper/plugin.py
View File

@ -24,7 +24,8 @@ class PluginHelper(metaclass=Singleton):
"""
if not repo_url:
return {}
res = RequestUtils(proxies=settings.PROXY, timeout=10).get_res(f"{repo_url}package.json")
res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS,
timeout=10).get_res(f"{repo_url}package.json")
if res:
return json.loads(res.text)
return {}
@ -49,7 +50,7 @@ class PluginHelper(metaclass=Singleton):
获取插件的文件列表
"""
file_api = f"https://api.github.com/repos/{user}/{repo}/contents/plugins/{_p.lower()}"
r = RequestUtils(proxies=settings.PROXY).get_res(file_api)
r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, timeout=10).get_res(file_api)
if not r or r.status_code != 200:
return None, f"连接仓库失败：{r.status_code} - {r.reason}"
ret = r.json()
@ -66,7 +67,8 @@ class PluginHelper(metaclass=Singleton):
for item in _l:
if item.get("download_url"):
# 下载插件文件
res = RequestUtils(proxies=settings.PROXY).get_res(item["download_url"])
res = RequestUtils(proxies=settings.PROXY,
headers=settings.GITHUB_HEADERS, timeout=30).get_res(item["download_url"])
if not res:
return False, f"文件 {item.get('name')} 下载失败！"
elif res.status_code != 200:

103
app/helper/resource.py Normal file
View File

@ -0,0 +1,103 @@
import json
from pathlib import Path
from app.core.config import settings
from app.helper.sites import SitesHelper
from app.log import logger
from app.utils.http import RequestUtils
from app.utils.singleton import Singleton
from app.utils.string import StringUtils
from app.utils.system import SystemUtils
class ResourceHelper(metaclass=Singleton):
"""
检测和更新资源包
"""
# 资源包的git仓库地址
_repo = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/package.json"
_files_api = f"https://api.github.com/repos/jxxghp/MoviePilot-Resources/contents/resources"
_base_dir: Path = settings.ROOT_PATH
def __init__(self):
self.siteshelper = SitesHelper()
self.check()
def check(self):
"""
检测是否有更新，如有则下载安装
"""
if not settings.AUTO_UPDATE_RESOURCE:
return
if SystemUtils.is_frozen():
return
logger.info("开始检测资源包版本...")
res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, timeout=10).get_res(self._repo)
if res:
resource_info = json.loads(res.text)
else:
logger.warn("无法连接资源包仓库！")
return
online_version = resource_info.get("version")
if online_version:
logger.info(f"最新资源包版本：v{online_version}")
# 需要更新的资源包
need_updates = {}
# 资源明细
resources: dict = resource_info.get("resources") or {}
for rname, resource in resources.items():
rtype = resource.get("type")
platform = resource.get("platform")
target = resource.get("target")
version = resource.get("version")
# 判断平台
if platform and platform != SystemUtils.platform:
continue
# 判断本地是否存在
local_path = self._base_dir / target
if not local_path.exists():
continue
# 判断版本号
if rtype == "auth":
# 站点认证资源
local_version = self.siteshelper.auth_version
elif rtype == "sites":
# 站点索引资源
local_version = self.siteshelper.indexer_version
else:
continue
if StringUtils.compare_version(version, local_version) > 0:
logger.info(f"{rname} 资源包有更新，最新版本：v{version}")
else:
continue
# 需要安装
need_updates[rname] = target
if need_updates:
# 下载文件信息列表
r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS,
timeout=10).get_res(self._files_api)
if not r or r.status_code != 200:
return None, f"连接仓库失败：{r.status_code} - {r.reason}"
files_info = r.json()
for item in files_info:
save_path = need_updates.get(item.get("name"))
if not save_path:
continue
if item.get("download_url"):
# 下载插件文件
res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS,
timeout=60).get_res(item["download_url"])
if not res:
logger.error(f"文件 {item.get('name')} 下载失败！")
elif res.status_code != 200:
logger.error(f"下载文件 {item.get('name')} 失败：{res.status_code} - {res.reason}")
# 创建插件文件夹
file_path = self._base_dir / save_path / item.get("name")
if not file_path.parent.exists():
file_path.parent.mkdir(parents=True, exist_ok=True)
# 写入文件
file_path.write_bytes(res.content)
logger.info("资源包更新完成，开始重启服务...")
SystemUtils.restart()
else:
logger.info("所有资源已最新，无需更新")

25
app/helper/rss.py
View File

@ -1,11 +1,14 @@
import re
import xml.dom.minidom
from typing import List, Tuple, Union
from urllib.parse import urljoin
import chardet
from lxml import etree
from app.core.config import settings
from app.helper.browser import PlaywrightHelper
from app.log import logger
from app.utils.dom import DomUtils
from app.utils.http import RequestUtils
from app.utils.string import StringUtils
@ -240,8 +243,28 @@ class RssHelper:
print(str(err))
return []
if ret:
ret_xml = ret.text
ret_xml = ""
try:
# 使用chardet检测字符编码
raw_data = ret.content
if raw_data:
try:
result = chardet.detect(raw_data)
encoding = result['encoding']
# 解码为字符串
ret_xml = raw_data.decode(encoding)
except Exception as e:
logger.debug(f"chardet解码失败：{str(e)}")
# 探测utf-8解码
match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text)
if match:
encoding = match.group(1)
if encoding:
ret_xml = raw_data.decode(encoding)
else:
ret.encoding = ret.apparent_encoding
if not ret_xml:
ret_xml = ret.text
# 解析XML
dom_tree = xml.dom.minidom.parseString(ret_xml)
rootNode = dom_tree.documentElement

4
app/main.py
View File

@ -22,10 +22,12 @@ from app.core.plugin import PluginManager
from app.db.init import init_db, update_db
from app.helper.thread import ThreadHelper
from app.helper.display import DisplayHelper
from app.helper.resource import ResourceHelper
from app.helper.sites import SitesHelper
from app.scheduler import Scheduler
from app.command import Command
# App
App = FastAPI(title=settings.PROJECT_NAME,
openapi_url=f"{settings.API_V1_STR}/openapi.json")
@ -169,6 +171,8 @@ def start_module():
DisplayHelper()
# 站点管理
SitesHelper()
# 资源包检测
ResourceHelper()
# 加载模块
ModuleManager()
# 加载插件

8
app/modules/filetransfer/__init__.py
View File

@ -416,14 +416,6 @@ class FileTransferModule(_ModuleBase):
rename_dict=self.__get_naming_dict(meta=in_meta,
mediainfo=mediainfo)
).parent
# 目录已存在时不处理
if new_path.exists():
logger.warn(f"目标目录已存在：{new_path}")
return TransferInfo(success=False,
message=f"目标目录已存在：{new_path}",
path=in_path,
target_path=new_path,
is_bluray=bluray_flag)
# 转移蓝光原盘
retcode = self.__transfer_dir(file_path=in_path,
new_path=new_path,

11
app/modules/telegram/telegram.py
View File

@ -12,6 +12,7 @@ from app.core.config import settings
from app.core.context import MediaInfo, Context
from app.core.metainfo import MetaInfo
from app.log import logger
from app.utils.common import retry
from app.utils.http import RequestUtils
from app.utils.singleton import Singleton
from app.utils.string import StringUtils
@ -174,6 +175,7 @@ class Telegram(metaclass=Singleton):
logger.error(f"发送消息失败：{msg_e}")
return False
@retry(Exception, logger=logger)
def __send_request(self, userid: str = None, image="", caption="") -> bool:
"""
向Telegram发送报文
@ -181,7 +183,9 @@ class Telegram(metaclass=Singleton):
if image:
req = RequestUtils(proxies=settings.PROXY).get_res(image)
if req and req.content:
if req is None:
raise Exception("获取图片失败")
if req.content:
image_file = Path(settings.TEMP_PATH) / Path(image).name
image_file.write_bytes(req.content)
photo = InputFile(image_file)
@ -189,12 +193,15 @@ class Telegram(metaclass=Singleton):
photo=photo,
caption=caption,
parse_mode="Markdown")
if ret is None:
raise Exception("发送图片消息失败")
if ret:
return True
ret = self._bot.send_message(chat_id=userid or self._telegram_chat_id,
text=caption,
parse_mode="Markdown")
if ret is None:
raise Exception("发送文本消息失败")
return True if ret else False
def register_commands(self, commands: Dict[str, dict]):