From 6dcc979fd5601eaa403e79f6e784aa37663d651e Mon Sep 17 00:00:00 2001 From: jxxghp Date: Tue, 14 Nov 2023 08:19:57 +0800 Subject: [PATCH 01/20] fix #1072 fix #1118 --- app/core/plugin.py | 3 +++ app/helper/rss.py | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/app/core/plugin.py b/app/core/plugin.py index a2475d94..872ecde3 100644 --- a/app/core/plugin.py +++ b/app/core/plugin.py @@ -12,6 +12,7 @@ from app.schemas.types import SystemConfigKey from app.utils.object import ObjectUtils from app.utils.singleton import Singleton from app.utils.string import StringUtils +from app.utils.system import SystemUtils class PluginManager(metaclass=Singleton): @@ -105,6 +106,8 @@ class PluginManager(metaclass=Singleton): """ 安装本地不存在的在线插件 """ + if SystemUtils.is_frozen(): + return logger.info("开始安装在线插件...") # 已安装插件 install_plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins) or [] diff --git a/app/helper/rss.py b/app/helper/rss.py index 957e5314..cd328220 100644 --- a/app/helper/rss.py +++ b/app/helper/rss.py @@ -1,11 +1,14 @@ +import re import xml.dom.minidom from typing import List, Tuple, Union from urllib.parse import urljoin +import chardet from lxml import etree from app.core.config import settings from app.helper.browser import PlaywrightHelper +from app.log import logger from app.utils.dom import DomUtils from app.utils.http import RequestUtils from app.utils.string import StringUtils @@ -240,8 +243,28 @@ class RssHelper: print(str(err)) return [] if ret: - ret_xml = ret.text + ret_xml = "" try: + # 使用chardet检测字符编码 + raw_data = ret.content + if raw_data: + try: + result = chardet.detect(raw_data) + encoding = result['encoding'] + # 解码为字符串 + ret_xml = raw_data.decode(encoding) + except Exception as e: + logger.debug(f"chardet解码失败：{str(e)}") + # 探测utf-8解码 + match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text) + if match: + encoding = match.group(1) + if encoding: + ret_xml = raw_data.decode(encoding) + else: + ret.encoding = ret.apparent_encoding + if not ret_xml: + ret_xml = ret.text # 解析XML dom_tree = xml.dom.minidom.parseString(ret_xml) rootNode = dom_tree.documentElement From 793a7460c6615cd392dee9f4154e5540d24d833f Mon Sep 17 00:00:00 2001 From: thsrite Date: Tue, 14 Nov 2023 14:10:48 +0800 Subject: [PATCH 02/20] =?UTF-8?q?fix=20=E5=8A=A8=E6=BC=AB=E7=94=B5?= =?UTF-8?q?=E5=BD=B1=20=E5=89=A7=E5=9C=BA=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/meta/metavideo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/core/meta/metavideo.py b/app/core/meta/metavideo.py index f95bc25a..f9dc40fe 100644 --- a/app/core/meta/metavideo.py +++ b/app/core/meta/metavideo.py @@ -32,7 +32,7 @@ class MetaVideo(MetaBase): _effect_re = r"^REMUX$|^UHD$|^SDR$|^HDR\d*$|^DOLBY$|^DOVI$|^DV$|^3D$|^REPACK$" _resources_type_re = r"%s|%s" % (_source_re, _effect_re) _name_no_begin_re = r"^\[.+?]" - _name_no_chinese_re = r".*版|.*字幕" + _name_no_chinese_re = r".*字幕" _name_se_words = ['共', '第', '季', '集', '话', '話', '期'] _name_nostring_re = r"^PTS|^JADE|^AOD|^CHC|^[A-Z]{1,4}TV[\-0-9UVHDK]*" \ r"|HBO$|\s+HBO|\d{1,2}th|\d{1,2}bit|NETFLIX|AMAZON|IMAX|^3D|\s+3D|^BBC\s+|\s+BBC|BBC$|DISNEY\+?|XXX|\s+DC$" \ From f514a5a416feef99173481bac40876c6a6e6db27 Mon Sep 17 00:00:00 2001 From: thsrite Date: Tue, 14 Nov 2023 14:17:14 +0800 Subject: [PATCH 03/20] fix 793a7460 --- app/core/meta/metavideo.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/core/meta/metavideo.py b/app/core/meta/metavideo.py index f9dc40fe..8fc50f90 100644 --- a/app/core/meta/metavideo.py +++ b/app/core/meta/metavideo.py @@ -32,8 +32,9 @@ class MetaVideo(MetaBase): _effect_re = r"^REMUX$|^UHD$|^SDR$|^HDR\d*$|^DOLBY$|^DOVI$|^DV$|^3D$|^REPACK$" _resources_type_re = r"%s|%s" % (_source_re, _effect_re) _name_no_begin_re = r"^\[.+?]" - _name_no_chinese_re = r".*字幕" + _name_no_chinese_re = r".*版|.*字幕" _name_se_words = ['共', '第', '季', '集', '话', '話', '期'] + _name_movie_words = ['剧场版', '劇場版', '电影版', '電影版'] _name_nostring_re = r"^PTS|^JADE|^AOD|^CHC|^[A-Z]{1,4}TV[\-0-9UVHDK]*" \ r"|HBO$|\s+HBO|\d{1,2}th|\d{1,2}bit|NETFLIX|AMAZON|IMAX|^3D|\s+3D|^BBC\s+|\s+BBC|BBC$|DISNEY\+?|XXX|\s+DC$" \ r"|[第\s共]+[0-9一二三四五六七八九十\-\s]+季" \ @@ -182,8 +183,9 @@ class MetaVideo(MetaBase): if not self.cn_name: self.cn_name = token elif not self._stop_cnname_flag: - if not re.search("%s" % self._name_no_chinese_re, token, flags=re.IGNORECASE) \ - and not re.search("%s" % self._name_se_words, token, flags=re.IGNORECASE): + if re.search("%s" % self._name_movie_words, token, flags=re.IGNORECASE) \ + or (not re.search("%s" % self._name_no_chinese_re, token, flags=re.IGNORECASE) + and not re.search("%s" % self._name_se_words, token, flags=re.IGNORECASE)): self.cn_name = "%s %s" % (self.cn_name, token) self._stop_cnname_flag = True else: From 0aad809c82c0506ecf153d320a70370e9b64a19e Mon Sep 17 00:00:00 2001 From: thsrite Date: Tue, 14 Nov 2023 16:12:02 +0800 Subject: [PATCH 04/20] =?UTF-8?q?feat=20=E5=A2=9E=E5=8A=A0GITHUB=5FTOKEN?= =?UTF-8?q?=EF=BC=8C=E6=8F=90=E9=AB=98API=E9=99=90=E6=B5=81=E9=98=88?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + app/api/endpoints/system.py | 8 +++++++- app/chain/system.py | 7 ++++++- app/core/config.py | 2 ++ app/helper/plugin.py | 7 ++++++- update | 9 +++++++-- 6 files changed, 29 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 03c580fe..e7fb64f1 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ MoviePilot需要配套下载器和媒体服务器配合使用。 --- - **OCR_HOST：** OCR识别服务器地址，格式：`http(s)://ip:port`，用于识别站点验证码实现自动登录获取Cookie等，不配置默认使用内建服务器`https://movie-pilot.org`，可使用 [这个镜像](https://hub.docker.com/r/jxxghp/moviepilot-ocr) 自行搭建。 - **PLUGIN_MARKET：** 插件市场仓库地址，多个地址使用`,`分隔，保留最后的/，默认为官方插件仓库：`https://raw.githubusercontent.com/jxxghp/MoviePilot-Plugins/main/`。 +- **GITHUB_TOKEN：** Github token，提高请求api限流阈值 ghp_****（仅支持环境变量配置） --- - **❗MESSAGER：** 消息通知渠道，支持 `telegram`/`wechat`/`slack`/`synologychat`，开启多个渠道时使用`,`分隔。同时还需要配置对应渠道的环境变量，非对应渠道的变量可删除，推荐使用`telegram` diff --git a/app/api/endpoints/system.py b/app/api/endpoints/system.py index 792978fa..36c48049 100644 --- a/app/api/endpoints/system.py +++ b/app/api/endpoints/system.py @@ -163,7 +163,13 @@ def latest_version(_: schemas.TokenPayload = Depends(verify_token)): """ 查询Github所有Release版本 """ - version_res = RequestUtils().get_res(f"https://api.github.com/repos/jxxghp/MoviePilot/releases") + headers = {} + if settings.GITHUB_TOKEN: + headers = { + "Authorization": f"Bearer {settings.GITHUB_TOKEN}" + } + version_res = RequestUtils(proxies=settings.PROXY, headers=headers).get_res( + f"https://api.github.com/repos/jxxghp/MoviePilot/releases") if version_res: ver_json = version_res.json() if ver_json: diff --git a/app/chain/system.py b/app/chain/system.py index 15162bea..512c7b88 100644 --- a/app/chain/system.py +++ b/app/chain/system.py @@ -87,7 +87,12 @@ class SystemChain(ChainBase, metaclass=Singleton): """ 获取最新版本 """ - version_res = RequestUtils(proxies=settings.PROXY).get_res( + headers = {} + if settings.GITHUB_TOKEN: + headers = { + "Authorization": f"Bearer {settings.GITHUB_TOKEN}" + } + version_res = RequestUtils(proxies=settings.PROXY, headers=headers).get_res( "https://api.github.com/repos/jxxghp/MoviePilot/releases/latest") if version_res: ver_json = version_res.json() diff --git a/app/core/config.py b/app/core/config.py index 71202706..d3d6dbff 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -212,6 +212,8 @@ class Settings(BaseSettings): BIG_MEMORY_MODE: bool = False # 插件市场仓库地址，多个地址使用,分隔，地址以/结尾 PLUGIN_MARKET: str = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Plugins/main/" + # Github token，提高请求api限流阈值 ghp_**** + GITHUB_TOKEN: str = None @property def INNER_CONFIG_PATH(self): diff --git a/app/helper/plugin.py b/app/helper/plugin.py index 4852123d..595f399c 100644 --- a/app/helper/plugin.py +++ b/app/helper/plugin.py @@ -49,7 +49,12 @@ class PluginHelper(metaclass=Singleton): 获取插件的文件列表 """ file_api = f"https://api.github.com/repos/{user}/{repo}/contents/plugins/{_p.lower()}" - r = RequestUtils(proxies=settings.PROXY).get_res(file_api) + headers = {} + if settings.GITHUB_TOKEN: + headers = { + "Authorization": f"Bearer {settings.GITHUB_TOKEN}" + } + r = RequestUtils(proxies=settings.PROXY, headers=headers).get_res(file_api) if not r or r.status_code != 200: return None, f"连接仓库失败：{r.status_code} - {r.reason}" ret = r.json() diff --git a/update b/update index d8580c89..882ee3fd 100644 --- a/update +++ b/update @@ -30,7 +30,7 @@ install_backend_and_download_resources() { download_and_unzip "https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip" "Resources" if [ $? -eq 0 ]; then echo "资源包下载成功" - frontend_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" | jq -r .tag_name) + frontend_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" ${CURL_HEADERS} | jq -r .tag_name) if [[ "${frontend_version}" == *v* ]]; then download_and_unzip "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${frontend_version}/dist.zip" "dist" if [ $? -eq 0 ]; then @@ -83,6 +83,11 @@ if [[ "${MOVIEPILOT_AUTO_UPDATE}" = "true" ]] || [[ "${MOVIEPILOT_AUTO_UPDATE}" CURL_OPTIONS="-sL" echo "不使用代理更新程序" fi + if [ -n "${GITHUB_TOKEN}" ]; then + CURL_HEADERS="--header 'Authorization: Bearer ${GITHUB_TOKEN}'" + else + CURL_HEADERS="" + fi if [ "${MOVIEPILOT_AUTO_UPDATE}" = "dev" ]; then echo "Dev 更新模式" install_backend_and_download_resources "heads/main.zip" @@ -92,7 +97,7 @@ if [[ "${MOVIEPILOT_AUTO_UPDATE}" = "true" ]] || [[ "${MOVIEPILOT_AUTO_UPDATE}" if [[ "${old_version}" == *APP_VERSION* ]]; then current_version=v$(echo ${old_version} | sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp") echo "当前版本号：${current_version}" - new_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot/releases/latest" | jq -r .tag_name) + new_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot/releases/latest" ${CURL_HEADERS} | jq -r .tag_name) if [[ "${new_version}" == *v* ]]; then release_version=${new_version} echo "最新版本号：${release_version}" From 4c69bb6c48a0c39d0597a3f661ec5c1e57194308 Mon Sep 17 00:00:00 2001 From: thsrite Date: Tue, 14 Nov 2023 16:22:04 +0800 Subject: [PATCH 05/20] fix --- update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update b/update index 882ee3fd..35b34d24 100644 --- a/update +++ b/update @@ -5,7 +5,7 @@ download_and_unzip() { url="$1" target_dir="$2" echo "正在下载 ${url}..." - curl ${CURL_OPTIONS} "$url" | busybox unzip -d /tmp - + curl ${CURL_OPTIONS} "$url" ${CURL_HEADERS} | busybox unzip -d /tmp - if [ $? -eq 0 ]; then if [ -e /tmp/MoviePilot-* ]; then mv /tmp/MoviePilot-* /tmp/${target_dir} From 42966c25371c9d5d402a470a79e6cf826e80d51f Mon Sep 17 00:00:00 2001 From: jxxghp Date: Tue, 14 Nov 2023 17:35:31 +0800 Subject: [PATCH 06/20] fix #1122 --- app/api/endpoints/system.py | 7 +------ app/chain/system.py | 7 +------ app/core/config.py | 11 +++++++++++ app/helper/plugin.py | 7 +------ 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/app/api/endpoints/system.py b/app/api/endpoints/system.py index 36c48049..c5dedacb 100644 --- a/app/api/endpoints/system.py +++ b/app/api/endpoints/system.py @@ -163,12 +163,7 @@ def latest_version(_: schemas.TokenPayload = Depends(verify_token)): """ 查询Github所有Release版本 """ - headers = {} - if settings.GITHUB_TOKEN: - headers = { - "Authorization": f"Bearer {settings.GITHUB_TOKEN}" - } - version_res = RequestUtils(proxies=settings.PROXY, headers=headers).get_res( + version_res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res( f"https://api.github.com/repos/jxxghp/MoviePilot/releases") if version_res: ver_json = version_res.json() diff --git a/app/chain/system.py b/app/chain/system.py index 512c7b88..e47c7be6 100644 --- a/app/chain/system.py +++ b/app/chain/system.py @@ -87,12 +87,7 @@ class SystemChain(ChainBase, metaclass=Singleton): """ 获取最新版本 """ - headers = {} - if settings.GITHUB_TOKEN: - headers = { - "Authorization": f"Bearer {settings.GITHUB_TOKEN}" - } - version_res = RequestUtils(proxies=settings.PROXY, headers=headers).get_res( + version_res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res( "https://api.github.com/repos/jxxghp/MoviePilot/releases/latest") if version_res: ver_json = version_res.json() diff --git a/app/core/config.py b/app/core/config.py index d3d6dbff..f738248f 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -323,6 +323,17 @@ class Settings(BaseSettings): return Path(self.DOWNLOAD_ANIME_PATH) return self.SAVE_TV_PATH + @property + def GITHUB_HEADERS(self): + """ + Github请求头 + """ + if self.GITHUB_TOKEN: + return { + "Authorization": f"Bearer {self.GITHUB_TOKEN}" + } + return {} + def __init__(self, **kwargs): super().__init__(**kwargs) with self.CONFIG_PATH as p: diff --git a/app/helper/plugin.py b/app/helper/plugin.py index 595f399c..de60ddca 100644 --- a/app/helper/plugin.py +++ b/app/helper/plugin.py @@ -49,12 +49,7 @@ class PluginHelper(metaclass=Singleton): 获取插件的文件列表 """ file_api = f"https://api.github.com/repos/{user}/{repo}/contents/plugins/{_p.lower()}" - headers = {} - if settings.GITHUB_TOKEN: - headers = { - "Authorization": f"Bearer {settings.GITHUB_TOKEN}" - } - r = RequestUtils(proxies=settings.PROXY, headers=headers).get_res(file_api) + r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res(file_api) if not r or r.status_code != 200: return None, f"连接仓库失败：{r.status_code} - {r.reason}" ret = r.json() From d4dc388d3f25eb306b7656139dbca97838bd4c34 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Tue, 14 Nov 2023 17:54:41 +0800 Subject: [PATCH 07/20] fix search --- app/chain/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/chain/search.py b/app/chain/search.py index bde73d29..7b790633 100644 --- a/app/chain/search.py +++ b/app/chain/search.py @@ -213,7 +213,7 @@ class SearchChain(ChainBase): continue # 在副标题中判断是否存在标题与原语种标题 if torrent.description: - subtitle = torrent.description.split() + subtitle = re.split(r'[\s/|]+', torrent.description) if (StringUtils.is_chinese(mediainfo.title) and str(mediainfo.title) in subtitle) \ or (StringUtils.is_chinese(mediainfo.original_title) From 5221fc4f6a20524d163da734b89b5a6078e54e3f Mon Sep 17 00:00:00 2001 From: jxxghp Date: Wed, 15 Nov 2023 08:08:46 +0800 Subject: [PATCH 08/20] fix #1125 --- app/modules/filetransfer/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/modules/filetransfer/__init__.py b/app/modules/filetransfer/__init__.py index df377ffd..9d41791f 100644 --- a/app/modules/filetransfer/__init__.py +++ b/app/modules/filetransfer/__init__.py @@ -416,14 +416,6 @@ class FileTransferModule(_ModuleBase): rename_dict=self.__get_naming_dict(meta=in_meta, mediainfo=mediainfo) ).parent - # 目录已存在时不处理 - if new_path.exists(): - logger.warn(f"目标目录已存在：{new_path}") - return TransferInfo(success=False, - message=f"目标目录已存在：{new_path}", - path=in_path, - target_path=new_path, - is_bluray=bluray_flag) # 转移蓝光原盘 retcode = self.__transfer_dir(file_path=in_path, new_path=new_path, From ce13987748aadc4e3fd3320be853911cba3b75ad Mon Sep 17 00:00:00 2001 From: jxxghp Date: Wed, 15 Nov 2023 08:20:57 +0800 Subject: [PATCH 09/20] =?UTF-8?q?-=20=E4=BF=AE=E5=A4=8DWindows=E6=89=93?= =?UTF-8?q?=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- version.py | 2 +- windows.spec | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/version.py b/version.py index a77dd65d..545667de 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -APP_VERSION = 'v1.4.2' +APP_VERSION = 'v1.4.2-1' diff --git a/windows.spec b/windows.spec index d376dcbe..ae88afc2 100644 --- a/windows.spec +++ b/windows.spec @@ -82,6 +82,7 @@ exe = EXE( collect_pkg_data('zhconv'), collect_pkg_data('cn2an'), collect_pkg_data('database', include_py_files=True), + collect_pkg_data('app.helper'), [], name='MoviePilot', debug=False, From c6fa3b9d25d5aec9c5090430960339c1be49cb28 Mon Sep 17 00:00:00 2001 From: honue Date: Thu, 16 Nov 2023 13:33:56 +0800 Subject: [PATCH 10/20] fix torrent_cache ttl --- app/chain/torrents.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/chain/torrents.py b/app/chain/torrents.py index 760b4311..0b914cfc 100644 --- a/app/chain/torrents.py +++ b/app/chain/torrents.py @@ -60,7 +60,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton): else: return self.load_cache(self._rss_file) or {} - @cached(cache=TTLCache(maxsize=128, ttl=600)) + @cached(cache=TTLCache(maxsize=128, ttl=595)) def browse(self, domain: str) -> List[TorrentInfo]: """ 浏览站点首页内容，返回种子清单，TTL缓存10分钟 @@ -73,7 +73,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton): return [] return self.refresh_torrents(site=site) - @cached(cache=TTLCache(maxsize=128, ttl=300)) + @cached(cache=TTLCache(maxsize=128, ttl=295)) def rss(self, domain: str) -> List[TorrentInfo]: """ 获取站点RSS内容，返回种子清单，TTL缓存5分钟 From 9344b2a324bfc3e615ede2231a81afe1547e7c31 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 16 Nov 2023 17:27:58 +0800 Subject: [PATCH 11/20] add AGSVPT --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e7fb64f1..8fce8051 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ MoviePilot需要配套下载器和媒体服务器配合使用。 `AUTH_SITE`支持配置多个认证站点，使用`,`分隔，如：`iyuu,hhclub`，会依次执行认证操作，直到有一个站点认证成功。 -- **❗AUTH_SITE：** 认证站点，认证资源`v1.0.1`支持`iyuu`/`hhclub`/`audiences`/`hddolby`/`zmpt`/`freefarm`/`hdfans`/`wintersakura`/`leaves`/`1ptba`/`icc2022`/`ptlsp`/`xingtan`/`ptvicomo` +- **❗AUTH_SITE：** 认证站点，认证资源`v1.0.2`支持`iyuu`/`hhclub`/`audiences`/`hddolby`/`zmpt`/`freefarm`/`hdfans`/`wintersakura`/`leaves`/`1ptba`/`icc2022`/`ptlsp`/`xingtan`/`ptvicomo`/`agsvpt` | 站点 | 参数 | |:------------:|:-----------------------------------------------------:| @@ -209,6 +209,7 @@ MoviePilot需要配套下载器和媒体服务器配合使用。 | ptlsp | `PTLSP_UID`：用户ID
`PTLSP_PASSKEY`：密钥 | | xingtan | `XINGTAN_UID`：用户ID
`XINGTAN_PASSKEY`：密钥 | | ptvicomo | `PTVICOMO_UID`：用户ID
`PTVICOMO_PASSKEY`：密钥 | +| agsvpt | `AGSVPT_UID`：用户ID
`AGSVPT_PASSKEY`：密钥 | ### 2. **进阶配置** From 47ddfec30ea822133840211fc0592cf403d83d91 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 16 Nov 2023 20:57:41 +0800 Subject: [PATCH 12/20] =?UTF-8?q?feat=20=E8=B5=84=E6=BA=90=E5=8C=85?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +- app/core/config.py | 2 + app/helper/plugin.py | 8 ++-- app/helper/resource.py | 101 +++++++++++++++++++++++++++++++++++++++++ app/main.py | 4 ++ config/app.env | 2 + 6 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 app/helper/resource.py diff --git a/README.md b/README.md index 8fce8051..58bb3ab5 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,8 @@ MoviePilot需要配套下载器和媒体服务器配合使用。 - **PGID**：运行程序用户的`gid`，默认`0`（仅支持环境变量配置） - **UMASK**：掩码权限，默认`000`，可以考虑设置为`022`（仅支持环境变量配置） - **PROXY_HOST：** 网络代理，访问themoviedb或者重启更新需要使用代理访问，格式为`http(s)://ip:port`、`socks5://user:pass@host:port`（仅支持环境变量配置） -- **MOVIEPILOT_AUTO_UPDATE**：重启更新，`true`/`release`/`dev`/`false`，默认`release` **注意：如果出现网络问题可以配置`PROXY_HOST`**（仅支持环境变量配置） +- **MOVIEPILOT_AUTO_UPDATE：** 重启时自动更新，`true`/`release`/`dev`/`false`，默认`release`，需要能正常连接Github **注意：如果出现网络问题可以配置`PROXY_HOST`**（仅支持环境变量配置） +- **AUTO_UPDATE_RESOURCE**：启动时自动检测和更新资源包（站点索引及认证等），`true`/`false`，默认`true`，需要能正常连接Github，仅支持Docker --- - **❗SUPERUSER：** 超级管理员用户名，默认`admin`，安装后使用该用户登录后台管理界面 - **❗SUPERUSER_PASSWORD：** 超级管理员初始密码，默认`password`，建议修改为复杂密码 diff --git a/app/core/config.py b/app/core/config.py index f738248f..7dbbf97e 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -214,6 +214,8 @@ class Settings(BaseSettings): PLUGIN_MARKET: str = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Plugins/main/" # Github token，提高请求api限流阈值 ghp_**** GITHUB_TOKEN: str = None + # 自动检查和更新站点资源包（站点索引、认证等） + AUTO_UPDATE_RESOURCE: bool = True @property def INNER_CONFIG_PATH(self): diff --git a/app/helper/plugin.py b/app/helper/plugin.py index de60ddca..c518a82d 100644 --- a/app/helper/plugin.py +++ b/app/helper/plugin.py @@ -24,7 +24,8 @@ class PluginHelper(metaclass=Singleton): """ if not repo_url: return {} - res = RequestUtils(proxies=settings.PROXY, timeout=10).get_res(f"{repo_url}package.json") + res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, + timeout=10).get_res(f"{repo_url}package.json") if res: return json.loads(res.text) return {} @@ -49,7 +50,7 @@ class PluginHelper(metaclass=Singleton): 获取插件的文件列表 """ file_api = f"https://api.github.com/repos/{user}/{repo}/contents/plugins/{_p.lower()}" - r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res(file_api) + r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, timeout=10).get_res(file_api) if not r or r.status_code != 200: return None, f"连接仓库失败：{r.status_code} - {r.reason}" ret = r.json() @@ -66,7 +67,8 @@ class PluginHelper(metaclass=Singleton): for item in _l: if item.get("download_url"): # 下载插件文件 - res = RequestUtils(proxies=settings.PROXY).get_res(item["download_url"]) + res = RequestUtils(proxies=settings.PROXY, + headers=settings.GITHUB_HEADERS, timeout=30).get_res(item["download_url"]) if not res: return False, f"文件 {item.get('name')} 下载失败！" elif res.status_code != 200: diff --git a/app/helper/resource.py b/app/helper/resource.py new file mode 100644 index 00000000..d1223edd --- /dev/null +++ b/app/helper/resource.py @@ -0,0 +1,101 @@ +import json +from pathlib import Path + +from app.core.config import settings +from app.helper.sites import SitesHelper +from app.log import logger +from app.utils.http import RequestUtils +from app.utils.singleton import Singleton +from app.utils.string import StringUtils +from app.utils.system import SystemUtils + + +class ResourceHelper(metaclass=Singleton): + """ + 检测和更新资源包 + """ + # 资源包的git仓库地址 + _repo = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/package.json" + _files_api = f"https://api.github.com/repos/jxxghp/MoviePilot-Resources/contents/resources" + _base_dir: Path = settings.ROOT_PATH / "app" + + def __init__(self): + self.siteshelper = SitesHelper() + self.check() + + def check(self): + """ + 检测是否有更新，如有则下载安装 + """ + if not settings.AUTO_UPDATE_RESOURCE: + return + if SystemUtils.is_frozen(): + return + logger.info("开始检测资源包版本...") + res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, timeout=10).get_res(self._repo) + if res: + resource_info = json.loads(res.text) + else: + logger.warn("无法连接资源包仓库！") + return + online_version = resource_info.get("version") + if online_version: + logger.info(f"最新资源包版本：v{online_version}") + # 需要更新的资源包 + need_updates = {} + # 资源明细 + resources: dict = resource_info.get("resources") or {} + for rname, resource in resources.items(): + rtype = resource.get("type") + platform = resource.get("platform") + target = resource.get("target") + version = resource.get("version") + # 判断平台 + if platform and platform != SystemUtils.platform: + continue + # 判断本地是否存在 + local_path = self._base_dir / target + if not local_path.exists(): + continue + # 判断版本号 + if rtype == "auth": + # 站点认证资源 + local_version = self.siteshelper.auth_version + elif rtype == "sites": + # 站点索引资源 + local_version = self.siteshelper.indexer_version + else: + continue + if StringUtils.compare_version(version, local_version) > 0: + logger.info(f"{rname} 资源包有更新，最新版本：v{version}") + # 需要安装 + need_updates[rname] = target + if need_updates: + # 下载文件信息列表 + r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, + timeout=10).get_res(self._files_api) + if not r or r.status_code != 200: + return None, f"连接仓库失败：{r.status_code} - {r.reason}" + files_info = r.json() + for item in files_info: + save_path = need_updates.get(item.get("name")) + if not save_path: + continue + if item.get("download_url"): + # 下载插件文件 + res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, + timeout=60).get_res(item["download_url"]) + if not res: + logger.error(f"文件 {item.get('name')} 下载失败！") + elif res.status_code != 200: + logger.error(f"下载文件 {item.get('name')} 失败：{res.status_code} - {res.reason}") + # 创建插件文件夹 + file_path = self._base_dir / save_path + if not file_path.parent.exists(): + file_path.parent.mkdir(parents=True, exist_ok=True) + # 写入文件 + file_path.write_bytes(res.content) + logger.info("资源包更新完成，开始重启服务...") + SystemUtils.restart() + else: + logger.info("所有资源已最新，无需更新") diff --git a/app/main.py b/app/main.py index 5973cb6b..ea48878b 100644 --- a/app/main.py +++ b/app/main.py @@ -22,10 +22,12 @@ from app.core.plugin import PluginManager from app.db.init import init_db, update_db from app.helper.thread import ThreadHelper from app.helper.display import DisplayHelper +from app.helper.resource import ResourceHelper from app.helper.sites import SitesHelper from app.scheduler import Scheduler from app.command import Command + # App App = FastAPI(title=settings.PROJECT_NAME, openapi_url=f"{settings.API_V1_STR}/openapi.json") @@ -169,6 +171,8 @@ def start_module(): DisplayHelper() # 站点管理 SitesHelper() + # 资源包检测 + ResourceHelper() # 加载模块 ModuleManager() # 加载插件 diff --git a/config/app.env b/config/app.env index 8bff6387..0aa6accf 100644 --- a/config/app.env +++ b/config/app.env @@ -27,6 +27,8 @@ TMDB_API_DOMAIN=api.themoviedb.org RECOGNIZE_SOURCE=themoviedb # 大内存模式，开启后会增加缓存数量，但会占用更多内存 BIG_MEMORY_MODE=false +# 自动检查和更新站点资源包（索引、认证等） +AUTO_UPDATE_RESOURCE=true #################################### # 媒体识别&刮削 # From e2bd5cc24558816c0ad484983fbd91d6f4ce54e6 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 16 Nov 2023 21:10:21 +0800 Subject: [PATCH 13/20] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20resource.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/helper/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/helper/resource.py b/app/helper/resource.py index d1223edd..0d50a07d 100644 --- a/app/helper/resource.py +++ b/app/helper/resource.py @@ -17,7 +17,7 @@ class ResourceHelper(metaclass=Singleton): # 资源包的git仓库地址 _repo = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/package.json" _files_api = f"https://api.github.com/repos/jxxghp/MoviePilot-Resources/contents/resources" - _base_dir: Path = settings.ROOT_PATH / "app" + _base_dir: Path = settings.ROOT_PATH def __init__(self): self.siteshelper = SitesHelper() From 2d8e45cd1b0ca961cf4852d3f2381c04bcc8ffcf Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 17 Nov 2023 10:41:14 +0800 Subject: [PATCH 14/20] =?UTF-8?q?feat=20TG=E6=B6=88=E6=81=AF=E9=87=8D?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/modules/telegram/telegram.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/modules/telegram/telegram.py b/app/modules/telegram/telegram.py index 1e086c2b..7121f559 100644 --- a/app/modules/telegram/telegram.py +++ b/app/modules/telegram/telegram.py @@ -12,6 +12,7 @@ from app.core.config import settings from app.core.context import MediaInfo, Context from app.core.metainfo import MetaInfo from app.log import logger +from app.utils.common import retry from app.utils.http import RequestUtils from app.utils.singleton import Singleton from app.utils.string import StringUtils @@ -174,6 +175,7 @@ class Telegram(metaclass=Singleton): logger.error(f"发送消息失败：{msg_e}") return False + @retry(Exception, logger=logger) def __send_request(self, userid: str = None, image="", caption="") -> bool: """ 向Telegram发送报文 @@ -181,7 +183,9 @@ class Telegram(metaclass=Singleton): if image: req = RequestUtils(proxies=settings.PROXY).get_res(image) - if req and req.content: + if req is None: + raise Exception("获取图片失败") + if req.content: image_file = Path(settings.TEMP_PATH) / Path(image).name image_file.write_bytes(req.content) photo = InputFile(image_file) @@ -189,12 +193,15 @@ class Telegram(metaclass=Singleton): photo=photo, caption=caption, parse_mode="Markdown") + if ret is None: + raise Exception("发送图片消息失败") if ret: return True ret = self._bot.send_message(chat_id=userid or self._telegram_chat_id, text=caption, parse_mode="Markdown") - + if ret is None: + raise Exception("发送文本消息失败") return True if ret else False def register_commands(self, commands: Dict[str, dict]): From c7c222b357cb1f81b37db60fe79c96889bd36c27 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 17 Nov 2023 11:12:52 +0800 Subject: [PATCH 15/20] v1.4.3 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 545667de..ca815266 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -APP_VERSION = 'v1.4.2-1' +APP_VERSION = 'v1.4.3' From 03e0118fb7b3817477b5d6a0fbed9097b3c2500a Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 17 Nov 2023 11:39:03 +0800 Subject: [PATCH 16/20] add linux frozen build --- .github/workflows/build.yml | 66 +++++++++++++++++++++++++++++++++++-- windows.spec => frozen.spec | 0 2 files changed, 64 insertions(+), 2 deletions(-) rename windows.spec => frozen.spec (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9502af29..c16b9d0d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,7 @@ jobs: - name: Pyinstaller run: | - pyinstaller windows.spec + pyinstaller frozen.spec shell: pwsh - name: Upload Windows File @@ -119,10 +119,71 @@ jobs: name: windows path: dist/MoviePilot.exe + Linux-build: + runs-on: ubuntu-latest + name: Build Linux Binary + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Init Python 3.11.4 + uses: actions/setup-python@v4 + with: + python-version: '3.11.4' + cache: 'pip' + + - name: Install Dependent Packages + run: | + python -m pip install --upgrade pip + pip install wheel pyinstaller + pip install -r requirements.txt + + - name: Prepare Frontend + run: | + wget http://nginx.org/download/nginx-1.25.2.zip + unzip nginx-1.25.2.zip + mv nginx-1.25.2/nginx-1.25.2 nginx + rm nginx-1.25.2.zip + rm -rf nginx-1.25.2 + + FRONTEND_VERSION=$(curl -s "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" | jq -r .tag_name) + wget "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/$FRONTEND_VERSION/dist.zip" + unzip dist.zip + mv dist/dist/* nginx/html + rm dist.zip + rm -rf dist + mv nginx/html/nginx.conf nginx/conf/nginx.conf + mkdir -p nginx/temp + touch nginx/temp/__keep__.txt + mkdir -p nginx/logs + touch nginx/logs/__keep__.txt + + wget https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip + unzip MoviePilot-Plugins-main.zip + mv MoviePilot-Plugins-main/MoviePilot-Plugins-main/plugins/* app/plugins/ + rm MoviePilot-Plugins-main.zip + rm -rf MoviePilot-Plugins-main + + wget https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip + unzip MoviePilot-Resources-main.zip + mv MoviePilot-Resources-main/MoviePilot-Resources-main/resources/* app/helper/ + rm MoviePilot-Resources-main.zip + rm -rf MoviePilot-Resources-main + + - name: Pyinstaller + run: | + pyinstaller frozen.spec + + - name: Upload Ubuntu File + uses: actions/upload-artifact@v3 + with: + name: ubuntu + path: dist/MoviePilot + Create-release: permissions: write-all runs-on: ubuntu-latest - needs: [ Windows-build, Docker-build ] + needs: [ Windows-build, Docker-build, Linux-build ] steps: - uses: actions/checkout@v2 @@ -140,6 +201,7 @@ jobs: run: | mkdir releases mv ./windows/MoviePilot.exe ./releases/MoviePilot_v${{ env.app_version }}.exe + mv ./ubuntu/dist/MoviePilot ./releases/MoviePilot_amd64_v${{ env.app_version }} - name: Create Release id: create_release diff --git a/windows.spec b/frozen.spec similarity index 100% rename from windows.spec rename to frozen.spec From 9841f3dd18fb99a478aa0b65099e8f9c820eff81 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 17 Nov 2023 11:43:01 +0800 Subject: [PATCH 17/20] fix build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c16b9d0d..c57868c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: run: | wget http://nginx.org/download/nginx-1.25.2.zip unzip nginx-1.25.2.zip - mv nginx-1.25.2/nginx-1.25.2 nginx + mv nginx-1.25.2 nginx rm nginx-1.25.2.zip rm -rf nginx-1.25.2 From 6eec4ef7f40c8191ec898df8a8c59dec5a9309d8 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 17 Nov 2023 12:27:43 +0800 Subject: [PATCH 18/20] fix build --- .github/workflows/build.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c57868c0..3f878427 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,12 +144,11 @@ jobs: unzip nginx-1.25.2.zip mv nginx-1.25.2 nginx rm nginx-1.25.2.zip - rm -rf nginx-1.25.2 FRONTEND_VERSION=$(curl -s "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" | jq -r .tag_name) wget "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/$FRONTEND_VERSION/dist.zip" unzip dist.zip - mv dist/dist/* nginx/html + mv dist/* nginx/html rm dist.zip rm -rf dist mv nginx/html/nginx.conf nginx/conf/nginx.conf @@ -159,15 +158,15 @@ jobs: touch nginx/logs/__keep__.txt wget https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip - unzip MoviePilot-Plugins-main.zip - mv MoviePilot-Plugins-main/MoviePilot-Plugins-main/plugins/* app/plugins/ - rm MoviePilot-Plugins-main.zip + unzip main.zip + mv MoviePilot-Plugins-main/plugins/* app/plugins/ + rm main.zip rm -rf MoviePilot-Plugins-main wget https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip - unzip MoviePilot-Resources-main.zip - mv MoviePilot-Resources-main/MoviePilot-Resources-main/resources/* app/helper/ - rm MoviePilot-Resources-main.zip + unzip main.zip + mv MoviePilot-Resources-main/resources/* app/helper/ + rm main.zip rm -rf MoviePilot-Resources-main - name: Pyinstaller From 2d146880ec428ac66b072de4399e1ab6b64323c9 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 17 Nov 2023 13:36:23 +0800 Subject: [PATCH 19/20] fix --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f878427..359261d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -176,7 +176,7 @@ jobs: - name: Upload Ubuntu File uses: actions/upload-artifact@v3 with: - name: ubuntu + name: linux path: dist/MoviePilot Create-release: @@ -200,7 +200,7 @@ jobs: run: | mkdir releases mv ./windows/MoviePilot.exe ./releases/MoviePilot_v${{ env.app_version }}.exe - mv ./ubuntu/dist/MoviePilot ./releases/MoviePilot_amd64_v${{ env.app_version }} + mv ./linux/dist/MoviePilot ./releases/MoviePilot_linux_v${{ env.app_version }} - name: Create Release id: create_release From 04339539d1c41cb724dca53dd68e85a270d2e50d Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 17 Nov 2023 14:25:00 +0800 Subject: [PATCH 20/20] fix bug --- app/helper/resource.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/helper/resource.py b/app/helper/resource.py index 0d50a07d..67f44e08 100644 --- a/app/helper/resource.py +++ b/app/helper/resource.py @@ -68,6 +68,8 @@ class ResourceHelper(metaclass=Singleton): continue if StringUtils.compare_version(version, local_version) > 0: logger.info(f"{rname} 资源包有更新，最新版本：v{version}") + else: + continue # 需要安装 need_updates[rname] = target if need_updates: @@ -90,7 +92,7 @@ class ResourceHelper(metaclass=Singleton): elif res.status_code != 200: logger.error(f"下载文件 {item.get('name')} 失败：{res.status_code} - {res.reason}") # 创建插件文件夹 - file_path = self._base_dir / save_path + file_path = self._base_dir / save_path / item.get("name") if not file_path.parent.exists(): file_path.parent.mkdir(parents=True, exist_ok=True) # 写入文件