Merge remote-tracking branch 'origin/main'

This commit is contained in:
jxxghp
2023-11-18 21:54:10 +08:00
19 changed files with 253 additions and 29 deletions

View File

@ -110,7 +110,7 @@ jobs:
- name: Pyinstaller
run: |
pyinstaller windows.spec
pyinstaller frozen.spec
shell: pwsh
- name: Upload Windows File
@ -119,10 +119,70 @@ jobs:
name: windows
path: dist/MoviePilot.exe
Linux-build:
runs-on: ubuntu-latest
name: Build Linux Binary
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Init Python 3.11.4
uses: actions/setup-python@v4
with:
python-version: '3.11.4'
cache: 'pip'
- name: Install Dependent Packages
run: |
python -m pip install --upgrade pip
pip install wheel pyinstaller
pip install -r requirements.txt
- name: Prepare Frontend
run: |
wget http://nginx.org/download/nginx-1.25.2.zip
unzip nginx-1.25.2.zip
mv nginx-1.25.2 nginx
rm nginx-1.25.2.zip
FRONTEND_VERSION=$(curl -s "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" | jq -r .tag_name)
wget "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/$FRONTEND_VERSION/dist.zip"
unzip dist.zip
mv dist/* nginx/html
rm dist.zip
rm -rf dist
mv nginx/html/nginx.conf nginx/conf/nginx.conf
mkdir -p nginx/temp
touch nginx/temp/__keep__.txt
mkdir -p nginx/logs
touch nginx/logs/__keep__.txt
wget https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip
unzip main.zip
mv MoviePilot-Plugins-main/plugins/* app/plugins/
rm main.zip
rm -rf MoviePilot-Plugins-main
wget https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip
unzip main.zip
mv MoviePilot-Resources-main/resources/* app/helper/
rm main.zip
rm -rf MoviePilot-Resources-main
- name: Pyinstaller
run: |
pyinstaller frozen.spec
- name: Upload Ubuntu File
uses: actions/upload-artifact@v3
with:
name: linux
path: dist/MoviePilot
Create-release:
permissions: write-all
runs-on: ubuntu-latest
needs: [ Windows-build, Docker-build ]
needs: [ Windows-build, Docker-build, Linux-build ]
steps:
- uses: actions/checkout@v2
@ -140,6 +200,7 @@ jobs:
run: |
mkdir releases
mv ./windows/MoviePilot.exe ./releases/MoviePilot_v${{ env.app_version }}.exe
mv ./linux/dist/MoviePilot ./releases/MoviePilot_linux_v${{ env.app_version }}
- name: Create Release
id: create_release

View File

@ -70,7 +70,8 @@ MoviePilot需要配套下载器和媒体服务器配合使用。
- **PGID**:运行程序用户的`gid`,默认`0`(仅支持环境变量配置)
- **UMASK**:掩码权限,默认`000`,可以考虑设置为`022`(仅支持环境变量配置)
- **PROXY_HOST** 网络代理访问themoviedb或者重启更新需要使用代理访问格式为`http(s)://ip:port`、`socks5://user:pass@host:port`(仅支持环境变量配置)
- **MOVIEPILOT_AUTO_UPDATE**重启更新,`true`/`release`/`dev`/`false`,默认`release` **注意:如果出现网络问题可以配置`PROXY_HOST`**(仅支持环境变量配置)
- **MOVIEPILOT_AUTO_UPDATE** 重启时自动更新,`true`/`release`/`dev`/`false`,默认`release`需要能正常连接Github **注意:如果出现网络问题可以配置`PROXY_HOST`**(仅支持环境变量配置)
- **AUTO_UPDATE_RESOURCE**:启动时自动检测和更新资源包(站点索引及认证等),`true`/`false`,默认`true`需要能正常连接Github仅支持Docker
---
- **❗SUPERUSER** 超级管理员用户名,默认`admin`,安装后使用该用户登录后台管理界面
- **❗SUPERUSER_PASSWORD** 超级管理员初始密码,默认`password`,建议修改为复杂密码
@ -105,6 +106,7 @@ MoviePilot需要配套下载器和媒体服务器配合使用。
---
- **OCR_HOST** OCR识别服务器地址格式`http(s)://ip:port`用于识别站点验证码实现自动登录获取Cookie等不配置默认使用内建服务器`https://movie-pilot.org`,可使用 [这个镜像](https://hub.docker.com/r/jxxghp/moviepilot-ocr) 自行搭建。
- **PLUGIN_MARKET** 插件市场仓库地址,多个地址使用`,`分隔,保留最后的/,默认为官方插件仓库:`https://raw.githubusercontent.com/jxxghp/MoviePilot-Plugins/main/`。
- **GITHUB_TOKEN** Github token提高请求api限流阈值 ghp_****(仅支持环境变量配置)
---
- **❗MESSAGER** 消息通知渠道,支持 `telegram`/`wechat`/`slack`/`synologychat`,开启多个渠道时使用`,`分隔。同时还需要配置对应渠道的环境变量,非对应渠道的变量可删除,推荐使用`telegram`
@ -190,7 +192,7 @@ MoviePilot需要配套下载器和媒体服务器配合使用。
`AUTH_SITE`支持配置多个认证站点,使用`,`分隔,如:`iyuu,hhclub`,会依次执行认证操作,直到有一个站点认证成功。
- **❗AUTH_SITE** 认证站点,认证资源`v1.0.1`支持`iyuu`/`hhclub`/`audiences`/`hddolby`/`zmpt`/`freefarm`/`hdfans`/`wintersakura`/`leaves`/`1ptba`/`icc2022`/`ptlsp`/`xingtan`/`ptvicomo`
- **❗AUTH_SITE** 认证站点,认证资源`v1.0.2`支持`iyuu`/`hhclub`/`audiences`/`hddolby`/`zmpt`/`freefarm`/`hdfans`/`wintersakura`/`leaves`/`1ptba`/`icc2022`/`ptlsp`/`xingtan`/`ptvicomo`/`agsvpt`
| 站点 | 参数 |
|:------------:|:-----------------------------------------------------:|
@ -208,6 +210,7 @@ MoviePilot需要配套下载器和媒体服务器配合使用。
| ptlsp | `PTLSP_UID`用户ID<br/>`PTLSP_PASSKEY`:密钥 |
| xingtan | `XINGTAN_UID`用户ID<br/>`XINGTAN_PASSKEY`:密钥 |
| ptvicomo | `PTVICOMO_UID`用户ID<br/>`PTVICOMO_PASSKEY`:密钥 |
| agsvpt | `AGSVPT_UID`用户ID<br/>`AGSVPT_PASSKEY`:密钥 |
### 2. **进阶配置**

View File

@ -163,7 +163,8 @@ def latest_version(_: schemas.TokenPayload = Depends(verify_token)):
"""
查询Github所有Release版本
"""
version_res = RequestUtils().get_res(f"https://api.github.com/repos/jxxghp/MoviePilot/releases")
version_res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res(
f"https://api.github.com/repos/jxxghp/MoviePilot/releases")
if version_res:
ver_json = version_res.json()
if ver_json:

View File

@ -213,7 +213,7 @@ class SearchChain(ChainBase):
continue
# 在副标题中判断是否存在标题与原语种标题
if torrent.description:
subtitle = torrent.description.split()
subtitle = re.split(r'[\s/|]+', torrent.description)
if (StringUtils.is_chinese(mediainfo.title)
and str(mediainfo.title) in subtitle) \
or (StringUtils.is_chinese(mediainfo.original_title)

View File

@ -87,7 +87,7 @@ class SystemChain(ChainBase, metaclass=Singleton):
"""
获取最新版本
"""
version_res = RequestUtils(proxies=settings.PROXY).get_res(
version_res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS).get_res(
"https://api.github.com/repos/jxxghp/MoviePilot/releases/latest")
if version_res:
ver_json = version_res.json()

View File

@ -60,7 +60,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
else:
return self.load_cache(self._rss_file) or {}
@cached(cache=TTLCache(maxsize=128, ttl=600))
@cached(cache=TTLCache(maxsize=128, ttl=595))
def browse(self, domain: str) -> List[TorrentInfo]:
"""
浏览站点首页内容返回种子清单TTL缓存10分钟
@ -73,7 +73,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
return []
return self.refresh_torrents(site=site)
@cached(cache=TTLCache(maxsize=128, ttl=300))
@cached(cache=TTLCache(maxsize=128, ttl=295))
def rss(self, domain: str) -> List[TorrentInfo]:
"""
获取站点RSS内容返回种子清单TTL缓存5分钟

View File

@ -212,6 +212,10 @@ class Settings(BaseSettings):
BIG_MEMORY_MODE: bool = False
# 插件市场仓库地址,多个地址使用,分隔,地址以/结尾
PLUGIN_MARKET: str = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Plugins/main/"
# Github token提高请求api限流阈值 ghp_****
GITHUB_TOKEN: str = None
# 自动检查和更新站点资源包(站点索引、认证等)
AUTO_UPDATE_RESOURCE: bool = True
@property
def INNER_CONFIG_PATH(self):
@ -321,6 +325,17 @@ class Settings(BaseSettings):
return Path(self.DOWNLOAD_ANIME_PATH)
return self.SAVE_TV_PATH
@property
def GITHUB_HEADERS(self):
"""
Github请求头
"""
if self.GITHUB_TOKEN:
return {
"Authorization": f"Bearer {self.GITHUB_TOKEN}"
}
return {}
def __init__(self, **kwargs):
super().__init__(**kwargs)
with self.CONFIG_PATH as p:

View File

@ -34,6 +34,7 @@ class MetaVideo(MetaBase):
_name_no_begin_re = r"^\[.+?]"
_name_no_chinese_re = r".*版|.*字幕"
_name_se_words = ['', '', '', '', '', '', '']
_name_movie_words = ['剧场版', '劇場版', '电影版', '電影版']
_name_nostring_re = r"^PTS|^JADE|^AOD|^CHC|^[A-Z]{1,4}TV[\-0-9UVHDK]*" \
r"|HBO$|\s+HBO|\d{1,2}th|\d{1,2}bit|NETFLIX|AMAZON|IMAX|^3D|\s+3D|^BBC\s+|\s+BBC|BBC$|DISNEY\+?|XXX|\s+DC$" \
r"|[第\s共]+[0-9一二三四五六七八九十\-\s]+季" \
@ -182,8 +183,9 @@ class MetaVideo(MetaBase):
if not self.cn_name:
self.cn_name = token
elif not self._stop_cnname_flag:
if not re.search("%s" % self._name_no_chinese_re, token, flags=re.IGNORECASE) \
and not re.search("%s" % self._name_se_words, token, flags=re.IGNORECASE):
if re.search("%s" % self._name_movie_words, token, flags=re.IGNORECASE) \
or (not re.search("%s" % self._name_no_chinese_re, token, flags=re.IGNORECASE)
and not re.search("%s" % self._name_se_words, token, flags=re.IGNORECASE)):
self.cn_name = "%s %s" % (self.cn_name, token)
self._stop_cnname_flag = True
else:

View File

@ -12,6 +12,7 @@ from app.schemas.types import SystemConfigKey
from app.utils.object import ObjectUtils
from app.utils.singleton import Singleton
from app.utils.string import StringUtils
from app.utils.system import SystemUtils
class PluginManager(metaclass=Singleton):
@ -105,6 +106,8 @@ class PluginManager(metaclass=Singleton):
"""
安装本地不存在的在线插件
"""
if SystemUtils.is_frozen():
return
logger.info("开始安装在线插件...")
# 已安装插件
install_plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins) or []

View File

@ -24,7 +24,8 @@ class PluginHelper(metaclass=Singleton):
"""
if not repo_url:
return {}
res = RequestUtils(proxies=settings.PROXY, timeout=10).get_res(f"{repo_url}package.json")
res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS,
timeout=10).get_res(f"{repo_url}package.json")
if res:
return json.loads(res.text)
return {}
@ -49,7 +50,7 @@ class PluginHelper(metaclass=Singleton):
获取插件的文件列表
"""
file_api = f"https://api.github.com/repos/{user}/{repo}/contents/plugins/{_p.lower()}"
r = RequestUtils(proxies=settings.PROXY).get_res(file_api)
r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, timeout=10).get_res(file_api)
if not r or r.status_code != 200:
return None, f"连接仓库失败:{r.status_code} - {r.reason}"
ret = r.json()
@ -66,7 +67,8 @@ class PluginHelper(metaclass=Singleton):
for item in _l:
if item.get("download_url"):
# 下载插件文件
res = RequestUtils(proxies=settings.PROXY).get_res(item["download_url"])
res = RequestUtils(proxies=settings.PROXY,
headers=settings.GITHUB_HEADERS, timeout=30).get_res(item["download_url"])
if not res:
return False, f"文件 {item.get('name')} 下载失败!"
elif res.status_code != 200:

103
app/helper/resource.py Normal file
View File

@ -0,0 +1,103 @@
import json
from pathlib import Path
from app.core.config import settings
from app.helper.sites import SitesHelper
from app.log import logger
from app.utils.http import RequestUtils
from app.utils.singleton import Singleton
from app.utils.string import StringUtils
from app.utils.system import SystemUtils
class ResourceHelper(metaclass=Singleton):
"""
检测和更新资源包
"""
# 资源包的git仓库地址
_repo = "https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/package.json"
_files_api = f"https://api.github.com/repos/jxxghp/MoviePilot-Resources/contents/resources"
_base_dir: Path = settings.ROOT_PATH
def __init__(self):
self.siteshelper = SitesHelper()
self.check()
def check(self):
"""
检测是否有更新,如有则下载安装
"""
if not settings.AUTO_UPDATE_RESOURCE:
return
if SystemUtils.is_frozen():
return
logger.info("开始检测资源包版本...")
res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS, timeout=10).get_res(self._repo)
if res:
resource_info = json.loads(res.text)
else:
logger.warn("无法连接资源包仓库!")
return
online_version = resource_info.get("version")
if online_version:
logger.info(f"最新资源包版本v{online_version}")
# 需要更新的资源包
need_updates = {}
# 资源明细
resources: dict = resource_info.get("resources") or {}
for rname, resource in resources.items():
rtype = resource.get("type")
platform = resource.get("platform")
target = resource.get("target")
version = resource.get("version")
# 判断平台
if platform and platform != SystemUtils.platform:
continue
# 判断本地是否存在
local_path = self._base_dir / target
if not local_path.exists():
continue
# 判断版本号
if rtype == "auth":
# 站点认证资源
local_version = self.siteshelper.auth_version
elif rtype == "sites":
# 站点索引资源
local_version = self.siteshelper.indexer_version
else:
continue
if StringUtils.compare_version(version, local_version) > 0:
logger.info(f"{rname} 资源包有更新最新版本v{version}")
else:
continue
# 需要安装
need_updates[rname] = target
if need_updates:
# 下载文件信息列表
r = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS,
timeout=10).get_res(self._files_api)
if not r or r.status_code != 200:
return None, f"连接仓库失败:{r.status_code} - {r.reason}"
files_info = r.json()
for item in files_info:
save_path = need_updates.get(item.get("name"))
if not save_path:
continue
if item.get("download_url"):
# 下载插件文件
res = RequestUtils(proxies=settings.PROXY, headers=settings.GITHUB_HEADERS,
timeout=60).get_res(item["download_url"])
if not res:
logger.error(f"文件 {item.get('name')} 下载失败!")
elif res.status_code != 200:
logger.error(f"下载文件 {item.get('name')} 失败:{res.status_code} - {res.reason}")
# 创建插件文件夹
file_path = self._base_dir / save_path / item.get("name")
if not file_path.parent.exists():
file_path.parent.mkdir(parents=True, exist_ok=True)
# 写入文件
file_path.write_bytes(res.content)
logger.info("资源包更新完成,开始重启服务...")
SystemUtils.restart()
else:
logger.info("所有资源已最新,无需更新")

View File

@ -1,11 +1,14 @@
import re
import xml.dom.minidom
from typing import List, Tuple, Union
from urllib.parse import urljoin
import chardet
from lxml import etree
from app.core.config import settings
from app.helper.browser import PlaywrightHelper
from app.log import logger
from app.utils.dom import DomUtils
from app.utils.http import RequestUtils
from app.utils.string import StringUtils
@ -240,8 +243,28 @@ class RssHelper:
print(str(err))
return []
if ret:
ret_xml = ret.text
ret_xml = ""
try:
# 使用chardet检测字符编码
raw_data = ret.content
if raw_data:
try:
result = chardet.detect(raw_data)
encoding = result['encoding']
# 解码为字符串
ret_xml = raw_data.decode(encoding)
except Exception as e:
logger.debug(f"chardet解码失败{str(e)}")
# 探测utf-8解码
match = re.search(r'encoding\s*=\s*["\']([^"\']+)["\']', ret.text)
if match:
encoding = match.group(1)
if encoding:
ret_xml = raw_data.decode(encoding)
else:
ret.encoding = ret.apparent_encoding
if not ret_xml:
ret_xml = ret.text
# 解析XML
dom_tree = xml.dom.minidom.parseString(ret_xml)
rootNode = dom_tree.documentElement

View File

@ -22,10 +22,12 @@ from app.core.plugin import PluginManager
from app.db.init import init_db, update_db
from app.helper.thread import ThreadHelper
from app.helper.display import DisplayHelper
from app.helper.resource import ResourceHelper
from app.helper.sites import SitesHelper
from app.scheduler import Scheduler
from app.command import Command
# App
App = FastAPI(title=settings.PROJECT_NAME,
openapi_url=f"{settings.API_V1_STR}/openapi.json")
@ -169,6 +171,8 @@ def start_module():
DisplayHelper()
# 站点管理
SitesHelper()
# 资源包检测
ResourceHelper()
# 加载模块
ModuleManager()
# 加载插件

View File

@ -416,14 +416,6 @@ class FileTransferModule(_ModuleBase):
rename_dict=self.__get_naming_dict(meta=in_meta,
mediainfo=mediainfo)
).parent
# 目录已存在时不处理
if new_path.exists():
logger.warn(f"目标目录已存在:{new_path}")
return TransferInfo(success=False,
message=f"目标目录已存在:{new_path}",
path=in_path,
target_path=new_path,
is_bluray=bluray_flag)
# 转移蓝光原盘
retcode = self.__transfer_dir(file_path=in_path,
new_path=new_path,

View File

@ -12,6 +12,7 @@ from app.core.config import settings
from app.core.context import MediaInfo, Context
from app.core.metainfo import MetaInfo
from app.log import logger
from app.utils.common import retry
from app.utils.http import RequestUtils
from app.utils.singleton import Singleton
from app.utils.string import StringUtils
@ -174,6 +175,7 @@ class Telegram(metaclass=Singleton):
logger.error(f"发送消息失败:{msg_e}")
return False
@retry(Exception, logger=logger)
def __send_request(self, userid: str = None, image="", caption="") -> bool:
"""
向Telegram发送报文
@ -181,7 +183,9 @@ class Telegram(metaclass=Singleton):
if image:
req = RequestUtils(proxies=settings.PROXY).get_res(image)
if req and req.content:
if req is None:
raise Exception("获取图片失败")
if req.content:
image_file = Path(settings.TEMP_PATH) / Path(image).name
image_file.write_bytes(req.content)
photo = InputFile(image_file)
@ -189,12 +193,15 @@ class Telegram(metaclass=Singleton):
photo=photo,
caption=caption,
parse_mode="Markdown")
if ret is None:
raise Exception("发送图片消息失败")
if ret:
return True
ret = self._bot.send_message(chat_id=userid or self._telegram_chat_id,
text=caption,
parse_mode="Markdown")
if ret is None:
raise Exception("发送文本消息失败")
return True if ret else False
def register_commands(self, commands: Dict[str, dict]):

View File

@ -27,6 +27,8 @@ TMDB_API_DOMAIN=api.themoviedb.org
RECOGNIZE_SOURCE=themoviedb
# 大内存模式,开启后会增加缓存数量,但会占用更多内存
BIG_MEMORY_MODE=false
# 自动检查和更新站点资源包(索引、认证等)
AUTO_UPDATE_RESOURCE=true
####################################
# 媒体识别&刮削 #

View File

@ -82,6 +82,7 @@ exe = EXE(
collect_pkg_data('zhconv'),
collect_pkg_data('cn2an'),
collect_pkg_data('database', include_py_files=True),
collect_pkg_data('app.helper'),
[],
name='MoviePilot',
debug=False,

11
update
View File

@ -5,7 +5,7 @@ download_and_unzip() {
url="$1"
target_dir="$2"
echo "正在下载 ${url}..."
curl ${CURL_OPTIONS} "$url" | busybox unzip -d /tmp -
curl ${CURL_OPTIONS} "$url" ${CURL_HEADERS} | busybox unzip -d /tmp -
if [ $? -eq 0 ]; then
if [ -e /tmp/MoviePilot-* ]; then
mv /tmp/MoviePilot-* /tmp/${target_dir}
@ -30,7 +30,7 @@ install_backend_and_download_resources() {
download_and_unzip "https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip" "Resources"
if [ $? -eq 0 ]; then
echo "资源包下载成功"
frontend_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" | jq -r .tag_name)
frontend_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" ${CURL_HEADERS} | jq -r .tag_name)
if [[ "${frontend_version}" == *v* ]]; then
download_and_unzip "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${frontend_version}/dist.zip" "dist"
if [ $? -eq 0 ]; then
@ -83,6 +83,11 @@ if [[ "${MOVIEPILOT_AUTO_UPDATE}" = "true" ]] || [[ "${MOVIEPILOT_AUTO_UPDATE}"
CURL_OPTIONS="-sL"
echo "不使用代理更新程序"
fi
if [ -n "${GITHUB_TOKEN}" ]; then
CURL_HEADERS="--header 'Authorization: Bearer ${GITHUB_TOKEN}'"
else
CURL_HEADERS=""
fi
if [ "${MOVIEPILOT_AUTO_UPDATE}" = "dev" ]; then
echo "Dev 更新模式"
install_backend_and_download_resources "heads/main.zip"
@ -92,7 +97,7 @@ if [[ "${MOVIEPILOT_AUTO_UPDATE}" = "true" ]] || [[ "${MOVIEPILOT_AUTO_UPDATE}"
if [[ "${old_version}" == *APP_VERSION* ]]; then
current_version=v$(echo ${old_version} | sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
echo "当前版本号:${current_version}"
new_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot/releases/latest" | jq -r .tag_name)
new_version=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot/releases/latest" ${CURL_HEADERS} | jq -r .tag_name)
if [[ "${new_version}" == *v* ]]; then
release_version=${new_version}
echo "最新版本号:${release_version}"

View File

@ -1 +1 @@
APP_VERSION = 'v1.4.2'
APP_VERSION = 'v1.4.3'