Merge branch 'main' of github.com:lingjiameng/MoviePilot

README.md

@@ -219,6 +219,14 @@ location / {
     proxy_set_header X-Forwarded-Proto $scheme;
 }
 ```
+- 反代使用ssl时，需要开启`http2`，否则会导致日志加载时间过长或不可用。以`Nginx`为例：
+```nginx configuration
+server{
+    listen 443 ssl;
+    http2 on;
+    ...
+}
+```
 - 新建的企业微信应用需要固定公网IP的代理才能收到消息，代理添加以下代码：
 ```nginx configuration
 location /cgi-bin/gettoken {

app/api/endpoints/douban.py

@@ -90,7 +90,7 @@ def movie_top250(page: int = 1,
     """
     浏览豆瓣剧集信息
     """
-    movies = DoubanChain().movie_top250(page=page, count=count)
+    movies = DoubanChain().movie_top250(page=page, count=count) or []
     return [MediaInfo(douban_info=movie).to_dict() for movie in movies]
 
 
@@ -101,7 +101,7 @@ def tv_weekly_chinese(page: int = 1,
     """
     中国每周剧集口碑榜
     """
-    tvs = DoubanChain().tv_weekly_chinese(page=page, count=count)
+    tvs = DoubanChain().tv_weekly_chinese(page=page, count=count) or []
     return [MediaInfo(douban_info=tv).to_dict() for tv in tvs]
 
 
@@ -112,7 +112,7 @@ def tv_weekly_global(page: int = 1,
     """
     全球每周剧集口碑榜
     """
-    tvs = DoubanChain().tv_weekly_global(page=page, count=count)
+    tvs = DoubanChain().tv_weekly_global(page=page, count=count) or []
     return [MediaInfo(douban_info=tv).to_dict() for tv in tvs]
 
 
@@ -123,7 +123,7 @@ def tv_animation(page: int = 1,
     """
     热门动画剧集
     """
-    tvs = DoubanChain().tv_animation(page=page, count=count)
+    tvs = DoubanChain().tv_animation(page=page, count=count) or []
     return [MediaInfo(douban_info=tv).to_dict() for tv in tvs]
 
 
@@ -134,7 +134,7 @@ def movie_hot(page: int = 1,
     """
     热门电影
     """
-    movies = DoubanChain().movie_hot(page=page, count=count)
+    movies = DoubanChain().movie_hot(page=page, count=count) or []
     return [MediaInfo(douban_info=movie).to_dict() for movie in movies]
 
 
@@ -145,7 +145,7 @@ def tv_hot(page: int = 1,
     """
     热门电视剧
     """
-    tvs = DoubanChain().tv_hot(page=page, count=count)
+    tvs = DoubanChain().tv_hot(page=page, count=count) or []
     return [MediaInfo(douban_info=tv).to_dict() for tv in tvs]
 
 

app/api/endpoints/media.py

@@ -113,20 +113,6 @@ def media_info(mediaid: str, type_name: str,
         doubanid = mediaid[7:]
     if not tmdbid and not doubanid:
         return schemas.MediaInfo()
-    if settings.RECOGNIZE_SOURCE == "themoviedb":
-        if not tmdbid and doubanid:
-            tmdbinfo = MediaChain().get_tmdbinfo_by_doubanid(doubanid=doubanid, mtype=mtype)
-            if tmdbinfo:
-                tmdbid = tmdbinfo.get("id")
-            else:
-                return schemas.MediaInfo()
-    else:
-        if not doubanid and tmdbid:
-            doubaninfo = MediaChain().get_doubaninfo_by_tmdbid(tmdbid=tmdbid, mtype=mtype)
-            if doubaninfo:
-                doubanid = doubaninfo.get("id")
-            else:
-                return schemas.MediaInfo()
     mediainfo = MediaChain().recognize_media(tmdbid=tmdbid, doubanid=doubanid, mtype=mtype)
     if mediainfo:
         MediaChain().obtain_images(mediainfo)

app/api/endpoints/message.py

@@ -1,18 +1,24 @@
+import json
 from typing import Union, Any, List
 
 from fastapi import APIRouter, BackgroundTasks, Depends
 from fastapi import Request
+from sqlalchemy.orm import Session
 from starlette.responses import PlainTextResponse
 
 from app import schemas
 from app.chain.message import MessageChain
 from app.core.config import settings
 from app.core.security import verify_token
+from app.db import get_db
+from app.db.models import User
+from app.db.models.message import Message
 from app.db.systemconfig_oper import SystemConfigOper
+from app.db.userauth import get_current_active_superuser
 from app.log import logger
 from app.modules.wechat.WXBizMsgCrypt3 import WXBizMsgCrypt
 from app.schemas import NotificationSwitch
-from app.schemas.types import SystemConfigKey, NotificationType
+from app.schemas.types import SystemConfigKey, NotificationType, MessageChannel
 
 router = APIRouter()
 
@@ -36,6 +42,39 @@ async def user_message(background_tasks: BackgroundTasks, request: Request):
     return schemas.Response(success=True)
 
 
+@router.post("/web", summary="接收WEB消息", response_model=schemas.Response)
+async def web_message(text: str, current_user: User = Depends(get_current_active_superuser)):
+    """
+    WEB消息响应
+    """
+    MessageChain().handle_message(
+        channel=MessageChannel.Web,
+        userid=current_user.id,
+        username=current_user.name,
+        text=text
+    )
+    return schemas.Response(success=True)
+
+
+@router.get("/web", summary="获取WEB消息", response_model=List[dict])
+def get_web_message(_: schemas.TokenPayload = Depends(verify_token),
+                    db: Session = Depends(get_db),
+                    page: int = 1,
+                    count: int = 20):
+    """
+    获取WEB消息列表
+    """
+    ret_messages = []
+    messages = Message.list_by_page(db, page=page, count=count)
+    for message in messages:
+        try:
+            ret_messages.append(message.to_dict())
+        except Exception as e:
+            logger.error(f"获取WEB消息列表失败: {str(e)}")
+            continue
+    return ret_messages
+
+
 def wechat_verify(echostr: str, msg_signature: str,
                   timestamp: Union[str, int], nonce: str) -> Any:
     """
@@ -103,7 +142,7 @@ def read_switchs(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
 def set_switchs(switchs: List[NotificationSwitch],
                 _: schemas.TokenPayload = Depends(verify_token)) -> Any:
     """
-    查询通知消息渠道开关
+    设置通知消息渠道开关
     """
     switch_list = []
     for switch in switchs:

app/api/endpoints/system.py

@@ -138,7 +138,7 @@ def set_setting(key: str, value: Union[list, dict, bool, int, str] = None,
 
 
 @router.get("/message", summary="实时消息")
-def get_message(token: str):
+def get_message(token: str, role: str = "sys"):
     """
     实时获取系统消息，返回格式为SSE
     """
@@ -152,7 +152,7 @@ def get_message(token: str):
 
     def event_generator():
         while True:
-            detail = message.get()
+            detail = message.get(role)
             yield 'data: %s\n\n' % (detail or '')
             time.sleep(3)
 

app/chain/__init__.py

@@ -15,6 +15,8 @@ from app.core.context import MediaInfo, TorrentInfo
 from app.core.event import EventManager
 from app.core.meta import MetaBase
 from app.core.module import ModuleManager
+from app.db.message_oper import MessageOper
+from app.helper.message import MessageHelper
 from app.log import logger
 from app.schemas import TransferInfo, TransferTorrent, ExistMediaInfo, DownloadingTorrent, CommingMessage, Notification, \
     WebhookEventInfo, TmdbEpisode
@@ -33,6 +35,8 @@ class ChainBase(metaclass=ABCMeta):
         """
         self.modulemanager = ModuleManager()
         self.eventmanager = EventManager()
+        self.messageoper = MessageOper()
+        self.messagehelper = MessageHelper()
 
     @staticmethod
     def load_cache(filename: str) -> Any:
@@ -403,6 +407,10 @@ class ChainBase(metaclass=ABCMeta):
         :param message:  消息体
         :return: 成功或失败
         """
+        logger.info(f"发送消息：channel={message.channel}，"
+                    f"title={message.title}, "
+                    f"text={message.text}，"
+                    f"userid={message.userid}")
         # 发送事件
         self.eventmanager.send_event(etype=EventType.NoticeMessage,
                                      data={
@@ -413,10 +421,13 @@ class ChainBase(metaclass=ABCMeta):
                                          "image": message.image,
                                          "userid": message.userid,
                                      })
-        logger.info(f"发送消息：channel={message.channel}，"
-                    f"title={message.title}, "
-                    f"text={message.text}，"
-                    f"userid={message.userid}")
+        # 保存消息
+        self.messagehelper.put(message, role="user")
+        self.messageoper.add(channel=message.channel, mtype=message.mtype,
+                             title=message.title, text=message.text,
+                             image=message.image, link=message.link,
+                             userid=message.userid, action=1)
+        # 发送
         self.run_module("post_message", message=message)
 
     def post_medias_message(self, message: Notification, medias: List[MediaInfo]) -> Optional[bool]:

app/chain/message.py

@@ -12,9 +12,11 @@ from app.core.config import settings
 from app.core.context import MediaInfo, Context
 from app.core.event import EventManager
 from app.core.meta import MetaBase
+from app.db.message_oper import MessageOper
+from app.helper.message import MessageHelper
 from app.helper.torrent import TorrentHelper
 from app.log import logger
-from app.schemas import Notification, NotExistMediaInfo
+from app.schemas import Notification, NotExistMediaInfo, CommingMessage
 from app.schemas.types import EventType, MessageChannel, MediaType
 from app.utils.string import StringUtils
 
@@ -43,6 +45,8 @@ class MessageChain(ChainBase):
         self.mediachain = MediaChain()
         self.eventmanager = EventManager()
         self.torrenthelper = TorrentHelper()
+        self.messagehelper = MessageHelper()
+        self.messageoper = MessageOper()
 
     def __get_noexits_info(
             self,
@@ -100,10 +104,8 @@ class MessageChain(ChainBase):
 
     def process(self, body: Any, form: Any, args: Any) -> None:
         """
-        识别消息内容，执行操作
+        调用模块识别消息内容
         """
-        # 申明全局变量
-        global _current_page, _current_meta, _current_media
         # 获取消息内容
         info = self.message_parser(body=body, form=form, args=args)
         if not info:
@@ -122,10 +124,35 @@ class MessageChain(ChainBase):
         if not text:
             logger.debug(f'未识别到消息内容：：{body}{form}{args}')
             return
+        # 处理消息
+        self.handle_message(channel=channel, userid=userid, username=username, text=text)
+
+    def handle_message(self, channel: MessageChannel, userid: Union[str, int], username: str, text: str) -> None:
+        """
+        识别消息内容，执行操作
+        """
+        # 申明全局变量
+        global _current_page, _current_meta, _current_media
         # 加载缓存
         user_cache: Dict[str, dict] = self.load_cache(self._cache_file) or {}
         # 处理消息
         logger.info(f'收到用户消息内容，用户：{userid}，内容：{text}')
+        # 保存消息
+        self.messagehelper.put(
+            CommingMessage(
+                userid=userid,
+                username=username,
+                channel=channel,
+                text=text
+            ), role="user")
+        self.messageoper.add(
+            channel=channel,
+            userid=userid,
+            username=username,
+            text=text,
+            action=0
+        )
+        # 处理消息
         if text.startswith('/'):
             # 执行命令
             self.eventmanager.send_event(

app/chain/search.py

@@ -124,14 +124,12 @@ class SearchChain(ChainBase):
         if keyword:
             keywords = [keyword]
         else:
-            keywords = list(
-                {
-                    mediainfo.title,
+            # 去重去空，但要保持顺序
+            keywords = list(dict.fromkeys([k for k in [mediainfo.title,
                                                        mediainfo.original_title,
                                                        mediainfo.en_title,
-                    mediainfo.sg_title
-                } - {None}
-            )
+                                                       mediainfo.sg_title] if k]))
+
         # 执行搜索
         torrents: List[TorrentInfo] = self.__search_all_sites(
             mediainfo=mediainfo,

app/chain/site.py

@@ -304,6 +304,8 @@ class SiteChain(ChainBase):
         if not site_info:
             return False, f"站点【{url}】不存在"
 
+        # 模拟登录
+        try:
             # 特殊站点测试
             if self.special_site_test.get(domain):
                 return self.special_site_test[domain](site_info)
@@ -316,8 +318,7 @@ class SiteChain(ChainBase):
             public = site_info.public
             proxies = settings.PROXY if site_info.proxy else None
             proxy_server = settings.PROXY_SERVER if site_info.proxy else None
-        # 模拟登录
-        try:
+
             # 访问链接
             if render:
                 page_source = PlaywrightHelper().get_page_source(url=site_url,

app/db/message_oper.py

@@ -0,0 +1,57 @@
+import time
+from typing import Optional, Union
+
+from sqlalchemy.orm import Session
+
+from app.db import DbOper
+from app.db.models.message import Message
+from app.schemas import MessageChannel, NotificationType
+
+
+class MessageOper(DbOper):
+    """
+    消息数据管理
+    """
+
+    def __init__(self, db: Session = None):
+        super().__init__(db)
+
+    def add(self,
+            channel: MessageChannel = None,
+            mtype: NotificationType = None,
+            title: str = None,
+            text: str = None,
+            image: str = None,
+            link: str = None,
+            userid: str = None,
+            action: int = 1,
+            **kwargs):
+        """
+        新增媒体服务器数据
+        :param channel: 消息渠道
+        :param mtype: 消息类型
+        :param title: 标题
+        :param text: 文本内容
+        :param image: 图片
+        :param link: 链接
+        :param userid: 用户ID
+        :param action: 消息方向：0-接收息，1-发送消息
+        """
+        kwargs.update({
+            "channel": channel.value if channel else '',
+            "mtype": mtype.value if mtype else '',
+            "title": title,
+            "text": text,
+            "image": image,
+            "link": link,
+            "userid": userid,
+            "action": action,
+            "reg_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        })
+        Message(**kwargs).create(self._db)
+
+    def list_by_page(self, page: int = 1, count: int = 30) -> Optional[str]:
+        """
+        获取媒体服务器数据ID
+        """
+        return Message.list_by_page(self._db, page, count)

app/db/models/downloadhistory.py

@@ -200,6 +200,7 @@ class DownloadFiles(Base):
         result = db.query(DownloadFiles).filter(DownloadFiles.savepath == savepath).all()
         return list(result)
 
+    @staticmethod
     @db_update
     def delete_by_fullpath(db: Session, fullpath: str):
         db.query(DownloadFiles).filter(DownloadFiles.fullpath == fullpath,

app/db/models/message.py

@@ -0,0 +1,36 @@
+from sqlalchemy import Column, Integer, String, Sequence
+from sqlalchemy.orm import Session
+
+from app.db import db_query, Base
+
+
+class Message(Base):
+    """
+    消息表
+    """
+    id = Column(Integer, Sequence('id'), primary_key=True, index=True)
+    # 消息渠道
+    channel = Column(String, nullable=False)
+    # 消息类型
+    mtype = Column(String, nullable=False)
+    # 标题
+    title = Column(String)
+    # 文本内容
+    text = Column(String)
+    # 图片
+    image = Column(String)
+    # 链接
+    link = Column(String)
+    # 用户ID
+    userid = Column(String)
+    # 登记时间
+    reg_time = Column(String)
+    # 消息方向：0-接收息，1-发送消息
+    action = Column(Integer)
+
+    @staticmethod
+    @db_query
+    def list_by_page(db: Session, page: int = 1, count: int = 30):
+        result = db.query(Message).order_by(Message.reg_time.desc()).offset((page - 1) * count).limit(
+            count).all()
+        return list(result)

app/helper/message.py

@@ -1,19 +1,44 @@
+import json
 import queue
+import time
+from typing import Optional, Any
 
 from app.utils.singleton import Singleton
 
 
 class MessageHelper(metaclass=Singleton):
     """
-    消息队列管理器
+    消息队列管理器，包括系统消息和用户消息
     """
     def __init__(self):
-        self.queue = queue.Queue()
+        self.sys_queue = queue.Queue()
+        self.user_queue = queue.Queue()
 
-    def put(self, message: str):
-        self.queue.put(message)
+    def put(self, message: Any, role: str = "sys"):
+        """
+        存消息
+        :param message: 消息
+        :param role: 消息通道 sys/user
+        """
+        if role == "sys":
+            self.sys_queue.put(message)
+        else:
+            if isinstance(message, str):
+                self.user_queue.put(message)
+            elif hasattr(message, "to_dict"):
+                content = message.to_dict()
+                content['date'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+                self.user_queue.put(json.dumps(content))
 
-    def get(self):
-        if not self.queue.empty():
-            return self.queue.get(block=False)
+    def get(self, role: str = "sys") -> Optional[str]:
+        """
+        取消息
+        :param role: 消息通道 sys/user
+        """
+        if role == "sys":
+            if not self.sys_queue.empty():
+                return self.sys_queue.get(block=False)
+        else:
+            if not self.user_queue.empty():
+                return self.user_queue.get(block=False)
         return None

app/modules/douban/__init__.py

@@ -57,7 +57,11 @@ class DoubanModule(_ModuleBase):
         :param cache:    是否使用缓存
         :return: 识别的媒体信息，包括剧集信息
         """
-        if settings.RECOGNIZE_SOURCE != "douban":
+        if not doubanid and not meta:
+            return None
+
+        if meta and not doubanid \
+                and settings.RECOGNIZE_SOURCE != "douban":
             return None
 
         if not meta:

app/modules/filetransfer/__init__.py

@@ -38,12 +38,12 @@ class FileTransferModule(_ModuleBase):
         for path in [settings.DOWNLOAD_PATH,
                      settings.DOWNLOAD_MOVIE_PATH,
                      settings.DOWNLOAD_TV_PATH,
-                     settings.DWONLOAD_ANIME_PATH]:
+                     settings.DOWNLOAD_ANIME_PATH]:
             if not path:
                 continue
             download_path = Path(path)
             if not download_path.exists():
-                return False, f"目录 {download_path} 不存在"
+                return False, f"下载目录 {download_path} 不存在"
             download_paths.append(path)
         # 下载目录的设备ID
         download_devids = [Path(path).stat().st_dev for path in download_paths]
@@ -54,7 +54,7 @@ class FileTransferModule(_ModuleBase):
         for path in settings.LIBRARY_PATHS:
             library_path = Path(path)
             if not library_path.exists():
-                return False, f"目录不存在：{library_path}"
+                return False, f"媒体库目录不存在：{library_path}"
             if settings.DOWNLOADER_MONITOR and settings.TRANSFER_TYPE == "link":
                 if library_path.stat().st_dev not in download_devids:
                     return False, f"媒体库目录 {library_path} " \

app/modules/qbittorrent/qbittorrent.py

@@ -372,6 +372,24 @@ class Qbittorrent:
             logger.error(f"设置速度限制出错：{str(err)}")
             return False
 
+    def get_speed_limit(self):
+        """
+        获取QB速度
+        :return: 返回download_limit 和upload_limit ，默认是0
+        """
+        if not self.qbc:
+            return False
+
+        download_limit = 0
+        upload_limit = 0
+        try:
+            download_limit = self.qbc.transfer.download_limit
+            upload_limit = self.qbc.transfer.upload_limit
+        except Exception as err:
+            logger.error(f"获取速度限制出错：{str(err)}")
+
+        return (download_limit/1024, upload_limit/1024)
+
     def recheck_torrents(self, ids: Union[str, list]) -> bool:
         """
         重新校验种子

app/modules/themoviedb/__init__.py

@@ -67,7 +67,11 @@ class TheMovieDbModule(_ModuleBase):
         :param cache:    是否使用缓存
         :return: 识别的媒体信息，包括剧集信息
         """
-        if settings.RECOGNIZE_SOURCE != "themoviedb":
+        if not tmdbid and not meta:
+            return None
+
+        if meta and not tmdbid \
+                and settings.RECOGNIZE_SOURCE != "themoviedb":
             return None
 
         if not meta:
@@ -182,7 +186,7 @@ class TheMovieDbModule(_ModuleBase):
         :param season:  季号
         """
         # 搜索
-        logger.info(f"开始使用 名称：{name}、年份：{year} 匹配TMDB信息 ...")
+        logger.info(f"开始使用 名称：{name} 年份：{year} 匹配TMDB信息 ...")
         info = self.tmdb.match(name=name,
                                year=year,
                                mtype=mtype,

app/modules/themoviedb/tmdbapi.py

@@ -189,9 +189,16 @@ class TmdbHelper:
                                                   season_year,
                                                   season_number)
             if not info:
+                year_range = [year]
+                if year:
+                    year_range.append(str(int(year) + 1))
+                    year_range.append(str(int(year) - 1))
+                for year in year_range:
                     logger.debug(
                         f"正在识别{mtype.value}：{name}, 年份={year} ...")
                     info = self.__search_tv_by_name(name, year)
+                    if info:
+                        break
             if info:
                 info['media_type'] = MediaType.TV
         # 返回

app/schemas/message.py

@@ -17,6 +17,20 @@ class CommingMessage(BaseModel):
     channel: Optional[MessageChannel] = None
     # 消息体
     text: Optional[str] = None
+    # 时间
+    date: Optional[str] = None
+    # 消息方向
+    action: Optional[int] = 0
+
+    def to_dict(self):
+        """
+        转换为字典
+        """
+        items = self.dict()
+        for k, v in items.items():
+            if isinstance(v, MessageChannel):
+                items[k] = v.value
+        return items
 
 
 class Notification(BaseModel):
@@ -37,6 +51,21 @@ class Notification(BaseModel):
     link: Optional[str] = None
     # 用户ID
     userid: Optional[Union[str, int]] = None
+    # 时间
+    date: Optional[str] = None
+    # 消息方向
+    action: Optional[int] = 1
+
+    def to_dict(self):
+        """
+        转换为字典
+        """
+        items = self.dict()
+        for k, v in items.items():
+            if isinstance(v, MessageChannel) \
+                    or isinstance(v, NotificationType):
+                items[k] = v.value
+        return items
 
 
 class NotificationSwitch(BaseModel):

app/schemas/types.py

@@ -117,3 +117,4 @@ class MessageChannel(Enum):
     Slack = "Slack"
     SynologyChat = "SynologyChat"
     VoceChat = "VoceChat"
+    Web = "Web"

version.py

@@ -1 +1 @@
-APP_VERSION = 'v1.7.1'
+APP_VERSION = 'v1.7.2'