codfrm/MoviePilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix DirMonitor

Browse Source
This commit is contained in:
jxxghp 2023-08-03 20:35:08 +08:00
parent dcbf9c0b20
commit ddab39da92
11 changed files with 382 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/chain/__init__.py
View File

@ -94,16 +94,6 @@ class ChainBase(metaclass=ABCMeta):
logger.error(f"运行模块 {method} 出错：{module.__class__.__name__} - {err}\n{traceback.print_exc()}")
return result
def prepare_recognize(self, title: str,
subtitle: str = None) -> Tuple[str, str]:
"""
处理各类特别命名，以便识别
:param title: 标题
:param subtitle: 副标题
:return: 处理后的标题、副标题，该方法可被多个模块同时处理
"""
return self.run_module("prepare_recognize", title=title, subtitle=subtitle)
def recognize_media(self, meta: MetaBase = None,
mtype: MediaType = None,
tmdbid: int = None) -> Optional[MediaInfo]:

4
app/chain/media.py
View File

@ -18,10 +18,6 @@ class MediaChain(ChainBase):
根据主副标题识别媒体信息
"""
logger.info(f'开始识别媒体信息，标题：{title}，副标题：{subtitle} ...')
# 识别前预处理
result: Optional[tuple] = self.prepare_recognize(title=title, subtitle=subtitle)
if result:
title, subtitle = result
# 识别元数据
metainfo = MetaInfo(title, subtitle)
# 识别媒体信息

8
app/chain/search.py
View File

@ -157,14 +157,8 @@ class SearchChain(ChainBase):
logger.info(f'{mediainfo.title} 匹配到资源：{torrent.site_name} - {torrent.title}')
_match_torrents.append(torrent)
continue
# 识别前预处理
result: Optional[tuple] = self.prepare_recognize(title=torrent.title, subtitle=torrent.description)
if result:
title, subtitle = result
else:
title, subtitle = torrent.title, torrent.description
# 识别
torrent_meta = MetaInfo(title=title, subtitle=subtitle)
torrent_meta = MetaInfo(title=torrent.title, subtitle=torrent.description)
# 比对类型
if torrent_meta.type == MediaType.TV and mediainfo.type != MediaType.TV:
logger.warn(f'{torrent.site_name} - {torrent.title} 类型不匹配')

13
app/chain/subscribe.py
View File

@ -52,10 +52,6 @@ class SubscribeChain(ChainBase):
识别媒体信息并添加订阅
"""
logger.info(f'开始添加订阅，标题：{title} ...')
# 识别前预处理
result: Optional[tuple] = self.prepare_recognize(title=title)
if result:
title, _ = result
# 识别元数据
metainfo = MetaInfo(title)
if year:
@ -385,15 +381,8 @@ class SubscribeChain(ChainBase):
continue
for torrent in torrents:
logger.info(f'处理资源：{torrent.title} ...')
# 识别前预处理
result: Optional[tuple] = self.prepare_recognize(title=torrent.title,
subtitle=torrent.description)
if result:
title, subtitle = result
else:
title, subtitle = torrent.title, torrent.description
# 识别
meta = MetaInfo(title=title, subtitle=subtitle)
meta = MetaInfo(title=torrent.title, subtitle=torrent.description)
# 识别媒体信息
mediainfo: MediaInfo = self.recognize_media(meta=meta)
if not mediainfo:

14
app/chain/transfer.py
View File

@ -89,16 +89,10 @@ class TransferChain(ChainBase):
self.progress.update(value=processed_num / total_num * 100,
text=f"正在转移 {torrent.title} ...",
key=ProgressKey.FileTransfer)
# 识别前预处理
result: Optional[tuple] = self.prepare_recognize(title=torrent.title)
if result:
title, subtitle = result
else:
title, subtitle = torrent.title, None
# 识别元数据
meta: MetaBase = MetaInfo(title=title, subtitle=subtitle)
meta: MetaBase = MetaInfo(title=torrent.title)
if not meta.name:
logger.error(f'未识别到元数据，标题：{title}')
logger.error(f'未识别到元数据，标题：{torrent.title}')
continue
if not arg_mediainfo:
# 查询下载记录识别情况
@ -110,7 +104,7 @@ class TransferChain(ChainBase):
if mtype == MediaType.TV \
and ((not meta.season_list and downloadhis.seasons)
or (not meta.episode_list and downloadhis.episodes)):
meta = MetaInfo(f"{title} {downloadhis.seasons} {downloadhis.episodes}")
meta = MetaInfo(f"{torrent.title} {downloadhis.seasons} {downloadhis.episodes}")
# 按TMDBID识别
mediainfo = self.recognize_media(mtype=mtype,
tmdbid=downloadhis.tmdbid)
@ -195,7 +189,7 @@ class TransferChain(ChainBase):
)
# 转移完成
self.transfer_completed(hashs=torrent.hash, transinfo=transferinfo)
# 刮剥
# 刮削元数据
self.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo)
# 刷新媒体库
self.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path)

6
app/db/models/transferhistory.py
View File

@ -30,7 +30,7 @@ class TransferHistory(Base):
tvdbid = Column(Integer)
doubanid = Column(String)
# Sxx
seasons = Column(Integer)
seasons = Column(String)
# Exx
episodes = Column(String)
# 海报
@ -59,6 +59,10 @@ class TransferHistory(Base):
def get_by_hash(db: Session, download_hash: str):
return db.query(TransferHistory).filter(TransferHistory.download_hash == download_hash).first()
@staticmethod
def get_by_src(db: Session, src: str):
return db.query(TransferHistory).filter(TransferHistory.src == src).first()
@staticmethod
def statistic(db: Session, days: int = 7):
"""

10
app/db/transferhistory_oper.py
View File

@ -17,6 +17,13 @@ class TransferHistoryOper(DbOper):
"""
return TransferHistory.search_by_title(self._db, title)
def get_by_src(self, src: str) -> Any:
"""
按源查询转移记录
:param src: 数据key
"""
return TransferHistory.get_by_src(self._db, src)
def add(self, **kwargs):
"""
新增转移历史
@ -36,7 +43,8 @@ class TransferHistoryOper(DbOper):
"""
return TransferHistory.statistic(self._db, days)
def get_by(self, mtype: str, title: str, year: int, season=None, episode=None) -> Any:
def get_by(self, mtype: str, title: str, year: int,
season: str = None, episode: str = None) -> Any:
"""
按类型、标题、年份、季集查询转移记录
"""

3
app/plugins/__init__.py
View File

@ -4,6 +4,7 @@ from typing import Any, List, Dict, Tuple
from app.chain import ChainBase
from app.core.config import settings
from app.core.event import EventManager
from app.db.models import Base
from app.db.plugindata_oper import PluginDataOper
from app.db.systemconfig_oper import SystemConfigOper
@ -44,6 +45,8 @@ class _PluginBase(metaclass=ABCMeta):
self.systemconfig = SystemConfigOper()
# 系统消息
self.systemmessage = MessageHelper()
# 事件管理器
self.eventmanager = EventManager()
@abstractmethod
def init_plugin(self, config: dict = None):

358
app/plugins/dirmonitor/__init__.py
View File

@ -1,6 +1,39 @@
import threading
import traceback
from pathlib import Path
from typing import List, Tuple, Dict, Any
from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer
from watchdog.observers.polling import PollingObserver
from app.core.config import settings
from app.core.context import MediaInfo
from app.core.metainfo import MetaInfo
from app.db.transferhistory_oper import TransferHistoryOper
from app.log import logger
from app.plugins import _PluginBase
from app.schemas import MediaType, Notification, NotificationType, TransferInfo
from app.schemas.types import EventType
lock = threading.Lock()
class FileMonitorHandler(FileSystemEventHandler):
"""
目录监控响应类
"""
def __init__(self, monpath: str, sync: Any, **kwargs):
super(FileMonitorHandler, self).__init__(**kwargs)
self._watch_path = monpath
self.sync = sync
def on_created(self, event):
self.sync.file_change_handler(event, "创建", event.src_path)
def on_moved(self, event):
self.sync.file_change_handler(event, "移动", event.dest_path)
class DirMonitor(_PluginBase):
@ -25,19 +58,205 @@ class DirMonitor(_PluginBase):
# 可使用的用户级别
user_level = 1
# 已处理的文件清单
_synced_files = []
# 私有属性
_monitor = None
transferhis = None
_observer = []
_enabled = False
# 模式 compatibility/fast
_mode = "fast"
# 转移方式
_transfer_type = settings.TRANSFER_TYPE
_monitor_dirs = ""
_exclude_keywords = ""
def init_plugin(self, config: dict = None):
self.transferhis = TransferHistoryOper()
# 读取配置
if config:
self._enabled = config.get("enabled")
self._mode = config.get("mode")
self._transfer_type = config.get("transfer_type")
self._monitor_dirs = config.get("monitor_dirs") or ""
self._exclude_keywords = config.get("exclude_keywords") or ""
# 停止现有任务
self.stop_service()
# TODO 启动任务
# 启动任务
monitor_dirs = self._monitor_dirs.split("\n")
if not monitor_dirs:
return
for mon_path in monitor_dirs:
if not mon_path:
continue
# 检查目录是不是媒体库目录的子目录
if Path(mon_path).is_relative_to(settings.LIBRARY_PATH):
logger.warn(f"{mon_path} 是媒体库目录的子目录，无法监控")
self.systemmessage.put(f"{mon_path} 是媒体库目录的子目录，无法监控")
continue
try:
if self._mode == "compatibility":
# 兼容模式，目录同步性能降低且NAS不能休眠，但可以兼容挂载的远程共享目录如SMB
observer = PollingObserver(timeout=10)
else:
# 内部处理系统操作类型选择最优解
observer = Observer(timeout=10)
self._observer.append(observer)
observer.schedule(FileMonitorHandler(mon_path, self), path=mon_path, recursive=True)
observer.daemon = True
observer.start()
logger.info(f"{mon_path} 的目录监控服务启动")
except Exception as e:
err_msg = str(e)
if "inotify" in err_msg and "reached" in err_msg:
logger.warn(f"目录监控服务启动出现异常：{err_msg}，请在宿主机上（不是docker容器内）执行以下命令并重启："
+ """
echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf
echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf
sudo sysctl -p
""")
else:
logger.error(f"{mon_path} 启动目录监控失败：{err_msg}")
self.systemmessage.put(f"{mon_path} 启动目录监控失败：{err_msg}")
def file_change_handler(self, event, text: str, event_path: str):
"""
处理文件变化
:param event: 事件
:param text: 事件描述
:param event_path: 事件文件路径
"""
if not event.is_directory:
# 文件发生变化
file_path = Path(event_path)
try:
if not file_path.exists():
return
logger.debug("文件%s：%s" % (text, event_path))
# 全程加锁
with lock:
if event_path not in self._synced_files:
self._synced_files.append(event_path)
else:
logger.debug("文件已处理过：%s" % event_path)
return
# 命中过滤关键字不处理
if self._exclude_keywords:
for keyword in self._exclude_keywords.split("\n"):
if keyword and keyword in event_path:
logger.debug(f"{event_path} 命中过滤关键字 {keyword}")
return
# 回收站及隐藏的文件不处理
if event_path.find('/@Recycle/') != -1 \
or event_path.find('/#recycle/') != -1 \
or event_path.find('/.') != -1 \
or event_path.find('/@eaDir') != -1:
logger.debug(f"{event_path} 是回收站或隐藏的文件")
return
# 不是媒体文件不处理
if file_path.suffix not in settings.RMT_MEDIAEXT:
logger.debug(f"{event_path} 不是媒体文件")
return
# 查询历史记录，已转移的不处理
if self.transferhis.get_by_src(event_path):
logger.info(f"{event_path} 已整理过")
return
# 文件元数据
file_meta = MetaInfo(title=file_path.name)
# 上级目录元数据
dir_meta = MetaInfo(title=file_path.parent.name)
# 整合元数据
if not file_meta.cn_name and dir_meta.cn_name:
file_meta.cn_name = dir_meta.cn_name
if not file_meta.en_name and dir_meta.en_name:
file_meta.en_name = dir_meta.en_name
if file_meta.type != MediaType.TV and dir_meta.type == MediaType.TV:
file_meta.type = MediaType.TV
if not file_meta.year and dir_meta.year:
file_meta.year = dir_meta.year
if not file_meta.begin_season and dir_meta.begin_season:
file_meta.begin_season = dir_meta.begin_season
if not file_meta.episode_list and dir_meta.episode_list:
file_meta.begin_episode = dir_meta.begin_episode
file_meta.end_episode = dir_meta.end_episode
if not file_meta.name:
logger.warn(f"{file_path.name} 无法识别有效信息")
return
# 识别媒体信息
mediainfo: MediaInfo = self.chain.recognize_media(meta=file_meta)
if not mediainfo:
logger.warn(f'未识别到媒体信息，标题：{file_meta.name}')
self.chain.post_message(Notification(
mtype=NotificationType.Manual,
title=f"{file_path.name} 未识别到媒体信息，无法入库！"
))
return
logger.info(f"{file_path.name} 识别为：{mediainfo.type.value} {mediainfo.title_year}")
# 更新媒体图片
self.chain.obtain_images(mediainfo=mediainfo)
# 转移
transferinfo: TransferInfo = self.chain.transfer(mediainfo=mediainfo, path=file_path)
if not transferinfo or not transferinfo.target_path:
# 转移失败
logger.warn(f"{file_path.name} 入库失败")
self.chain.post_message(Notification(
title=f"{mediainfo.title_year}{file_meta.season_episode} 入库失败！",
text=f"原因：{transferinfo.message if transferinfo else '未知'}",
image=mediainfo.get_message_image()
))
return
# 新增转移成功历史记录
self.transferhis.add(
src=event_path,
dest=str(transferinfo.target_path) if transferinfo else None,
mode=settings.TRANSFER_TYPE,
type=mediainfo.type.value,
category=mediainfo.category,
title=mediainfo.title,
year=mediainfo.year,
tmdbid=mediainfo.tmdb_id,
imdbid=mediainfo.imdb_id,
tvdbid=mediainfo.tvdb_id,
doubanid=mediainfo.douban_id,
seasons=file_meta.season,
episodes=file_meta.episode,
image=mediainfo.get_poster_image(),
status=1
)
# 刮削元数据
self.chain.scrape_metadata(path=transferinfo.target_path, mediainfo=mediainfo)
# 刷新媒体库
self.chain.refresh_mediaserver(mediainfo=mediainfo, file_path=transferinfo.target_path)
# 发送通知
self.chain.__send_transfer_message(meta=file_meta, mediainfo=mediainfo, transferinfo=transferinfo)
# 广播事件
self.eventmanager.send_event(EventType.TransferComplete, {
'meta': file_meta,
'mediainfo': mediainfo,
'transferinfo': transferinfo
})
except Exception as e:
logger.error("目录监控发生错误：%s - %s" % (str(e), traceback.format_exc()))
def get_state(self) -> bool:
return self._enabled
@ -50,7 +269,131 @@ class DirMonitor(_PluginBase):
pass
def get_form(self) -> Tuple[List[dict], Dict[str, Any]]:
pass
return [
{
'component': 'VForm',
'content': [
{
'component': 'VRow',
'content': [
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 6
},
'content': [
{
'component': 'VSwitch',
'props': {
'model': 'enabled',
'label': '启用插件',
}
}
]
}
]
},
{
'component': 'VRow',
'content': [
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 6
},
'content': [
{
'component': 'VSelect',
'props': {
'model': 'mode',
'label': '监控模式',
'items': [
{'title': '兼容模式', 'value': 'compatibility'},
{'title': '性能模式', 'value': 'fast'}
]
}
}
]
},
{
'component': 'VCol',
'props': {
'cols': 12,
'md': 6
},
'content': [
{
'component': 'VSelect',
'props': {
'model': 'transfer_type',
'label': '转移方式',
'items': [
{'title': '移动', 'value': 'move'},
{'title': '复制', 'value': 'copy'},
{'title': '硬链接', 'value': 'link'},
{'title': '软链接', 'value': 'softlink'}
]
}
}
]
}
]
},
{
'component': 'VRow',
'content': [
{
'component': 'VCol',
'props': {
'cols': 12
},
'content': [
{
'component': 'VTextarea',
'props': {
'model': 'monitor_dirs',
'label': '监控目录',
'rows': 5,
'placeholder': '每一行一个目录'
}
}
]
}
]
},
{
'component': 'VRow',
'content': [
{
'component': 'VCol',
'props': {
'cols': 12
},
'content': [
{
'component': 'VTextarea',
'props': {
'model': 'exclude_keywords',
'label': '排除关键词',
'rows': 2,
'placeholder': '每一行一个关键词'
}
}
]
}
]
}
]
}
], {
"enabled": False,
"mode": "fast",
"transfer_type": settings.TRANSFER_TYPE,
"monitor_dirs": "",
"exclude_keywords": ""
}
def get_page(self) -> List[dict]:
pass
@ -59,4 +402,11 @@ class DirMonitor(_PluginBase):
"""
退出插件
"""
pass
if self._observer:
for observer in self._observer:
try:
observer.stop()
observer.join()
except Exception as e:
print(str(e))
self._observer = []

4
app/plugins/libraryscraper/__init__.py
View File

@ -54,8 +54,8 @@ class LibraryScraper(_PluginBase):
if config:
self._enabled = config.get("enabled")
self._cron = config.get("cron")
self._scraper_paths = config.get("scraper_paths")
self._exclude_paths = config.get("exclude_paths")
self._scraper_paths = config.get("scraper_paths") or ""
self._exclude_paths = config.get("exclude_paths") or ""
# 停止现有任务
self.stop_service()

2
requirements.txt
View File

@ -44,3 +44,5 @@ Cython~=0.29.35
tvdb_api~=3.1
psutil~=5.9.4
python_hosts~=1.0.3
tmdbv3api~=1.7.7
watchdog~=3.0.0