优化订阅内存占用
This commit is contained in:
parent
f8e2cdf64b
commit
37a1222ea5
@ -1,11 +1,14 @@
|
|||||||
import json
|
import json
|
||||||
|
import pickle
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
from typing import Dict, List, Optional, Union, Tuple
|
from typing import Dict, List, Optional, Union, Tuple
|
||||||
|
|
||||||
from app.chain import ChainBase
|
from app.chain import ChainBase
|
||||||
from app.chain.download import DownloadChain
|
from app.chain.download import DownloadChain
|
||||||
from app.chain.search import SearchChain
|
from app.chain.search import SearchChain
|
||||||
|
from app.core.config import settings
|
||||||
from app.core.context import TorrentInfo, Context, MediaInfo
|
from app.core.context import TorrentInfo, Context, MediaInfo
|
||||||
from app.core.metainfo import MetaInfo
|
from app.core.metainfo import MetaInfo
|
||||||
from app.db.models.subscribe import Subscribe
|
from app.db.models.subscribe import Subscribe
|
||||||
@ -24,8 +27,7 @@ class SubscribeChain(ChainBase):
|
|||||||
订阅管理处理链
|
订阅管理处理链
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# 站点最新种子缓存 {站点域名: 种子上下文}
|
__cache_path: Path = None
|
||||||
_torrents_cache: Dict[str, List[Context]] = {}
|
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@ -36,6 +38,9 @@ class SubscribeChain(ChainBase):
|
|||||||
self.message = MessageHelper()
|
self.message = MessageHelper()
|
||||||
self.systemconfig = SystemConfigOper()
|
self.systemconfig = SystemConfigOper()
|
||||||
|
|
||||||
|
# 缓存路径
|
||||||
|
self.__cache_path = settings.TEMP_PATH / "__torrents_cache__"
|
||||||
|
|
||||||
def add(self, title: str, year: str,
|
def add(self, title: str, year: str,
|
||||||
mtype: MediaType = None,
|
mtype: MediaType = None,
|
||||||
tmdbid: int = None,
|
tmdbid: int = None,
|
||||||
@ -218,8 +223,9 @@ class SubscribeChain(ChainBase):
|
|||||||
)
|
)
|
||||||
# 打印缺失集信息
|
# 打印缺失集信息
|
||||||
if no_exists and no_exists.get(subscribe.tmdbid):
|
if no_exists and no_exists.get(subscribe.tmdbid):
|
||||||
no_exists_episodes = no_exists.get(subscribe.tmdbid).get(subscribe.season)
|
no_exists_info = no_exists.get(subscribe.tmdbid).get(subscribe.season)
|
||||||
logger.info(f'订阅 {mediainfo.title_year}{meta.season} 缺失集:{no_exists_episodes}')
|
if no_exists_info:
|
||||||
|
logger.info(f'订阅 {mediainfo.title_year}{meta.season} 缺失集:{no_exists_info.episodes}')
|
||||||
# 站点范围
|
# 站点范围
|
||||||
if subscribe.sites:
|
if subscribe.sites:
|
||||||
sites = json.loads(subscribe.sites)
|
sites = json.loads(subscribe.sites)
|
||||||
@ -295,6 +301,9 @@ class SubscribeChain(ChainBase):
|
|||||||
"""
|
"""
|
||||||
刷新站点最新资源
|
刷新站点最新资源
|
||||||
"""
|
"""
|
||||||
|
# 读取缓存
|
||||||
|
torrents_cache: Dict[str, List[Context]] = self.__load_cache()
|
||||||
|
|
||||||
# 所有站点索引
|
# 所有站点索引
|
||||||
indexers = self.siteshelper.get_indexers()
|
indexers = self.siteshelper.get_indexers()
|
||||||
# 配置的索引站点
|
# 配置的索引站点
|
||||||
@ -304,11 +313,10 @@ class SubscribeChain(ChainBase):
|
|||||||
# 未开启的站点不搜索
|
# 未开启的站点不搜索
|
||||||
if config_indexers and str(indexer.get("id")) not in config_indexers:
|
if config_indexers and str(indexer.get("id")) not in config_indexers:
|
||||||
continue
|
continue
|
||||||
logger.info(f'开始刷新站点资源,站点:{indexer.get("name")} ...')
|
logger.info(f'开始刷新 {indexer.get("name")} 最新种子 ...')
|
||||||
domain = StringUtils.get_url_domain(indexer.get("domain"))
|
domain = StringUtils.get_url_domain(indexer.get("domain"))
|
||||||
torrents: List[TorrentInfo] = self.refresh_torrents(site=indexer)
|
torrents: List[TorrentInfo] = self.refresh_torrents(site=indexer)
|
||||||
if torrents:
|
if torrents:
|
||||||
self._torrents_cache[domain] = []
|
|
||||||
# 过滤种子
|
# 过滤种子
|
||||||
result: List[TorrentInfo] = self.filter_torrents(
|
result: List[TorrentInfo] = self.filter_torrents(
|
||||||
rule_string=self.systemconfig.get(SystemConfigKey.FilterRules),
|
rule_string=self.systemconfig.get(SystemConfigKey.FilterRules),
|
||||||
@ -316,7 +324,17 @@ class SubscribeChain(ChainBase):
|
|||||||
if result is not None:
|
if result is not None:
|
||||||
torrents = result
|
torrents = result
|
||||||
if not torrents:
|
if not torrents:
|
||||||
logger.warn(f'{indexer.get("name")} 没有符合过滤条件的资源')
|
logger.warn(f'{indexer.get("name")} 没有符合过滤条件的种子')
|
||||||
|
continue
|
||||||
|
# 过滤出没有处理过的种子
|
||||||
|
torrents = [torrent for torrent in torrents
|
||||||
|
if f'{torrent.title}{torrent.description}'
|
||||||
|
not in [f'{t.torrent_info.title}{t.torrent_info.description}'
|
||||||
|
for t in torrents_cache.get(domain) or []]]
|
||||||
|
if torrents:
|
||||||
|
logger.info(f'{indexer.get("name")} 有 {len(torrents)} 个新种子')
|
||||||
|
else:
|
||||||
|
logger.info(f'{indexer.get("name")} 没有新种子')
|
||||||
continue
|
continue
|
||||||
for torrent in torrents:
|
for torrent in torrents:
|
||||||
logger.info(f'处理资源:{torrent.title} ...')
|
logger.info(f'处理资源:{torrent.title} ...')
|
||||||
@ -333,17 +351,46 @@ class SubscribeChain(ChainBase):
|
|||||||
mediainfo: MediaInfo = self.recognize_media(meta=meta)
|
mediainfo: MediaInfo = self.recognize_media(meta=meta)
|
||||||
if not mediainfo:
|
if not mediainfo:
|
||||||
logger.warn(f'未识别到媒体信息,标题:{torrent.title}')
|
logger.warn(f'未识别到媒体信息,标题:{torrent.title}')
|
||||||
continue
|
# 存储空的媒体信息
|
||||||
|
mediainfo = MediaInfo()
|
||||||
# 上下文
|
# 上下文
|
||||||
context = Context(meta_info=meta, media_info=mediainfo, torrent_info=torrent)
|
context = Context(meta_info=meta, media_info=mediainfo, torrent_info=torrent)
|
||||||
self._torrents_cache[domain].append(context)
|
# 添加到缓存
|
||||||
|
if not torrents_cache.get(domain):
|
||||||
|
torrents_cache[domain] = [context]
|
||||||
|
else:
|
||||||
|
torrents_cache[domain].append(context)
|
||||||
|
# 如果超过了200条则移除最早的一条
|
||||||
|
if len(torrents_cache[domain]) > 200:
|
||||||
|
torrents_cache[domain].pop(0)
|
||||||
|
else:
|
||||||
|
logger.info(f'{indexer.get("name")} 获取到种子')
|
||||||
# 从缓存中匹配订阅
|
# 从缓存中匹配订阅
|
||||||
self.match()
|
self.__match(torrents_cache)
|
||||||
|
# 保存缓存到本地
|
||||||
|
self.__save_cache(torrents_cache)
|
||||||
|
|
||||||
def match(self):
|
def __load_cache(self) -> Dict[str, List[Context]]:
|
||||||
|
"""
|
||||||
|
从本地加载缓存
|
||||||
|
"""
|
||||||
|
if self.__cache_path.exists():
|
||||||
|
return pickle.load(self.__cache_path.open('rb')) or {}
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def __save_cache(self, cache: Dict[str, List[Context]]):
|
||||||
|
"""
|
||||||
|
保存缓存到本地
|
||||||
|
"""
|
||||||
|
pickle.dump(cache, self.__cache_path.open('wb'))
|
||||||
|
|
||||||
|
def __match(self, torrents_cache: Dict[str, List[Context]]):
|
||||||
"""
|
"""
|
||||||
从缓存中匹配订阅,并自动下载
|
从缓存中匹配订阅,并自动下载
|
||||||
"""
|
"""
|
||||||
|
if not torrents_cache:
|
||||||
|
logger.warn('没有缓存资源,无法匹配订阅')
|
||||||
|
return
|
||||||
# 所有订阅
|
# 所有订阅
|
||||||
subscribes = self.subscribehelper.list('R')
|
subscribes = self.subscribehelper.list('R')
|
||||||
# 遍历订阅
|
# 遍历订阅
|
||||||
@ -382,11 +429,12 @@ class SubscribeChain(ChainBase):
|
|||||||
)
|
)
|
||||||
# 打印缺失集信息
|
# 打印缺失集信息
|
||||||
if no_exists and no_exists.get(subscribe.tmdbid):
|
if no_exists and no_exists.get(subscribe.tmdbid):
|
||||||
no_exists_episodes = no_exists.get(subscribe.tmdbid).get(subscribe.season)
|
no_exists_info = no_exists.get(subscribe.tmdbid).get(subscribe.season)
|
||||||
logger.info(f'订阅 {mediainfo.title_year}{meta.season} 缺失集:{no_exists_episodes}')
|
if no_exists_info:
|
||||||
|
logger.info(f'订阅 {mediainfo.title_year}{meta.season} 缺失集:{no_exists_info.episodes}')
|
||||||
# 遍历缓存种子
|
# 遍历缓存种子
|
||||||
_match_context = []
|
_match_context = []
|
||||||
for domain, contexts in self._torrents_cache.items():
|
for domain, contexts in torrents_cache.items():
|
||||||
for context in contexts:
|
for context in contexts:
|
||||||
# 检查是否匹配
|
# 检查是否匹配
|
||||||
torrent_meta = context.meta_info
|
torrent_meta = context.meta_info
|
||||||
@ -418,11 +466,12 @@ class SubscribeChain(ChainBase):
|
|||||||
# 不是缺失的剧集不要
|
# 不是缺失的剧集不要
|
||||||
if no_exists and no_exists.get(subscribe.tmdbid):
|
if no_exists and no_exists.get(subscribe.tmdbid):
|
||||||
# 缺失集
|
# 缺失集
|
||||||
no_exists_episodes = no_exists.get(subscribe.tmdbid).get(subscribe.season)
|
no_exists_info = no_exists.get(subscribe.tmdbid).get(subscribe.season)
|
||||||
# 是否有交集
|
if no_exists_info:
|
||||||
if not set(no_exists_episodes).intersection(set(torrent_meta.episode_list)):
|
# 是否有交集
|
||||||
logger.info(f'{torrent_info.title} 对应剧集 {torrent_meta.episodes} 未包含缺失的剧集')
|
if not set(no_exists_info.episodes).intersection(set(torrent_meta.episode_list)):
|
||||||
continue
|
logger.info(f'{torrent_info.title} 对应剧集 {torrent_meta.episodes} 未包含缺失的剧集')
|
||||||
|
continue
|
||||||
# 过滤掉已经下载的集数
|
# 过滤掉已经下载的集数
|
||||||
if self.__check_subscribe_note(subscribe, torrent_meta.episode_list):
|
if self.__check_subscribe_note(subscribe, torrent_meta.episode_list):
|
||||||
logger.info(f'{torrent_info.title} 对应剧集 {torrent_meta.episodes} 已下载过')
|
logger.info(f'{torrent_info.title} 对应剧集 {torrent_meta.episodes} 已下载过')
|
||||||
|
Loading…
x
Reference in New Issue
Block a user