fix #694 按站点多次检索
This commit is contained in:
jxxghp
parent
dbb3bead6b
commit
2e89eeca2c
@ -197,21 +197,19 @@ class ChainBase(metaclass=ABCMeta):
|
||||
return self.run_module("search_medias", meta=meta)
|
||||
|
||||
def search_torrents(self, site: CommentedMap,
|
||||
mediainfo: MediaInfo,
|
||||
keyword: str = None,
|
||||
page: int = 0,
|
||||
area: str = "title") -> List[TorrentInfo]:
|
||||
keywords: List[str],
|
||||
mtype: MediaType = None,
|
||||
page: int = 0) -> List[TorrentInfo]:
|
||||
"""
|
||||
搜索一个站点的种子资源
|
||||
:param site: 站点
|
||||
:param mediainfo: 识别的媒体信息
|
||||
:param keyword: 搜索关键词，如有按关键词搜索，否则按媒体信息名称搜索
|
||||
:param keywords: 搜索关键词列表
|
||||
:param mtype: 媒体类型
|
||||
:param page: 页码
|
||||
:param area: 搜索区域
|
||||
:reutrn: 资源列表
|
||||
"""
|
||||
return self.run_module("search_torrents", mediainfo=mediainfo, site=site,
|
||||
keyword=keyword, page=page, area=area)
|
||||
return self.run_module("search_torrents", site=site, keywords=keywords,
|
||||
mtype=mtype, page=page)
|
||||
|
||||
def refresh_torrents(self, site: CommentedMap) -> List[TorrentInfo]:
|
||||
"""
|
||||
|
||||
@ -62,7 +62,7 @@ class SearchChain(ChainBase):
|
||||
else:
|
||||
logger.info(f'开始浏览资源，站点：{site} ...')
|
||||
# 搜索
|
||||
return self.__search_all_sites(keyword=title, sites=[site] if site else None, page=page) or []
|
||||
return self.__search_all_sites(keywords=[title], sites=[site] if site else None, page=page) or []
|
||||
|
||||
def last_search_results(self) -> List[Context]:
|
||||
"""
|
||||
@ -117,16 +117,12 @@ class SearchChain(ChainBase):
|
||||
else:
|
||||
keywords = [mediainfo.title]
|
||||
# 执行搜索
|
||||
torrents: List[TorrentInfo] = []
|
||||
for keyword in keywords:
|
||||
torrents = self.__search_all_sites(
|
||||
mediainfo=mediainfo,
|
||||
keyword=keyword,
|
||||
sites=sites,
|
||||
area=area
|
||||
)
|
||||
if torrents:
|
||||
break
|
||||
torrents: List[TorrentInfo] = self.__search_all_sites(
|
||||
mediainfo=mediainfo,
|
||||
keywords=keywords,
|
||||
sites=sites,
|
||||
area=area
|
||||
)
|
||||
if not torrents:
|
||||
logger.warn(f'{keyword or mediainfo.title} 未搜索到资源')
|
||||
return []
|
||||
@ -241,15 +237,15 @@ class SearchChain(ChainBase):
|
||||
# 返回
|
||||
return contexts
|
||||
|
||||
def __search_all_sites(self, mediainfo: Optional[MediaInfo] = None,
|
||||
keyword: str = None,
|
||||
def __search_all_sites(self, keywords: List[str],
|
||||
mediainfo: Optional[MediaInfo] = None,
|
||||
sites: List[int] = None,
|
||||
page: int = 0,
|
||||
area: str = "title") -> Optional[List[TorrentInfo]]:
|
||||
"""
|
||||
多线程搜索多个站点
|
||||
:param mediainfo: 识别的媒体信息
|
||||
:param keyword: 搜索关键词，如有按关键词搜索，否则按媒体信息名称搜索
|
||||
:param keywords: 搜索关键词列表
|
||||
:param sites: 指定站点ID列表，如有则只搜索指定站点，否则搜索所有站点
|
||||
:param page: 搜索页码
|
||||
:param area: 搜索区域 title or imdbid
|
||||
@ -291,8 +287,18 @@ class SearchChain(ChainBase):
|
||||
executor = ThreadPoolExecutor(max_workers=len(indexer_sites))
|
||||
all_task = []
|
||||
for site in indexer_sites:
|
||||
task = executor.submit(self.search_torrents, mediainfo=mediainfo,
|
||||
site=site, keyword=keyword, page=page, area=area)
|
||||
if area == "imdbid":
|
||||
# 搜索IMDBID
|
||||
task = executor.submit(self.search_torrents, site=site,
|
||||
keywords=[mediainfo.imdb_id] if mediainfo else None,
|
||||
mtype=mediainfo.type if mediainfo else None,
|
||||
page=page)
|
||||
else:
|
||||
# 搜索标题
|
||||
task = executor.submit(self.search_torrents, site=site,
|
||||
keywords=keywords,
|
||||
mtype=mediainfo.type if mediainfo else None,
|
||||
page=page)
|
||||
all_task.append(task)
|
||||
# 结果集
|
||||
results = []
|
||||
@ -303,7 +309,7 @@ class SearchChain(ChainBase):
|
||||
results.extend(result)
|
||||
logger.info(f"站点搜索进度：{finish_count} / {total_num}")
|
||||
self.progress.update(value=finish_count / total_num * 100,
|
||||
text=f"正在搜索{keyword or ''}，已完成 {finish_count} / {total_num} 个站点 ...",
|
||||
text=f"正在搜索{keywords or ''}，已完成 {finish_count} / {total_num} 个站点 ...",
|
||||
key=ProgressKey.Search)
|
||||
# 计算耗时
|
||||
end_time = datetime.now()
|
||||
|
||||
@ -3,7 +3,7 @@ from typing import List, Optional, Tuple, Union
|
||||
|
||||
from ruamel.yaml import CommentedMap
|
||||
|
||||
from app.core.context import MediaInfo, TorrentInfo
|
||||
from app.core.context import TorrentInfo
|
||||
from app.log import logger
|
||||
from app.modules import _ModuleBase
|
||||
from app.modules.indexer.mtorrent import MTorrentSpider
|
||||
@ -28,69 +28,71 @@ class IndexerModule(_ModuleBase):
|
||||
def init_setting(self) -> Tuple[str, Union[str, bool]]:
|
||||
return "INDEXER", "builtin"
|
||||
|
||||
def search_torrents(self, site: CommentedMap, mediainfo: MediaInfo = None,
|
||||
keyword: str = None, page: int = 0, area: str = "title") -> List[TorrentInfo]:
|
||||
def search_torrents(self, site: CommentedMap,
|
||||
keywords: List[str] = None,
|
||||
mtype: MediaType = None,
|
||||
page: int = 0) -> List[TorrentInfo]:
|
||||
"""
|
||||
搜索一个站点
|
||||
:param mediainfo: 识别的媒体信息
|
||||
:param site: 站点
|
||||
:param keyword: 搜索关键词，如有按关键词搜索，否则按媒体信息名称搜索
|
||||
:param keywords: 搜索关键词列表
|
||||
:param mtype: 媒体类型
|
||||
:param page: 页码
|
||||
:param area: 搜索区域 title or imdbid
|
||||
:return: 资源列表
|
||||
"""
|
||||
# 确认搜索的名字
|
||||
if keyword:
|
||||
search_word = keyword
|
||||
elif mediainfo:
|
||||
search_word = mediainfo.title
|
||||
else:
|
||||
search_word = None
|
||||
|
||||
if search_word \
|
||||
and site.get('language') == "en" \
|
||||
and StringUtils.is_chinese(search_word):
|
||||
# 不支持中文
|
||||
logger.warn(f"{site.get('name')} 不支持中文搜索")
|
||||
return []
|
||||
|
||||
# 去除搜索关键字中的特殊字符
|
||||
if search_word:
|
||||
search_word = StringUtils.clear(search_word, replace_word=" ", allow_space=True)
|
||||
if not keywords:
|
||||
# 浏览种子页
|
||||
keywords = [None]
|
||||
|
||||
# 开始索引
|
||||
result_array = []
|
||||
# 开始计时
|
||||
start_time = datetime.now()
|
||||
try:
|
||||
imdbid = mediainfo.imdb_id if mediainfo and area == "imdbid" else None
|
||||
if site.get('parser') == "TNodeSpider":
|
||||
error_flag, result_array = TNodeSpider(site).search(
|
||||
keyword=search_word,
|
||||
imdbid=imdbid,
|
||||
page=page
|
||||
)
|
||||
elif site.get('parser') == "TorrentLeech":
|
||||
error_flag, result_array = TorrentLeech(site).search(
|
||||
keyword=search_word,
|
||||
page=page
|
||||
)
|
||||
elif site.get('parser') == "mTorrent":
|
||||
error_flag, result_array = MTorrentSpider(site).search(
|
||||
keyword=search_word,
|
||||
mtype=mediainfo.type if mediainfo else None,
|
||||
page=page
|
||||
)
|
||||
else:
|
||||
error_flag, result_array = self.__spider_search(
|
||||
keyword=search_word,
|
||||
imdbid=imdbid,
|
||||
indexer=site,
|
||||
mtype=mediainfo.type if mediainfo else None,
|
||||
page=page
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"{site.get('name')} 搜索出错：{err}")
|
||||
|
||||
# 搜索多个关键字
|
||||
for search_word in keywords:
|
||||
# 可能为关键字或ttxxxx
|
||||
if search_word \
|
||||
and site.get('language') == "en" \
|
||||
and StringUtils.is_chinese(search_word):
|
||||
# 不支持中文
|
||||
logger.warn(f"{site.get('name')} 不支持中文搜索")
|
||||
continue
|
||||
|
||||
# 去除搜索关键字中的特殊字符
|
||||
if search_word:
|
||||
search_word = StringUtils.clear(search_word, replace_word=" ", allow_space=True)
|
||||
|
||||
try:
|
||||
if site.get('parser') == "TNodeSpider":
|
||||
error_flag, result_array = TNodeSpider(site).search(
|
||||
keyword=search_word,
|
||||
page=page
|
||||
)
|
||||
elif site.get('parser') == "TorrentLeech":
|
||||
error_flag, result_array = TorrentLeech(site).search(
|
||||
keyword=search_word,
|
||||
page=page
|
||||
)
|
||||
elif site.get('parser') == "mTorrent":
|
||||
error_flag, result_array = MTorrentSpider(site).search(
|
||||
keyword=search_word,
|
||||
mtype=mtype,
|
||||
page=page
|
||||
)
|
||||
else:
|
||||
error_flag, result_array = self.__spider_search(
|
||||
search_word=search_word,
|
||||
indexer=site,
|
||||
mtype=mtype,
|
||||
page=page
|
||||
)
|
||||
# 有结果后停止
|
||||
if result_array:
|
||||
break
|
||||
except Exception as err:
|
||||
logger.error(f"{site.get('name')} 搜索出错：{err}")
|
||||
|
||||
# 索引花费的时间
|
||||
seconds = round((datetime.now() - start_time).seconds, 1)
|
||||
@ -112,15 +114,13 @@ class IndexerModule(_ModuleBase):
|
||||
|
||||
@staticmethod
|
||||
def __spider_search(indexer: CommentedMap,
|
||||
keyword: str = None,
|
||||
imdbid: str = None,
|
||||
search_word: str = None,
|
||||
mtype: MediaType = None,
|
||||
page: int = 0) -> (bool, List[dict]):
|
||||
"""
|
||||
根据关键字搜索单个站点
|
||||
:param: indexer: 站点配置
|
||||
:param: keyword: 关键字
|
||||
:param: imdbid: imdbid
|
||||
:param: search_word: 关键字
|
||||
:param: page: 页码
|
||||
:param: mtype: 媒体类型
|
||||
:param: timeout: 超时时间
|
||||
@ -128,8 +128,7 @@ class IndexerModule(_ModuleBase):
|
||||
"""
|
||||
_spider = TorrentSpider(indexer=indexer,
|
||||
mtype=mtype,
|
||||
keyword=keyword,
|
||||
imdbid=imdbid,
|
||||
keyword=search_word,
|
||||
page=page)
|
||||
|
||||
return _spider.is_error, _spider.get_torrents()
|
||||
|
||||
@ -40,8 +40,6 @@ class TorrentSpider:
|
||||
referer: str = None
|
||||
# 搜索关键字
|
||||
keyword: str = None
|
||||
# 搜索IMDBID
|
||||
imdbid: str = None
|
||||
# 媒体类型
|
||||
mtype: MediaType = None
|
||||
# 搜索路径、方式配置
|
||||
@ -68,7 +66,6 @@ class TorrentSpider:
|
||||
def __init__(self,
|
||||
indexer: CommentedMap,
|
||||
keyword: [str, list] = None,
|
||||
imdbid: str = None,
|
||||
page: int = 0,
|
||||
referer: str = None,
|
||||
mtype: MediaType = None):
|
||||
@ -76,7 +73,6 @@ class TorrentSpider:
|
||||
设置查询参数
|
||||
:param indexer: 索引器
|
||||
:param keyword: 搜索关键字，如果数组则为批量搜索
|
||||
:param imdbid: IMDB ID
|
||||
:param page: 页码
|
||||
:param referer: Referer
|
||||
:param mtype: 媒体类型
|
||||
@ -84,7 +80,6 @@ class TorrentSpider:
|
||||
if not indexer:
|
||||
return
|
||||
self.keyword = keyword
|
||||
self.imdbid = imdbid
|
||||
self.mtype = mtype
|
||||
self.indexerid = indexer.get('id')
|
||||
self.indexername = indexer.get('name')
|
||||
@ -159,20 +154,17 @@ class TorrentSpider:
|
||||
# 搜索URL
|
||||
indexer_params = self.search.get("params") or {}
|
||||
if indexer_params:
|
||||
# 支持IMDBID时优先使用IMDBID搜索
|
||||
search_area = indexer_params.get("search_area") or 0
|
||||
if self.imdbid and search_area:
|
||||
search_word = self.imdbid
|
||||
else:
|
||||
search_word = self.keyword
|
||||
# 不启用IMDBID搜索时需要将search_area移除
|
||||
if search_area:
|
||||
indexer_params.pop('search_area')
|
||||
search_area = indexer_params.get('search_area')
|
||||
# search_area非0表示支持imdbid搜索
|
||||
if (search_area and
|
||||
(not self.keyword or not self.keyword.startswith('tt'))):
|
||||
# 支持imdbid搜索，但关键字不是imdbid时，不启用imdbid搜索
|
||||
indexer_params.pop('search_area')
|
||||
# 变量字典
|
||||
inputs_dict = {
|
||||
"keyword": search_word
|
||||
}
|
||||
# 查询参数
|
||||
# 查询参数，默认查询标题
|
||||
params = {
|
||||
"search_mode": search_mode,
|
||||
"search_area": 0,
|
||||
|
||||
@ -49,16 +49,16 @@ class TNodeSpider:
|
||||
if csrf_token:
|
||||
self._token = csrf_token.group(1)
|
||||
|
||||
def search(self, keyword: str, imdbid: str = None, page: int = 0) -> Tuple[bool, List[dict]]:
|
||||
def search(self, keyword: str, page: int = 0) -> Tuple[bool, List[dict]]:
|
||||
if not self._token:
|
||||
logger.warn(f"{self._name} 未获取到token，无法搜索")
|
||||
return True, []
|
||||
search_type = "imdbid" if imdbid else "title"
|
||||
search_type = "imdbid" if (keyword and keyword.startswith('tt')) else "title"
|
||||
params = {
|
||||
"page": int(page) + 1,
|
||||
"size": self._size,
|
||||
"type": search_type,
|
||||
"keyword": imdbid or keyword or "",
|
||||
"keyword": keyword or "",
|
||||
"sorter": "id",
|
||||
"order": "desc",
|
||||
"tags": [],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user