From dde3b765730ce9c7f37d1d0db248909966252f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=AE=E5=8F=AE=E5=BD=93?= <604054726@qq.com> Date: Fri, 9 Feb 2024 22:45:58 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AE=A9=E8=87=AA=E5=AE=9A=E4=B9=89=E7=AB=99?= =?UTF-8?q?=E7=82=B9=E5=8F=AF=E8=87=AA=E8=A1=8C=E8=AE=BE=E7=BD=AE:=20?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E7=BB=93=E6=9E=9C=E6=9D=A1=E6=95=B0/?= =?UTF-8?q?=E8=AF=B7=E6=B1=82=E8=B6=85=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/chain/torrents.py | 2 +- app/helper/rss.py | 5 +++-- app/modules/indexer/spider.py | 11 ++++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/app/chain/torrents.py b/app/chain/torrents.py index cfa8df33..5db3ce7f 100644 --- a/app/chain/torrents.py +++ b/app/chain/torrents.py @@ -98,7 +98,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton): if not site.get("rss"): logger.error(f'站点 {domain} 未配置RSS地址!') return [] - rss_items = self.rsshelper.parse(site.get("rss"), True if site.get("proxy") else False) + rss_items = self.rsshelper.parse(site.get("rss"), True if site.get("proxy") else False, timeout=int(site.get("timeout") or 30)) if rss_items is None: # rss过期,尝试保留原配置生成新的rss self.__renew_rss_url(domain=domain, site=site) diff --git a/app/helper/rss.py b/app/helper/rss.py index cd328220..7ce39409 100644 --- a/app/helper/rss.py +++ b/app/helper/rss.py @@ -224,11 +224,12 @@ class RssHelper: } @staticmethod - def parse(url, proxy: bool = False) -> Union[List[dict], None]: + def parse(url, proxy: bool = False, timeout: int = 30) -> Union[List[dict], None]: """ 解析RSS订阅URL,获取RSS中的种子信息 :param url: RSS地址 :param proxy: 是否使用代理 + :param timeout: 请求超时 :return: 种子信息列表,如为None代表Rss过期 """ # 开始处理 @@ -236,7 +237,7 @@ class RssHelper: if not url: return [] try: - ret = RequestUtils(proxies=settings.PROXY if proxy else None).get_res(url) + ret = RequestUtils(proxies=settings.PROXY if proxy else None, timeout=timeout).get_res(url) if not ret: return [] except Exception as err: diff --git a/app/modules/indexer/spider.py b/app/modules/indexer/spider.py index d0d714eb..b3299deb 100644 --- a/app/modules/indexer/spider.py +++ b/app/modules/indexer/spider.py @@ -56,12 +56,14 @@ class TorrentSpider: fields: dict = {} # 页码 page: int = 0 - # 搜索条数 + # 搜索条数, 默认: 100条 result_num: int = 100 # 单个种子信息 torrents_info: dict = {} # 种子列表 torrents_info_array: list = [] + # 搜索超时, 默认: 30秒 + _timeout = 30 def __init__(self, indexer: CommentedMap, @@ -91,6 +93,8 @@ class TorrentSpider: self.fields = indexer.get('torrents').get('fields') self.render = indexer.get('render') self.domain = indexer.get('domain') + self.result_num = int(indexer.get('result_num') or 100) + self._timeout = int(indexer.get('timeout') or 30) self.page = page if self.domain and not str(self.domain).endswith("/"): self.domain = self.domain + "/" @@ -233,14 +237,15 @@ class TorrentSpider: url=searchurl, cookies=self.cookie, ua=self.ua, - proxies=self.proxy_server + proxies=self.proxy_server, + timeout=self._timeout ) else: # requests请求 ret = RequestUtils( ua=self.ua, cookies=self.cookie, - timeout=30, + timeout=self._timeout, referer=self.referer, proxies=self.proxies ).get_res(searchurl, allow_redirects=True)