361 lines
12 KiB
Python
361 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
||
import base64
|
||
import json
|
||
import re
|
||
from abc import ABCMeta, abstractmethod
|
||
from typing import Optional
|
||
from urllib.parse import urljoin, urlsplit
|
||
|
||
import requests
|
||
from lxml import etree
|
||
from requests import Session
|
||
|
||
from app.core import settings
|
||
from app.helper.cloudflare import under_challenge
|
||
from app.log import logger
|
||
from app.utils.http import RequestUtils
|
||
from app.utils.types import SiteSchema
|
||
|
||
SITE_BASE_ORDER = 1000
|
||
|
||
|
||
class ISiteUserInfo(metaclass=ABCMeta):
|
||
# 站点模版
|
||
schema = SiteSchema.NexusPhp
|
||
# 站点解析时判断顺序，值越小越先解析
|
||
order = SITE_BASE_ORDER
|
||
|
||
def __init__(self, site_name: str,
|
||
url: str,
|
||
site_cookie: str,
|
||
index_html: str,
|
||
session: Session = None,
|
||
ua: str = None,
|
||
emulate: bool = False,
|
||
proxy: bool = None):
|
||
super().__init__()
|
||
# 站点信息
|
||
self.site_name = None
|
||
self.site_url = None
|
||
self.site_favicon = None
|
||
# 用户信息
|
||
self.username = None
|
||
self.userid = None
|
||
# 未读消息
|
||
self.message_unread = 0
|
||
self.message_unread_contents = []
|
||
|
||
# 流量信息
|
||
self.upload = 0
|
||
self.download = 0
|
||
self.ratio = 0
|
||
|
||
# 种子信息
|
||
self.seeding = 0
|
||
self.leeching = 0
|
||
self.uploaded = 0
|
||
self.completed = 0
|
||
self.incomplete = 0
|
||
self.seeding_size = 0
|
||
self.leeching_size = 0
|
||
self.uploaded_size = 0
|
||
self.completed_size = 0
|
||
self.incomplete_size = 0
|
||
# 做种人数, 种子大小
|
||
self.seeding_info = []
|
||
|
||
# 用户详细信息
|
||
self.user_level = None
|
||
self.join_at = None
|
||
self.bonus = 0.0
|
||
|
||
# 错误信息
|
||
self.err_msg = None
|
||
# 内部数据
|
||
self._base_url = None
|
||
self._site_cookie = None
|
||
self._index_html = None
|
||
self._addition_headers = None
|
||
|
||
# 站点页面
|
||
self._brief_page = "index.php"
|
||
self._user_detail_page = "userdetails.php?id="
|
||
self._user_traffic_page = "index.php"
|
||
self._torrent_seeding_page = "getusertorrentlistajax.php?userid="
|
||
self._user_mail_unread_page = "messages.php?action=viewmailbox&box=1&unread=yes"
|
||
self._sys_mail_unread_page = "messages.php?action=viewmailbox&box=-2&unread=yes"
|
||
self._torrent_seeding_params = None
|
||
self._torrent_seeding_headers = None
|
||
|
||
split_url = urlsplit(url)
|
||
self.site_name = site_name
|
||
self.site_url = url
|
||
self._base_url = f"{split_url.scheme}://{split_url.netloc}"
|
||
self._favicon_url = urljoin(self._base_url, "favicon.ico")
|
||
self.site_favicon = ""
|
||
self._site_cookie = site_cookie
|
||
self._index_html = index_html
|
||
self._session = session if session else requests.Session()
|
||
self._ua = ua
|
||
|
||
self._emulate = emulate
|
||
self._proxy = proxy
|
||
|
||
def site_schema(self):
|
||
"""
|
||
站点解析模型
|
||
:return: 站点解析模型
|
||
"""
|
||
return self.schema
|
||
|
||
@classmethod
|
||
def match(cls, html_text: str) -> bool:
|
||
"""
|
||
是否匹配当前解析模型
|
||
:param html_text: 站点首页html
|
||
:return: 是否匹配
|
||
"""
|
||
pass
|
||
|
||
def parse(self):
|
||
"""
|
||
解析站点信息
|
||
:return:
|
||
"""
|
||
self._parse_favicon(self._index_html)
|
||
if not self._parse_logged_in(self._index_html):
|
||
return
|
||
|
||
self._parse_site_page(self._index_html)
|
||
self._parse_user_base_info(self._index_html)
|
||
self._pase_unread_msgs()
|
||
if self._user_traffic_page:
|
||
self._parse_user_traffic_info(self._get_page_content(urljoin(self._base_url, self._user_traffic_page)))
|
||
if self._user_detail_page:
|
||
self._parse_user_detail_info(self._get_page_content(urljoin(self._base_url, self._user_detail_page)))
|
||
|
||
self._parse_seeding_pages()
|
||
self.seeding_info = json.dumps(self.seeding_info)
|
||
|
||
def _pase_unread_msgs(self):
|
||
"""
|
||
解析所有未读消息标题和内容
|
||
:return:
|
||
"""
|
||
unread_msg_links = []
|
||
if self.message_unread > 0:
|
||
links = {self._user_mail_unread_page, self._sys_mail_unread_page}
|
||
for link in links:
|
||
if not link:
|
||
continue
|
||
|
||
msg_links = []
|
||
next_page = self._parse_message_unread_links(
|
||
self._get_page_content(urljoin(self._base_url, link)), msg_links)
|
||
while next_page:
|
||
next_page = self._parse_message_unread_links(
|
||
self._get_page_content(urljoin(self._base_url, next_page)), msg_links)
|
||
|
||
unread_msg_links.extend(msg_links)
|
||
|
||
for msg_link in unread_msg_links:
|
||
logger.debug(f"{self.site_name} 信息链接 {msg_link}")
|
||
head, date, content = self._parse_message_content(self._get_page_content(urljoin(self._base_url, msg_link)))
|
||
logger.debug(f"{self.site_name} 标题 {head} 时间 {date} 内容 {content}")
|
||
self.message_unread_contents.append((head, date, content))
|
||
|
||
def _parse_seeding_pages(self):
|
||
if self._torrent_seeding_page:
|
||
# 第一页
|
||
next_page = self._parse_user_torrent_seeding_info(
|
||
self._get_page_content(urljoin(self._base_url, self._torrent_seeding_page),
|
||
self._torrent_seeding_params,
|
||
self._torrent_seeding_headers))
|
||
|
||
# 其他页处理
|
||
while next_page:
|
||
next_page = self._parse_user_torrent_seeding_info(
|
||
self._get_page_content(urljoin(urljoin(self._base_url, self._torrent_seeding_page), next_page),
|
||
self._torrent_seeding_params,
|
||
self._torrent_seeding_headers),
|
||
multi_page=True)
|
||
|
||
@staticmethod
|
||
def _prepare_html_text(html_text):
|
||
"""
|
||
处理掉HTML中的干扰部分
|
||
"""
|
||
return re.sub(r"#\d+", "", re.sub(r"\d+px", "", html_text))
|
||
|
||
@abstractmethod
|
||
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
|
||
"""
|
||
获取未阅读消息链接
|
||
:param html_text:
|
||
:return:
|
||
"""
|
||
pass
|
||
|
||
def _parse_favicon(self, html_text):
|
||
"""
|
||
解析站点favicon,返回base64 fav图标
|
||
:param html_text:
|
||
:return:
|
||
"""
|
||
html = etree.HTML(html_text)
|
||
if html:
|
||
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
|
||
if fav_link:
|
||
self._favicon_url = urljoin(self._base_url, fav_link[0])
|
||
|
||
res = RequestUtils(cookies=self._site_cookie, session=self._session, timeout=60, headers=self._ua).get_res(
|
||
url=self._favicon_url)
|
||
if res:
|
||
self.site_favicon = base64.b64encode(res.content).decode()
|
||
|
||
def _get_page_content(self, url, params=None, headers=None):
|
||
"""
|
||
:param url: 网页地址
|
||
:param params: post参数
|
||
:param headers: 额外的请求头
|
||
:return:
|
||
"""
|
||
req_headers = None
|
||
proxies = settings.PROXY if self._proxy else None
|
||
if self._ua or headers or self._addition_headers:
|
||
req_headers = {}
|
||
if headers:
|
||
req_headers.update(headers)
|
||
|
||
req_headers.update({
|
||
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
||
"User-Agent": f"{self._ua}"
|
||
})
|
||
|
||
if self._addition_headers:
|
||
req_headers.update(self._addition_headers)
|
||
|
||
if params:
|
||
res = RequestUtils(cookies=self._site_cookie,
|
||
session=self._session,
|
||
timeout=60,
|
||
proxies=proxies,
|
||
headers=req_headers).post_res(url=url, data=params)
|
||
else:
|
||
res = RequestUtils(cookies=self._site_cookie,
|
||
session=self._session,
|
||
timeout=60,
|
||
proxies=proxies,
|
||
headers=req_headers).get_res(url=url)
|
||
if res is not None and res.status_code in (200, 500, 403):
|
||
# 如果cloudflare 有防护，尝试使用浏览器仿真
|
||
if under_challenge(res.text):
|
||
logger.warn(
|
||
f"{self.site_name} 检测到Cloudflare，请更新Cookie和UA")
|
||
return ""
|
||
if "charset=utf-8" in res.text or "charset=UTF-8" in res.text:
|
||
res.encoding = "UTF-8"
|
||
else:
|
||
res.encoding = res.apparent_encoding
|
||
return res.text
|
||
|
||
return ""
|
||
|
||
@abstractmethod
|
||
def _parse_site_page(self, html_text: str):
|
||
"""
|
||
解析站点相关信息页面
|
||
:param html_text:
|
||
:return:
|
||
"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
def _parse_user_base_info(self, html_text: str):
|
||
"""
|
||
解析用户基础信息
|
||
:param html_text:
|
||
:return:
|
||
"""
|
||
pass
|
||
|
||
def _parse_logged_in(self, html_text):
|
||
"""
|
||
解析用户是否已经登陆
|
||
:param html_text:
|
||
:return: True/False
|
||
"""
|
||
logged_in = self.is_logged_in(html_text)
|
||
if not logged_in:
|
||
self.err_msg = "未检测到已登陆，请检查cookies是否过期"
|
||
logger.warn(f"{self.site_name} 未登录，跳过后续操作")
|
||
|
||
return logged_in
|
||
|
||
@abstractmethod
|
||
def _parse_user_traffic_info(self, html_text: str):
|
||
"""
|
||
解析用户的上传，下载，分享率等信息
|
||
:param html_text:
|
||
:return:
|
||
"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]:
|
||
"""
|
||
解析用户的做种相关信息
|
||
:param html_text:
|
||
:param multi_page: 是否多页数据
|
||
:return: 下页地址
|
||
"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
def _parse_user_detail_info(self, html_text: str):
|
||
"""
|
||
解析用户的详细信息
|
||
加入时间/等级/魔力值等
|
||
:param html_text:
|
||
:return:
|
||
"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
def _parse_message_content(self, html_text):
|
||
"""
|
||
解析短消息内容
|
||
:param html_text:
|
||
:return: head: message, date: time, content: message content
|
||
"""
|
||
pass
|
||
|
||
@classmethod
|
||
def is_logged_in(cls, html_text: str) -> bool:
|
||
"""
|
||
判断站点是否已经登陆
|
||
:param html_text:
|
||
:return:
|
||
"""
|
||
html = etree.HTML(html_text)
|
||
if not html:
|
||
return False
|
||
# 存在明显的密码输入框，说明未登录
|
||
if html.xpath("//input[@type='password']"):
|
||
return False
|
||
# 是否存在登出和用户面板等链接
|
||
xpaths = ['//a[contains(@href, "logout")'
|
||
' or contains(@data-url, "logout")'
|
||
' or contains(@href, "mybonus") '
|
||
' or contains(@onclick, "logout")'
|
||
' or contains(@href, "usercp")]',
|
||
'//form[contains(@action, "logout")]']
|
||
for xpath in xpaths:
|
||
if html.xpath(xpath):
|
||
return True
|
||
user_info_div = html.xpath('//div[@class="user-info-side"]')
|
||
if user_info_div:
|
||
return True
|
||
|
||
return False
|