codfrm/MoviePilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add 站点连通性

Browse Source
This commit is contained in:
jxxghp
2023-06-20 07:15:10 +08:00
parent c200f13dad
commit 35f5f46873
3 changed files with 100 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/api/endpoints/site.py
View File

@ -9,6 +9,7 @@ from app.chain.site import SiteChain
from app.db import get_db
from app.db.models.site import Site
from app.db.models.user import User
from app.db.siteicon_oper import SiteIconOper
from app.db.userauth import get_current_active_user, get_current_active_superuser
router = APIRouter()
@ -97,3 +98,25 @@ async def update_cookie(
return schemas.Response(success=False, message=msg)
else:
return schemas.Response(success=True, message=msg)
@router.get("/test", summary="连接测试", response_model=schemas.Response)
async def test_site(domain: str, _: User = Depends(get_current_active_user)) -> Any:
"""
测试站点是否可用
"""
status, message = SiteChain().test(domain)
return schemas.Response(success=status, message=message)
@router.get("/icon", summary="站点图标", response_model=schemas.Response)
async def site_icon(domain: str, _: User = Depends(get_current_active_user)) -> Any:
"""
获取站点图标：base64或者url
"""
icon = SiteIconOper().get_by_domain(domain)
if not icon:
return schemas.Response(success=False, message="站点图标不存在！")
return schemas.Response(success=True, data={
"icon": icon.base64 if icon.base64 else icon.url
})

26
app/chain/cookiecloud.py
View File

@ -5,6 +5,7 @@ from urllib.parse import urljoin
from lxml import etree
from app.chain import ChainBase
from app.chain.site import SiteChain
from app.core.config import settings
from app.db.siteicon_oper import SiteIconOper
from app.db.site_oper import SiteOper
@ -24,6 +25,7 @@ class CookieCloudChain(ChainBase):
self.siteoper = SiteOper()
self.siteiconoper = SiteIconOper()
self.siteshelper = SitesHelper()
self.sitechain = SiteChain()
self.cookiecloud = CookieCloudHelper(
server=settings.COOKIECLOUD_HOST,
key=settings.COOKIECLOUD_KEY,
@ -58,6 +60,11 @@ class CookieCloudChain(ChainBase):
# 获取站点信息
indexer = self.siteshelper.get_indexer(domain)
if self.siteoper.exists(domain):
# 检查站点连通性
status, msg = self.sitechain.test(domain)
if status:
logger.info(f"站点【{indexer.get('name')}】连通性正常，不同步CookieCloud数据")
continue
# 更新站点Cookie
self.siteoper.update_cookie(domain=domain, cookies=cookie)
_update_count += 1
@ -70,14 +77,17 @@ class CookieCloudChain(ChainBase):
_add_count += 1
# 保存站点图标
if indexer:
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
cookie=cookie,
ua=settings.USER_AGENT)
if icon_url:
self.siteiconoper.update_icon(name=indexer.get("name"),
domain=domain,
icon_url=icon_url,
icon_base64=icon_base64)
site_icon = self.siteiconoper.get_by_domain(domain)
if not site_icon or not site_icon.base64:
logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...")
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
cookie=cookie,
ua=settings.USER_AGENT)
if icon_url:
self.siteiconoper.update_icon(name=indexer.get("name"),
domain=domain,
icon_url=icon_url,
icon_base64=icon_base64)
# 处理完成
ret_msg = f"更新了{_update_count}个站点，新增了{_add_count}个站点"
logger.info(f"CookieCloud同步成功：{ret_msg}")

59
app/chain/site.py
View File

@ -4,8 +4,13 @@ from app.chain import ChainBase
from app.core.config import settings
from app.db.models.site import Site
from app.db.site_oper import SiteOper
from app.helper.browser import PlaywrightHelper
from app.helper.cloudflare import under_challenge
from app.helper.cookie import CookieHelper
from app.log import logger
from app.utils.http import RequestUtils
from app.utils.site import SiteUtils
from app.utils.string import StringUtils
class SiteChain(ChainBase):
@ -21,6 +26,60 @@ class SiteChain(ChainBase):
self._siteoper = SiteOper()
self._cookiehelper = CookieHelper()
def test(self, url: str) -> Tuple[bool, str]:
"""
测试站点是否可用
:param url: 站点域名
:return: (是否可用, 错误信息)
"""
# 检查域名是否可用
domain = StringUtils.get_url_domain(url)
site_info = self._siteoper.get_by_domain(domain)
if not site_info:
return False, f"站点【{url}】不存在"
site_url = site_info.url
site_cookie = site_info.cookie
ua = site_info.ua
render = site_info.render
proxies = settings.PROXY if site_info.proxy else None
proxy_server = settings.PROXY_SERVER if site_info.proxy else None
# 模拟登录
try:
# 访问链接
if render:
page_source = PlaywrightHelper().get_page_source(url=site_url,
cookies=site_cookie,
ua=ua,
proxies=proxy_server)
if not SiteUtils.is_logged_in(page_source):
if under_challenge(page_source):
return False, f"无法通过Cloudflare！"
return False, f"仿真登录失败，Cookie已失效！"
else:
res = RequestUtils(cookies=site_cookie,
ua=ua,
proxies=proxies
).get_res(url=site_url)
# 判断登录状态
if res and res.status_code in [200, 500, 403]:
if not SiteUtils.is_logged_in(res.text):
if under_challenge(res.text):
msg = "站点被Cloudflare防护，请打开站点浏览器仿真"
elif res.status_code == 200:
msg = "Cookie已失效"
else:
msg = f"状态码：{res.status_code}"
return False, f"连接失败，{msg}！"
else:
return True, f"连接成功"
elif res is not None:
return False, f"连接失败，状态码：{res.status_code}！"
else:
return False, f"连接失败，无法打开网站！"
except Exception as e:
return False, f"连接失败：{str(e)}！"
return True, "连接成功"
def remote_list(self, userid: Union[str, int] = None):
"""
查询所有站点，发送消息