injectlib/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2025-11-25 03:15:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: weibo search #771

Browse Source
This commit is contained in:
程序员阿江(Relakkes)
2025-11-17 17:24:47 +08:00
parent 6dcfd7e0a5
commit 5288bddb42
2 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
media_platform/weibo/client.py
View File

@@ -51,7 +51,7 @@ class WeiboClient:
self.cookie_dict = cookie_dict
self._image_agent_host = "https://i1.wp.com/"
@retry(stop=stop_after_attempt(5), wait=wait_fixed(2))
@retry(stop=stop_after_attempt(5), wait=wait_fixed(3))
async def request(self, method, url, **kwargs) -> Union[Response, Dict]:
enable_return_response = kwargs.pop("return_response", False)
async with httpx.AsyncClient(proxy=self.proxy) as client:
@@ -63,7 +63,11 @@ class WeiboClient:
try:
data: Dict = response.json()
except json.decoder.JSONDecodeError:
# issue: #771 搜索接口会报错432， 多次重试 + 更新 h5 cookies
utils.logger.error(f"[WeiboClient.request] request {method}:{url} err code: {response.status_code} res:{response.text}")
await self.playwright_page.goto(self._host)
await asyncio.sleep(2)
await self.update_cookies(browser_context=self.playwright_page.context)
raise DataFetchError(f"get response code error: {response.status_code}")
ok_code = data.get("ok")

3
media_platform/weibo/core.py
View File

@@ -86,6 +86,7 @@ class WeiboCrawler(AbstractCrawler):
await self.context_page.goto(self.index_url)
await asyncio.sleep(2)
# Create a client to interact with the xiaohongshu website.
self.wb_client = await self.create_weibo_client(httpx_proxy_format)
if not await self.wb_client.pong():
@@ -312,7 +313,7 @@ class WeiboCrawler(AbstractCrawler):
async def create_weibo_client(self, httpx_proxy: Optional[str]) -> WeiboClient:
"""Create xhs client"""
utils.logger.info("[WeiboCrawler.create_weibo_client] Begin create weibo API client ...")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies(urls=[self.mobile_index_url]))
weibo_client_obj = WeiboClient(
proxy=httpx_proxy,
headers={