From 5288bddb4203cbf017c1e601bfe995452594ed36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Mon, 17 Nov 2025 17:24:47 +0800 Subject: [PATCH] refactor: weibo search #771 --- media_platform/weibo/client.py | 6 +++++- media_platform/weibo/core.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/media_platform/weibo/client.py b/media_platform/weibo/client.py index 0f89993..191c9b2 100644 --- a/media_platform/weibo/client.py +++ b/media_platform/weibo/client.py @@ -51,7 +51,7 @@ class WeiboClient: self.cookie_dict = cookie_dict self._image_agent_host = "https://i1.wp.com/" - @retry(stop=stop_after_attempt(5), wait=wait_fixed(2)) + @retry(stop=stop_after_attempt(5), wait=wait_fixed(3)) async def request(self, method, url, **kwargs) -> Union[Response, Dict]: enable_return_response = kwargs.pop("return_response", False) async with httpx.AsyncClient(proxy=self.proxy) as client: @@ -63,7 +63,11 @@ class WeiboClient: try: data: Dict = response.json() except json.decoder.JSONDecodeError: + # issue: #771 搜索接口会报错432， 多次重试 + 更新 h5 cookies utils.logger.error(f"[WeiboClient.request] request {method}:{url} err code: {response.status_code} res:{response.text}") + await self.playwright_page.goto(self._host) + await asyncio.sleep(2) + await self.update_cookies(browser_context=self.playwright_page.context) raise DataFetchError(f"get response code error: {response.status_code}") ok_code = data.get("ok") diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index 6c05b4e..93f7ba4 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -86,6 +86,7 @@ class WeiboCrawler(AbstractCrawler): await self.context_page.goto(self.index_url) await asyncio.sleep(2) + # Create a client to interact with the xiaohongshu website. self.wb_client = await self.create_weibo_client(httpx_proxy_format) if not await self.wb_client.pong(): @@ -312,7 +313,7 @@ class WeiboCrawler(AbstractCrawler): async def create_weibo_client(self, httpx_proxy: Optional[str]) -> WeiboClient: """Create xhs client""" utils.logger.info("[WeiboCrawler.create_weibo_client] Begin create weibo API client ...") - cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies()) + cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies(urls=[self.mobile_index_url])) weibo_client_obj = WeiboClient( proxy=httpx_proxy, headers={