injectlib/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2025-11-25 11:29:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: move format_proxy_info to utils and update crawler classes to use it

Browse Source
This commit is contained in:
korruz
2025-07-29 14:16:24 +08:00
parent 2935841872
commit 07a6e387ea
8 changed files with 10 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
media_platform/xhs/core.py
View File

@@ -61,7 +61,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
config.IP_PROXY_POOL_COUNT, enable_validate_ip=True
)
ip_proxy_info: IpInfoModel = await ip_proxy_pool.get_proxy()
playwright_proxy_format, httpx_proxy_format = self.format_proxy_info(
playwright_proxy_format, httpx_proxy_format = utils.format_proxy_info(
ip_proxy_info
)
@@ -378,21 +378,6 @@ class XiaoHongShuCrawler(AbstractCrawler):
max_count=CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
)
@staticmethod
def format_proxy_info(
ip_proxy_info: IpInfoModel,
) -> Tuple[Optional[Dict], Optional[Dict]]:
"""format proxy info for playwright and httpx"""
playwright_proxy = {
"server": f"{ip_proxy_info.protocol}{ip_proxy_info.ip}:{ip_proxy_info.port}",
"username": ip_proxy_info.user,
"password": ip_proxy_info.password,
}
httpx_proxy = {
f"{ip_proxy_info.protocol}": f"http://{ip_proxy_info.user}:{ip_proxy_info.password}@{ip_proxy_info.ip}:{ip_proxy_info.port}"
}
return playwright_proxy, httpx_proxy
async def create_xhs_client(self, httpx_proxy: Optional[str]) -> XiaoHongShuClient:
"""Create xhs client"""
utils.logger.info(