injectlib/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2025-11-25 11:29:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(bilibili): process creator videos in batches

Browse Source
This commit is contained in:
mirza-samad-ahmed-baig
2025-07-04 21:04:10 +05:00
parent 66a68fbb13
commit 7edf3bcc15
1 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
media_platform/bilibili/core.py
View File

@@ -290,16 +290,14 @@ class BilibiliCrawler(AbstractCrawler):
"""
ps = 30
pn = 1
video_bvids_list = []
while True:
result = await self.bili_client.get_creator_videos(creator_id, pn, ps)
for video in result["list"]["vlist"]:
video_bvids_list.append(video["bvid"])
if (int(result["page"]["count"]) <= pn * ps):
video_bvids_list = [video["bvid"] for video in result["list"]["vlist"]]
await self.get_specified_videos(video_bvids_list)
if int(result["page"]["count"]) <= pn * ps:
break
await asyncio.sleep(random.random())
pn += 1
await self.get_specified_videos(video_bvids_list)
async def get_specified_videos(self, bvids_list: List[str]):
"""