injectlib/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2025-11-26 03:45:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(bilibili): process creator videos in batches

Browse Source
This commit is contained in:
mirza-samad-ahmed-baig
2025-07-04 21:04:10 +05:00
parent 66a68fbb13
commit 7edf3bcc15
1 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
media_platform/bilibili/core.py
View File

@@ -290,16 +290,14 @@ class BilibiliCrawler(AbstractCrawler):
""" """
ps = 30 ps = 30
pn = 1 pn = 1
video_bvids_list = []
while True: while True:
result = await self.bili_client.get_creator_videos(creator_id, pn, ps) result = await self.bili_client.get_creator_videos(creator_id, pn, ps)
for video in result["list"]["vlist"]: video_bvids_list = [video["bvid"] for video in result["list"]["vlist"]]
video_bvids_list.append(video["bvid"]) await self.get_specified_videos(video_bvids_list)
if (int(result["page"]["count"]) <= pn * ps): if int(result["page"]["count"]) <= pn * ps:
break break
await asyncio.sleep(random.random()) await asyncio.sleep(random.random())
pn += 1 pn += 1
await self.get_specified_videos(video_bvids_list)
async def get_specified_videos(self, bvids_list: List[str]): async def get_specified_videos(self, bvids_list: List[str]):
""" """