From e6db6be1cace5ca5d0e7b4eb2c7cc3cf802a8dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=9F=E6=8C=81=E6=B1=9F?= <129171955+2513502304@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:45:38 +0800 Subject: [PATCH 1/3] =?UTF-8?q?Update=20=5F=5Finit=5F=5F.py=EF=BC=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E6=8F=90=E5=8F=96=E6=8A=96=E9=9F=B3?= =?UTF-8?q?=E7=AC=94=E8=AE=B0=E5=9B=BE=E7=89=87=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- store/douyin/__init__.py | 71 ++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/store/douyin/__init__.py b/store/douyin/__init__.py index 1c3b6a3..a44a010 100644 --- a/store/douyin/__init__.py +++ b/store/douyin/__init__.py @@ -8,7 +8,6 @@ # 详细许可条款请参阅项目根目录下的LICENSE文件。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 - # -*- coding: utf-8 -*- # @Author : relakkes@gmail.com # @Time : 2024/1/14 18:46 @@ -26,19 +25,41 @@ class DouyinStoreFactory: "csv": DouyinCsvStoreImplement, "db": DouyinDbStoreImplement, "json": DouyinJsonStoreImplement, - "sqlite": DouyinSqliteStoreImplement + "sqlite": DouyinSqliteStoreImplement, } @staticmethod def create_store() -> AbstractStore: store_class = DouyinStoreFactory.STORES.get(config.SAVE_DATA_OPTION) if not store_class: - raise ValueError( - "[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ..." - ) + raise ValueError("[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...") return store_class() +def _extract_note_image_list(aweme_detail: Dict) -> List[str]: + """ + 提取笔记图片列表 + + Args: + aweme_detail (Dict): 抖音内容详情 + + Returns: + List[str]: 笔记图片列表 + """ + images_res: List[str] = [] + images: List[Dict] = aweme_detail.get("images", []) + + if not images: + return [] + + for image in images: + image_url_list = image.get("url_list", []) # download_url_list 为带水印的图片，url_list 为无水印的图片 + if image_url_list: + images_res.append(image_url_list[-1]) + + return images_res + + def _extract_comment_image_list(comment_item: Dict) -> List[str]: """ 提取评论图片列表 @@ -76,9 +97,7 @@ def _extract_content_cover_url(aweme_detail: Dict) -> str: res_cover_url = "" video_item = aweme_detail.get("video", {}) - raw_cover_url_list = ( - video_item.get("raw_cover", {}) or video_item.get("origin_cover", {}) - ).get("url_list", []) + raw_cover_url_list = (video_item.get("raw_cover", {}) or video_item.get("origin_cover", {})).get("url_list", []) if raw_cover_url_list and len(raw_cover_url_list) > 1: res_cover_url = raw_cover_url_list[1] @@ -148,14 +167,11 @@ async def update_douyin_aweme(aweme_item: Dict): "cover_url": _extract_content_cover_url(aweme_item), "video_download_url": _extract_video_download_url(aweme_item), "music_download_url": _extract_music_download_url(aweme_item), + "note_download_url": ",".join(_extract_note_image_list(aweme_item)), "source_keyword": source_keyword_var.get(), } - utils.logger.info( - f"[store.douyin.update_douyin_aweme] douyin aweme id:{aweme_id}, title:{save_content_item.get('title')}" - ) - await DouyinStoreFactory.create_store().store_content( - content_item=save_content_item - ) + utils.logger.info(f"[store.douyin.update_douyin_aweme] douyin aweme id:{aweme_id}, title:{save_content_item.get('title')}") + await DouyinStoreFactory.create_store().store_content(content_item=save_content_item) async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]): @@ -168,20 +184,12 @@ async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]): async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict): comment_aweme_id = comment_item.get("aweme_id") if aweme_id != comment_aweme_id: - utils.logger.error( - f"[store.douyin.update_dy_aweme_comment] comment_aweme_id: {comment_aweme_id} != aweme_id: {aweme_id}" - ) + utils.logger.error(f"[store.douyin.update_dy_aweme_comment] comment_aweme_id: {comment_aweme_id} != aweme_id: {aweme_id}") return user_info = comment_item.get("user", {}) comment_id = comment_item.get("cid") parent_comment_id = comment_item.get("reply_id", "0") - avatar_info = ( - user_info.get("avatar_medium", {}) - or user_info.get("avatar_300x300", {}) - or user_info.get("avatar_168x168", {}) - or user_info.get("avatar_thumb", {}) - or {} - ) + avatar_info = (user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get("avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {}) save_comment_item = { "comment_id": comment_id, "create_time": comment_item.get("create_time"), @@ -196,20 +204,14 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict): "nickname": user_info.get("nickname"), "avatar": avatar_info.get("url_list", [""])[0], "sub_comment_count": str(comment_item.get("reply_comment_total", 0)), - "like_count": ( - comment_item.get("digg_count") if comment_item.get("digg_count") else 0 - ), + "like_count": (comment_item.get("digg_count") if comment_item.get("digg_count") else 0), "last_modify_ts": utils.get_current_timestamp(), "parent_comment_id": parent_comment_id, "pictures": ",".join(_extract_comment_image_list(comment_item)), } - utils.logger.info( - f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}" - ) + utils.logger.info(f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}") - await DouyinStoreFactory.create_store().store_comment( - comment_item=save_comment_item - ) + await DouyinStoreFactory.create_store().store_comment(comment_item=save_comment_item) async def save_creator(user_id: str, creator: Dict): @@ -220,8 +222,7 @@ async def save_creator(user_id: str, creator: Dict): "user_id": user_id, "nickname": user_info.get("nickname"), "gender": gender_map.get(user_info.get("gender"), "未知"), - "avatar": f"https://p3-pc.douyinpic.com/img/{avatar_uri}" - + r"~c5_300x300.jpeg?from=2956013662", + "avatar": f"https://p3-pc.douyinpic.com/img/{avatar_uri}" + r"~c5_300x300.jpeg?from=2956013662", "desc": user_info.get("signature"), "ip_location": user_info.get("ip_location"), "follows": user_info.get("following_count", 0), From 612a9b53d3e9c62c173984122daf1b5c06511a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=9F=E6=8C=81=E6=B1=9F?= <129171955+2513502304@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:46:46 +0800 Subject: [PATCH 2/3] =?UTF-8?q?Update=20tables.sql=EF=BC=8C=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E6=9B=B4=E6=96=B0=E8=AF=A5=E6=96=87=E4=BB=B6=E4=BB=A5?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=BF=9D=E5=AD=98=E7=AC=94=E8=AE=B0=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- schema/tables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schema/tables.sql b/schema/tables.sql index 676e8a6..7303971 100644 --- a/schema/tables.sql +++ b/schema/tables.sql @@ -150,6 +150,7 @@ CREATE TABLE `douyin_aweme` `cover_url` varchar(500) DEFAULT NULL COMMENT '视频封面图URL', `video_download_url` varchar(1024) DEFAULT NULL COMMENT '视频下载地址', `music_download_url` varchar(1024) DEFAULT NULL COMMENT '音乐下载地址', + `note_download_url` varchar(5120) DEFAULT NULL COMMENT '笔记下载地址', PRIMARY KEY (`id`), KEY `idx_douyin_awem_aweme_i_6f7bc6` (`aweme_id`), KEY `idx_douyin_awem_create__299dfe` (`create_time`) @@ -593,4 +594,4 @@ alter table douyin_aweme_comment add column `like_count` varchar(255) NOT NULL D alter table xhs_note add column xsec_token varchar(50) default null comment '签名算法'; alter table douyin_aweme_comment add column `pictures` varchar(500) NOT NULL DEFAULT '' COMMENT '评论图片列表'; -alter table bilibili_video_comment add column `like_count` varchar(255) NOT NULL DEFAULT '0' COMMENT '点赞数'; \ No newline at end of file +alter table bilibili_video_comment add column `like_count` varchar(255) NOT NULL DEFAULT '0' COMMENT '点赞数'; From 214ccaa294cc45c878cc34f0736fe9d97c22167b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=9F=E6=8C=81=E6=B1=9F?= <129171955+2513502304@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:48:52 +0800 Subject: [PATCH 3/3] =?UTF-8?q?Update=20sqlite=5Ftables.sql=EF=BC=8C?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E6=9B=B4=E6=96=B0sqlite=E4=BB=A5=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E4=BF=9D=E5=AD=98=E7=AC=94=E8=AE=B0=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- schema/sqlite_tables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schema/sqlite_tables.sql b/schema/sqlite_tables.sql index 84696a0..4c6d252 100644 --- a/schema/sqlite_tables.sql +++ b/schema/sqlite_tables.sql @@ -150,6 +150,7 @@ CREATE TABLE douyin_aweme ( cover_url TEXT DEFAULT NULL, video_download_url TEXT DEFAULT NULL, music_download_url TEXT DEFAULT NULL, + note_download_url TEXT DEFAULT NULL, source_keyword TEXT DEFAULT '' ); @@ -565,4 +566,4 @@ CREATE TABLE zhihu_creator ( last_modify_ts INTEGER NOT NULL ); -CREATE UNIQUE INDEX idx_zhihu_creator_user_id ON zhihu_creator(user_id); \ No newline at end of file +CREATE UNIQUE INDEX idx_zhihu_creator_user_id ON zhihu_creator(user_id);