Merge pull request #684 from 2513502304/main

添加抖音笔记图片的存储逻辑

schema/sqlite_tables.sql

@@ -150,6 +150,7 @@ CREATE TABLE douyin_aweme (
     cover_url TEXT DEFAULT NULL,
     video_download_url TEXT DEFAULT NULL,
     music_download_url TEXT DEFAULT NULL,
+    note_download_url TEXT DEFAULT NULL,
     source_keyword TEXT DEFAULT ''
 );
 
@@ -565,4 +566,4 @@ CREATE TABLE zhihu_creator (
     last_modify_ts INTEGER NOT NULL
 );
 
-CREATE UNIQUE INDEX idx_zhihu_creator_user_id ON zhihu_creator(user_id);
+CREATE UNIQUE INDEX idx_zhihu_creator_user_id ON zhihu_creator(user_id);

schema/tables.sql

@@ -150,6 +150,7 @@ CREATE TABLE `douyin_aweme`
     `cover_url`       varchar(500) DEFAULT NULL COMMENT '视频封面图URL',
     `video_download_url`       varchar(1024) DEFAULT NULL COMMENT '视频下载地址',
     `music_download_url`       varchar(1024) DEFAULT NULL COMMENT '音乐下载地址',
+    `note_download_url`        varchar(5120) DEFAULT NULL COMMENT '笔记下载地址',
     PRIMARY KEY (`id`),
     KEY               `idx_douyin_awem_aweme_i_6f7bc6` (`aweme_id`),
     KEY               `idx_douyin_awem_create__299dfe` (`create_time`)
@@ -593,4 +594,4 @@ alter table douyin_aweme_comment add column `like_count` varchar(255) NOT NULL D
 
 alter table xhs_note add column xsec_token varchar(50) default null comment '签名算法';
 alter table douyin_aweme_comment add column `pictures` varchar(500) NOT NULL DEFAULT '' COMMENT '评论图片列表';
-alter table bilibili_video_comment add column `like_count` varchar(255) NOT NULL DEFAULT '0' COMMENT '点赞数';
+alter table bilibili_video_comment add column `like_count` varchar(255) NOT NULL DEFAULT '0' COMMENT '点赞数';

store/douyin/__init__.py

@@ -8,7 +8,6 @@
 # 详细许可条款请参阅项目根目录下的LICENSE文件。
 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
 
-
 # -*- coding: utf-8 -*-
 # @Author  : relakkes@gmail.com
 # @Time    : 2024/1/14 18:46
@@ -26,19 +25,41 @@ class DouyinStoreFactory:
         "csv": DouyinCsvStoreImplement,
         "db": DouyinDbStoreImplement,
         "json": DouyinJsonStoreImplement,
-        "sqlite": DouyinSqliteStoreImplement
+        "sqlite": DouyinSqliteStoreImplement,
     }
 
     @staticmethod
     def create_store() -> AbstractStore:
         store_class = DouyinStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
         if not store_class:
-            raise ValueError(
+            raise ValueError("[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...")
-                "[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ..."
-            )
         return store_class()
 
 
+def _extract_note_image_list(aweme_detail: Dict) -> List[str]:
+    """
+    提取笔记图片列表
+
+    Args:
+        aweme_detail (Dict): 抖音内容详情
+
+    Returns:
+        List[str]: 笔记图片列表
+    """
+    images_res: List[str] = []
+    images: List[Dict] = aweme_detail.get("images", [])
+
+    if not images:
+        return []
+
+    for image in images:
+        image_url_list = image.get("url_list", [])  # download_url_list 为带水印的图片，url_list 为无水印的图片
+        if image_url_list:
+            images_res.append(image_url_list[-1])
+
+    return images_res
+
+
 def _extract_comment_image_list(comment_item: Dict) -> List[str]:
     """
     提取评论图片列表
@@ -76,9 +97,7 @@ def _extract_content_cover_url(aweme_detail: Dict) -> str:
     res_cover_url = ""
 
     video_item = aweme_detail.get("video", {})
-    raw_cover_url_list = (
+    raw_cover_url_list = (video_item.get("raw_cover", {}) or video_item.get("origin_cover", {})).get("url_list", [])
-            video_item.get("raw_cover", {}) or video_item.get("origin_cover", {})
-    ).get("url_list", [])
     if raw_cover_url_list and len(raw_cover_url_list) > 1:
         res_cover_url = raw_cover_url_list[1]
 
@@ -148,14 +167,11 @@ async def update_douyin_aweme(aweme_item: Dict):
         "cover_url": _extract_content_cover_url(aweme_item),
         "video_download_url": _extract_video_download_url(aweme_item),
         "music_download_url": _extract_music_download_url(aweme_item),
+        "note_download_url": ",".join(_extract_note_image_list(aweme_item)),
         "source_keyword": source_keyword_var.get(),
     }
-    utils.logger.info(
+    utils.logger.info(f"[store.douyin.update_douyin_aweme] douyin aweme id:{aweme_id}, title:{save_content_item.get('title')}")
-        f"[store.douyin.update_douyin_aweme] douyin aweme id:{aweme_id}, title:{save_content_item.get('title')}"
+    await DouyinStoreFactory.create_store().store_content(content_item=save_content_item)
-    )
-    await DouyinStoreFactory.create_store().store_content(
-        content_item=save_content_item
-    )
 
 
 async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]):
@@ -168,20 +184,12 @@ async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]):
 async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
     comment_aweme_id = comment_item.get("aweme_id")
     if aweme_id != comment_aweme_id:
-        utils.logger.error(
+        utils.logger.error(f"[store.douyin.update_dy_aweme_comment] comment_aweme_id: {comment_aweme_id} != aweme_id: {aweme_id}")
-            f"[store.douyin.update_dy_aweme_comment] comment_aweme_id: {comment_aweme_id} != aweme_id: {aweme_id}"
-        )
         return
     user_info = comment_item.get("user", {})
     comment_id = comment_item.get("cid")
     parent_comment_id = comment_item.get("reply_id", "0")
-    avatar_info = (
+    avatar_info = (user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get("avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {})
-            user_info.get("avatar_medium", {})
-            or user_info.get("avatar_300x300", {})
-            or user_info.get("avatar_168x168", {})
-            or user_info.get("avatar_thumb", {})
-            or {}
-    )
     save_comment_item = {
         "comment_id": comment_id,
         "create_time": comment_item.get("create_time"),
@@ -196,20 +204,14 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
         "nickname": user_info.get("nickname"),
         "avatar": avatar_info.get("url_list", [""])[0],
         "sub_comment_count": str(comment_item.get("reply_comment_total", 0)),
-        "like_count": (
+        "like_count": (comment_item.get("digg_count") if comment_item.get("digg_count") else 0),
-            comment_item.get("digg_count") if comment_item.get("digg_count") else 0
-        ),
         "last_modify_ts": utils.get_current_timestamp(),
         "parent_comment_id": parent_comment_id,
         "pictures": ",".join(_extract_comment_image_list(comment_item)),
     }
-    utils.logger.info(
+    utils.logger.info(f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}")
-        f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}"
-    )
 
-    await DouyinStoreFactory.create_store().store_comment(
+    await DouyinStoreFactory.create_store().store_comment(comment_item=save_comment_item)
-        comment_item=save_comment_item
-    )
 
 
 async def save_creator(user_id: str, creator: Dict):
@@ -220,8 +222,7 @@ async def save_creator(user_id: str, creator: Dict):
         "user_id": user_id,
         "nickname": user_info.get("nickname"),
         "gender": gender_map.get(user_info.get("gender"), "未知"),
-        "avatar": f"https://p3-pc.douyinpic.com/img/{avatar_uri}"
+        "avatar": f"https://p3-pc.douyinpic.com/img/{avatar_uri}" + r"~c5_300x300.jpeg?from=2956013662",
-                  + r"~c5_300x300.jpeg?from=2956013662",
         "desc": user_info.get("signature"),
         "ip_location": user_info.get("ip_location"),
         "follows": user_info.get("following_count", 0),