injectlib/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2025-11-25 03:15:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: 增加--help参数中文显示支持及douyin_aweme表music_download_url字段\n\n- 为命令行参数增加中文显示支持，提升用户体验\n- 在douyin_aweme表中新增music_download_url字段用于存储视频音乐下载链接\n- 更新相关数据库表结构文件(tables.sql, sqlite_tables.sql)\n- 实现音乐下载URL提取逻辑并集成到数据存储流程

Browse Source
This commit is contained in:
persist-1
2025-07-24 22:39:53 +08:00
parent fc06c783f5
commit 19df1734f1
7 changed files with 39 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@@ -169,4 +169,8 @@ cython_debug/
*/.DS_Store
.vscode
/node_modules
docs/.vitepress/cache
docs/.vitepress/cache
# other gitignore
.venv
.refer

24
cmd_arg/arg.py
View File

@@ -17,25 +17,29 @@ from tools.utils import str2bool
async def parse_cmd():
# 读取command arg
parser = argparse.ArgumentParser(description='Media crawler program.')
parser.add_argument('--platform', type=str, help='Media platform select (xhs | dy | ks | bili | wb | tieba | zhihu)',
parser = argparse.ArgumentParser(description='Media crawler program. / 媒体爬虫程序')
parser.add_argument('--platform', type=str,
help='Media platform select / 选择媒体平台 (xhs=小红书 | dy=抖音 | ks=快手 | bili=哔哩哔哩 | wb=微博 | tieba=百度贴吧 | zhihu=知乎)',
choices=["xhs", "dy", "ks", "bili", "wb", "tieba", "zhihu"], default=config.PLATFORM)
parser.add_argument('--lt', type=str, help='Login type (qrcode | phone | cookie)',
parser.add_argument('--lt', type=str,
help='Login type / 登录方式 (qrcode=二维码 | phone=手机号 | cookie=Cookie)',
choices=["qrcode", "phone", "cookie"], default=config.LOGIN_TYPE)
parser.add_argument('--type', type=str, help='crawler type (search | detail | creator)',
parser.add_argument('--type', type=str,
help='Crawler type / 爬取类型 (search=搜索 | detail=详情 | creator=创作者)',
choices=["search", "detail", "creator"], default=config.CRAWLER_TYPE)
parser.add_argument('--start', type=int,
help='number of start page', default=config.START_PAGE)
help='Number of start page / 起始页码', default=config.START_PAGE)
parser.add_argument('--keywords', type=str,
help='please input keywords', default=config.KEYWORDS)
help='Please input keywords / 请输入关键词', default=config.KEYWORDS)
parser.add_argument('--get_comment', type=str2bool,
help='''whether to crawl level one comment, supported values case insensitive ('yes', 'true', 't', 'y', '1', 'no', 'false', 'f', 'n', '0')''', default=config.ENABLE_GET_COMMENTS)
help='''Whether to crawl level one comment / 是否爬取一级评论, supported values case insensitive / 支持的值(不区分大小写) ('yes', 'true', 't', 'y', '1', 'no', 'false', 'f', 'n', '0')''', default=config.ENABLE_GET_COMMENTS)
parser.add_argument('--get_sub_comment', type=str2bool,
help=''''whether to crawl level two comment, supported values case insensitive ('yes', 'true', 't', 'y', '1', 'no', 'false', 'f', 'n', '0')''', default=config.ENABLE_GET_SUB_COMMENTS)
help=''''Whether to crawl level two comment / 是否爬取二级评论, supported values case insensitive / 支持的值(不区分大小写) ('yes', 'true', 't', 'y', '1', 'no', 'false', 'f', 'n', '0')''', default=config.ENABLE_GET_SUB_COMMENTS)
parser.add_argument('--save_data_option', type=str,
help='where to save the data (csv or db or json or sqlite)', choices=['csv', 'db', 'json', 'sqlite'], default=config.SAVE_DATA_OPTION)
help='Where to save the data / 数据保存方式 (csv=CSV文件 | db=MySQL数据库 | json=JSON文件 | sqlite=SQLite数据库)',
choices=['csv', 'db', 'json', 'sqlite'], default=config.SAVE_DATA_OPTION)
parser.add_argument('--cookies', type=str,
help='cookies used for cookie login type', default=config.COOKIES)
help='Cookies used for cookie login type / Cookie登录方式使用的Cookie值', default=config.COOKIES)
args = parser.parse_args()

2
config/base_config.py
View File

@@ -39,7 +39,7 @@ SAVE_LOGIN_STATE = True
# 是否启用CDP模式 - 使用用户现有的Chrome/Edge浏览器进行爬取，提供更好的反检测能力
# 启用后将自动检测并启动用户的Chrome/Edge浏览器，通过CDP协议进行控制
# 这种方式使用真实的浏览器环境，包括用户的扩展、Cookie和设置，大大降低被检测的风险
ENABLE_CDP_MODE = True
ENABLE_CDP_MODE = False
# CDP调试端口，用于与浏览器通信
# 如果端口被占用，系统会自动尝试下一个可用端口

BIN
schema/sqlite_tables.db
View File

Binary file not shown.

1
schema/sqlite_tables.sql
View File

@@ -149,6 +149,7 @@ CREATE TABLE douyin_aweme (
aweme_url TEXT DEFAULT NULL,
cover_url TEXT DEFAULT NULL,
video_download_url TEXT DEFAULT NULL,
music_download_url TEXT DEFAULT NULL,
source_keyword TEXT DEFAULT ''
);

1
schema/tables.sql
View File

@@ -149,6 +149,7 @@ CREATE TABLE `douyin_aweme`
`aweme_url` varchar(255) DEFAULT NULL COMMENT '视频详情页URL',
`cover_url` varchar(500) DEFAULT NULL COMMENT '视频封面图URL',
`video_download_url` varchar(1024) DEFAULT NULL COMMENT '视频下载地址',
`music_download_url` varchar(1024) DEFAULT NULL COMMENT '音乐下载地址',
PRIMARY KEY (`id`),
KEY `idx_douyin_awem_aweme_i_6f7bc6` (`aweme_id`),
KEY `idx_douyin_awem_create__299dfe` (`create_time`)

17
store/douyin/__init__.py
View File

@@ -105,6 +105,22 @@ def _extract_video_download_url(aweme_detail: Dict) -> str:
return actual_url_list[-1]
def _extract_music_download_url(aweme_detail: Dict) -> str:
"""
提取音乐下载地址
Args:
aweme_detail (Dict): 抖音视频
Returns:
str: 音乐下载地址
"""
music_item = aweme_detail.get("music", {})
play_url = music_item.get("play_url", {})
music_url = play_url.get("uri", "")
return music_url
async def update_douyin_aweme(aweme_item: Dict):
aweme_id = aweme_item.get("aweme_id")
user_info = aweme_item.get("author", {})
@@ -131,6 +147,7 @@ async def update_douyin_aweme(aweme_item: Dict):
"aweme_url": f"https://www.douyin.com/video/{aweme_id}",
"cover_url": _extract_content_cover_url(aweme_item),
"video_download_url": _extract_video_download_url(aweme_item),
"music_download_url": _extract_music_download_url(aweme_item),
"source_keyword": source_keyword_var.get(),
}
utils.logger.info(