Merge pull request #729 from NanmiCoder/codex/replace-argparse-with-typer-for-cli

feat(cli): migrate CLI argument parsing to Typer
2025-11-25 03:15:17 +08:00 · 2025-09-26 18:08:21 +08:00
parent 879a72ea30 84f6f650f8
commit eb625b0b48
5 changed files with 961 additions and 1503 deletions
--- a/.python-version
+++ b/.python-version
@@ -1 +1 @@
-3.9
+3.11
--- a/cmd_arg/arg.py
+++ b/cmd_arg/arg.py
@@ -1,60 +1,257 @@
-# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：  
-# 1. 不得用于任何商业用途。  
-# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。  
-# 3. 不得进行大规模爬取或对平台造成运营干扰。  
-# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。   
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
 # 5. 不得用于任何非法或不当的用途。
-#   
-# 详细许可条款请参阅项目根目录下的LICENSE文件。  
-# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。  
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。


-import argparse
+from __future__ import annotations
+
+
+import sys
+from enum import Enum
+from types import SimpleNamespace
+from typing import Iterable, Optional, Sequence, Type, TypeVar
+
+import typer
+from typing_extensions import Annotated

 import config
 from tools.utils import str2bool


-async def parse_cmd():
-    # 读取command arg
-    parser = argparse.ArgumentParser(description='Media crawler program. / 媒体爬虫程序')
-    parser.add_argument('--platform', type=str, 
-                        help='Media platform select / 选择媒体平台 (xhs=小红书 | dy=抖音 | ks=快手 | bili=哔哩哔哩 | wb=微博 | tieba=百度贴吧 | zhihu=知乎)',
-                        choices=["xhs", "dy", "ks", "bili", "wb", "tieba", "zhihu"], default=config.PLATFORM)
-    parser.add_argument('--lt', type=str, 
-                        help='Login type / 登录方式 (qrcode=二维码 | phone=手机号 | cookie=Cookie)',
-                        choices=["qrcode", "phone", "cookie"], default=config.LOGIN_TYPE)
-    parser.add_argument('--type', type=str, 
-                        help='Crawler type / 爬取类型 (search=搜索 | detail=详情 | creator=创作者)',
-                        choices=["search", "detail", "creator"], default=config.CRAWLER_TYPE)
-    parser.add_argument('--start', type=int,
-                        help='Number of start page / 起始页码', default=config.START_PAGE)
-    parser.add_argument('--keywords', type=str,
-                        help='Please input keywords / 请输入关键词', default=config.KEYWORDS)
-    parser.add_argument('--get_comment', type=str2bool,
-                        help='''Whether to crawl level one comment / 是否爬取一级评论, supported values case insensitive / 支持的值(不区分大小写) ('yes', 'true', 't', 'y', '1', 'no', 'false', 'f', 'n', '0')''', default=config.ENABLE_GET_COMMENTS)
-    parser.add_argument('--get_sub_comment', type=str2bool,
-                        help=''''Whether to crawl level two comment / 是否爬取二级评论, supported values case insensitive / 支持的值(不区分大小写) ('yes', 'true', 't', 'y', '1', 'no', 'false', 'f', 'n', '0')''', default=config.ENABLE_GET_SUB_COMMENTS)
-    parser.add_argument('--save_data_option', type=str,
-                        help='Where to save the data / 数据保存方式 (csv=CSV文件 | db=MySQL数据库 | json=JSON文件 | sqlite=SQLite数据库)', 
-                        choices=['csv', 'db', 'json', 'sqlite'], default=config.SAVE_DATA_OPTION)
-    parser.add_argument('--init_db', type=str,
-                        help='Initialize database schema / 初始化数据库表结构 (sqlite | mysql)',
-                        choices=['sqlite', 'mysql'], default=None)
-    parser.add_argument('--cookies', type=str,
-                        help='Cookies used for cookie login type / Cookie登录方式使用的Cookie值', default=config.COOKIES)
+EnumT = TypeVar("EnumT", bound=Enum)

-    args = parser.parse_args()

-    # override config
-    config.PLATFORM = args.platform
-    config.LOGIN_TYPE = args.lt
-    config.CRAWLER_TYPE = args.type
-    config.START_PAGE = args.start
-    config.KEYWORDS = args.keywords
-    config.ENABLE_GET_COMMENTS = args.get_comment
-    config.ENABLE_GET_SUB_COMMENTS = args.get_sub_comment
-    config.SAVE_DATA_OPTION = args.save_data_option
-    config.COOKIES = args.cookies
+class PlatformEnum(str, Enum):
+    """支持的媒体平台枚举"""

-    return args
+    XHS = "xhs"
+    DOUYIN = "dy"
+    KUAISHOU = "ks"
+    BILIBILI = "bili"
+    WEIBO = "wb"
+    TIEBA = "tieba"
+    ZHIHU = "zhihu"
+
+
+class LoginTypeEnum(str, Enum):
+    """登录方式枚举"""
+
+    QRCODE = "qrcode"
+    PHONE = "phone"
+    COOKIE = "cookie"
+
+
+class CrawlerTypeEnum(str, Enum):
+    """爬虫类型枚举"""
+
+    SEARCH = "search"
+    DETAIL = "detail"
+    CREATOR = "creator"
+
+
+class SaveDataOptionEnum(str, Enum):
+    """数据保存方式枚举"""
+
+    CSV = "csv"
+    DB = "db"
+    JSON = "json"
+    SQLITE = "sqlite"
+
+
+class InitDbOptionEnum(str, Enum):
+    """数据库初始化选项"""
+
+    SQLITE = "sqlite"
+    MYSQL = "mysql"
+
+
+def _to_bool(value: bool | str) -> bool:
+    if isinstance(value, bool):
+        return value
+    return str2bool(value)
+
+
+def _coerce_enum(
+    enum_cls: Type[EnumT],
+    value: EnumT | str,
+    default: EnumT,
+) -> EnumT:
+    """Safely convert a raw config value to an enum member."""
+
+    if isinstance(value, enum_cls):
+        return value
+
+    try:
+        return enum_cls(value)
+    except ValueError:
+        typer.secho(
+            f"⚠️ 配置值 '{value}' 不在 {enum_cls.__name__} 支持的范围内，已回退到默认值 '{default.value}'.",
+            fg=typer.colors.YELLOW,
+        )
+        return default
+
+
+def _normalize_argv(argv: Optional[Sequence[str]]) -> Iterable[str]:
+    if argv is None:
+        return list(sys.argv[1:])
+    return list(argv)
+
+
+def _inject_init_db_default(args: Sequence[str]) -> list[str]:
+    """Ensure bare --init_db defaults to sqlite for backward compatibility."""
+
+    normalized: list[str] = []
+    i = 0
+    while i < len(args):
+        arg = args[i]
+        normalized.append(arg)
+
+        if arg == "--init_db":
+            next_arg = args[i + 1] if i + 1 < len(args) else None
+            if not next_arg or next_arg.startswith("-"):
+                normalized.append(InitDbOptionEnum.SQLITE.value)
+        i += 1
+
+    return normalized
+
+
+async def parse_cmd(argv: Optional[Sequence[str]] = None):
+    """使用 Typer 解析命令行参数。"""
+
+    app = typer.Typer(add_completion=False)
+
+    @app.callback(invoke_without_command=True)
+    def main(
+        platform: Annotated[
+            PlatformEnum,
+            typer.Option(
+                "--platform",
+                help="媒体平台选择 (xhs=小红书 | dy=抖音 | ks=快手 | bili=哔哩哔哩 | wb=微博 | tieba=百度贴吧 | zhihu=知乎)",
+                rich_help_panel="基础配置",
+            ),
+        ] = _coerce_enum(PlatformEnum, config.PLATFORM, PlatformEnum.XHS),
+        lt: Annotated[
+            LoginTypeEnum,
+            typer.Option(
+                "--lt",
+                help="登录方式 (qrcode=二维码 | phone=手机号 | cookie=Cookie)",
+                rich_help_panel="账号配置",
+            ),
+        ] = _coerce_enum(LoginTypeEnum, config.LOGIN_TYPE, LoginTypeEnum.QRCODE),
+        crawler_type: Annotated[
+            CrawlerTypeEnum,
+            typer.Option(
+                "--type",
+                help="爬取类型 (search=搜索 | detail=详情 | creator=创作者)",
+                rich_help_panel="基础配置",
+            ),
+        ] = _coerce_enum(CrawlerTypeEnum, config.CRAWLER_TYPE, CrawlerTypeEnum.SEARCH),
+        start: Annotated[
+            int,
+            typer.Option(
+                "--start",
+                help="起始页码",
+                rich_help_panel="基础配置",
+            ),
+        ] = config.START_PAGE,
+        keywords: Annotated[
+            str,
+            typer.Option(
+                "--keywords",
+                help="请输入关键词，多个关键词用逗号分隔",
+                rich_help_panel="基础配置",
+            ),
+        ] = config.KEYWORDS,
+        get_comment: Annotated[
+            str,
+            typer.Option(
+                "--get_comment",
+                help="是否爬取一级评论，支持 yes/true/t/y/1 或 no/false/f/n/0",
+                rich_help_panel="评论配置",
+                show_default=True,
+            ),
+        ] = str(config.ENABLE_GET_COMMENTS),
+        get_sub_comment: Annotated[
+            str,
+            typer.Option(
+                "--get_sub_comment",
+                help="是否爬取二级评论，支持 yes/true/t/y/1 或 no/false/f/n/0",
+                rich_help_panel="评论配置",
+                show_default=True,
+            ),
+        ] = str(config.ENABLE_GET_SUB_COMMENTS),
+        save_data_option: Annotated[
+            SaveDataOptionEnum,
+            typer.Option(
+                "--save_data_option",
+                help="数据保存方式 (csv=CSV文件 | db=MySQL数据库 | json=JSON文件 | sqlite=SQLite数据库)",
+                rich_help_panel="存储配置",
+            ),
+        ] = _coerce_enum(
+            SaveDataOptionEnum, config.SAVE_DATA_OPTION, SaveDataOptionEnum.JSON
+        ),
+        init_db: Annotated[
+            Optional[InitDbOptionEnum],
+            typer.Option(
+                "--init_db",
+                help="初始化数据库表结构 (sqlite | mysql)",
+                rich_help_panel="存储配置",
+            ),
+        ] = None,
+        cookies: Annotated[
+            str,
+            typer.Option(
+                "--cookies",
+                help="Cookie 登录方式使用的 Cookie 值",
+                rich_help_panel="账号配置",
+            ),
+        ] = config.COOKIES,
+    ) -> SimpleNamespace:
+        """MediaCrawler 命令行入口"""
+
+        enable_comment = _to_bool(get_comment)
+        enable_sub_comment = _to_bool(get_sub_comment)
+        init_db_value = init_db.value if init_db else None
+
+        # override global config
+        config.PLATFORM = platform.value
+        config.LOGIN_TYPE = lt.value
+        config.CRAWLER_TYPE = crawler_type.value
+        config.START_PAGE = start
+        config.KEYWORDS = keywords
+        config.ENABLE_GET_COMMENTS = enable_comment
+        config.ENABLE_GET_SUB_COMMENTS = enable_sub_comment
+        config.SAVE_DATA_OPTION = save_data_option.value
+        config.COOKIES = cookies
+
+        return SimpleNamespace(
+            platform=config.PLATFORM,
+            lt=config.LOGIN_TYPE,
+            type=config.CRAWLER_TYPE,
+            start=config.START_PAGE,
+            keywords=config.KEYWORDS,
+            get_comment=config.ENABLE_GET_COMMENTS,
+            get_sub_comment=config.ENABLE_GET_SUB_COMMENTS,
+            save_data_option=config.SAVE_DATA_OPTION,
+            init_db=init_db_value,
+            cookies=config.COOKIES,
+        )
+
+    command = typer.main.get_command(app)
+
+    cli_args = _normalize_argv(argv)
+    cli_args = _inject_init_db_default(cli_args)
+
+    try:
+        result = command.main(args=cli_args, standalone_mode=False)
+        if isinstance(result, int):  # help/options handled by Typer; propagate exit code
+            raise SystemExit(result)
+        return result
+    except typer.Exit as exc:  # pragma: no cover - CLI exit paths
+        raise SystemExit(exc.exit_code) from exc
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ author = "程序员阿江-Relakkes <relakkes@gmail.com>"
 version = "0.1.0"
 description = "A social media crawler project, support Xiaohongshu, Weibo, Zhihu, Bilibili, Douyin, BaiduTieBa etc."
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.11"
 dependencies = [
    "aiofiles~=23.2.1",
    "aiomysql==0.2.0",
@@ -29,6 +29,7 @@ dependencies = [
    "requests==2.32.3",
    "sqlalchemy>=2.0.43",
    "tenacity==8.2.2",
+    "typer>=0.12.3",
    "uvicorn==0.29.0",
    "wordcloud==1.9.3",
 ]
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,6 +2,7 @@ httpx==0.28.1
 Pillow==9.5.0
 playwright==1.45.0
 tenacity==8.2.2
+typer>=0.12.3
 opencv-python
 aiomysql==0.2.0
 redis~=4.6.0
--- a/uv.lock
+++ b/uv.lock
@@ -1 +1 @@
 .9
 .11