mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2025-11-25 03:15:17 +08:00
improve base config reading command line arg logic
This commit is contained in:
29
main.py
29
main.py
@@ -1,7 +1,7 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
import cmd_arg
|
||||
import config
|
||||
import db
|
||||
from base.base_crawler import AbstractCrawler
|
||||
@@ -28,34 +28,15 @@ class CrawlerFactory:
|
||||
raise ValueError("Invalid Media Platform Currently only supported xhs or dy or ks or bili ...")
|
||||
return crawler_class()
|
||||
|
||||
|
||||
async def main():
|
||||
# define command line params ...
|
||||
parser = argparse.ArgumentParser(description='Media crawler program.')
|
||||
parser.add_argument('--platform', type=str, help='Media platform select (xhs | dy | ks | bili | wb)',
|
||||
choices=["xhs", "dy", "ks", "bili", "wb"], default=config.PLATFORM)
|
||||
parser.add_argument('--lt', type=str, help='Login type (qrcode | phone | cookie)',
|
||||
choices=["qrcode", "phone", "cookie"], default=config.LOGIN_TYPE)
|
||||
parser.add_argument('--type', type=str, help='crawler type (search | detail | creator)',
|
||||
choices=["search", "detail", "creator"], default=config.CRAWLER_TYPE)
|
||||
parser.add_argument('--start', type=int, help='crawler type (number of start page)',
|
||||
default=config.START_PAGE)
|
||||
parser.add_argument('--keywords', type=str, help='crawler type (please input keywords)',
|
||||
default=config.KEYWORDS)
|
||||
|
||||
# parse cmd
|
||||
await cmd_arg.parse_cmd()
|
||||
|
||||
# init db
|
||||
if config.SAVE_DATA_OPTION == "db":
|
||||
await db.init_db()
|
||||
|
||||
args = parser.parse_args()
|
||||
crawler = CrawlerFactory.create_crawler(platform=args.platform)
|
||||
crawler.init_config(
|
||||
platform=args.platform,
|
||||
login_type=args.lt,
|
||||
crawler_type=args.type,
|
||||
start_page=args.start,
|
||||
keyword=args.keywords
|
||||
)
|
||||
crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM)
|
||||
await crawler.start()
|
||||
|
||||
if config.SAVE_DATA_OPTION == "db":
|
||||
|
||||
Reference in New Issue
Block a user