diff --git a/requirements.txt b/requirements.txt index 4778f7e..70cc2dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ requests==2.28.2 -pytelegrambotapi==4.9.0 +pytelegrambotapi==4.10.0 beautifulsoup4==4.11.2 tgbot-ping==1.0.7 -redis==4.4.2 +redis==4.5.1 apscheduler==3.10.0 pymongo==4.3.3 tornado==6.2 captcha==0.4 passlib==1.7.4 -fakeredis==2.7.1 +fakeredis==2.8.0 pytz==2022.7.1 filetype==1.2.0 requests[socks] diff --git a/yyetsweb/migration/add_year.py b/yyetsweb/commands/add_year.py similarity index 94% rename from yyetsweb/migration/add_year.py rename to yyetsweb/commands/add_year.py index 8b6bd6f..4d87b11 100644 --- a/yyetsweb/migration/add_year.py +++ b/yyetsweb/commands/add_year.py @@ -11,9 +11,10 @@ import logging import re import time -import pymongo from tqdm import tqdm +from common import Mongo + logging.basicConfig(level=logging.INFO) @@ -29,8 +30,7 @@ def extract_year(name: str) -> int: return r -mongo_client = pymongo.MongoClient() -col = mongo_client["zimuzu"]["yyets"] +col = Mongo().client["zimuzu"]["yyets"] data = col.find() diff --git a/yyetsweb/migration/aliyun.xlsx b/yyetsweb/commands/aliyun.xlsx similarity index 100% rename from yyetsweb/migration/aliyun.xlsx rename to yyetsweb/commands/aliyun.xlsx diff --git a/yyetsweb/commands/ban_user.py b/yyetsweb/commands/ban_user.py index 8b107ed..f39ad5b 100644 --- a/yyetsweb/commands/ban_user.py +++ b/yyetsweb/commands/ban_user.py @@ -7,9 +7,10 @@ __author__ = "Benny " -from common import Mongo from tqdm import tqdm +from common import Mongo + client = Mongo() user_col = client.db["users"] diff --git a/yyetsweb/migration/change_datetime.py b/yyetsweb/commands/change_datetime.py similarity index 92% rename from yyetsweb/migration/change_datetime.py rename to yyetsweb/commands/change_datetime.py index 2c0ba28..0cb51a0 100644 --- a/yyetsweb/migration/change_datetime.py +++ b/yyetsweb/commands/change_datetime.py @@ -9,10 +9,9 @@ __author__ = "Benny " import time -import pymongo +from common import Mongo -client = pymongo.MongoClient() -from bson import ObjectId +client = Mongo().client comment = client["zimuzu"]["comment"] # date users = client["zimuzu"]["users"] # date diff --git a/yyetsweb/commands/common.py b/yyetsweb/commands/common.py index cd9986c..549960b 100644 --- a/yyetsweb/commands/common.py +++ b/yyetsweb/commands/common.py @@ -7,9 +7,10 @@ __author__ = "Benny " -import pymongo import os +import pymongo + class Mongo: def __init__(self): diff --git a/yyetsweb/migration/douban_data_clean.py b/yyetsweb/commands/douban_data_clean.py similarity index 99% rename from yyetsweb/migration/douban_data_clean.py rename to yyetsweb/commands/douban_data_clean.py index 766d36f..105681d 100644 --- a/yyetsweb/migration/douban_data_clean.py +++ b/yyetsweb/commands/douban_data_clean.py @@ -14,9 +14,8 @@ import sys logging.basicConfig(level=logging.INFO) lib_path = pathlib.Path(__file__).parent.parent.resolve().as_posix() sys.path.append(lib_path) -from tqdm import tqdm - from Mongo import DoubanMongoResource +from tqdm import tqdm m = DoubanMongoResource() diff --git a/yyetsweb/migration/douban_fix.py b/yyetsweb/commands/douban_fix.py similarity index 100% rename from yyetsweb/migration/douban_fix.py rename to yyetsweb/commands/douban_fix.py diff --git a/yyetsweb/migration/douban_sync.py b/yyetsweb/commands/douban_sync.py similarity index 100% rename from yyetsweb/migration/douban_sync.py rename to yyetsweb/commands/douban_sync.py diff --git a/yyetsweb/migration/format_order.py b/yyetsweb/commands/format_order.py similarity index 100% rename from yyetsweb/migration/format_order.py rename to yyetsweb/commands/format_order.py diff --git a/yyetsweb/migration/grafana_test_data.py b/yyetsweb/commands/grafana_test_data.py similarity index 92% rename from yyetsweb/migration/grafana_test_data.py rename to yyetsweb/commands/grafana_test_data.py index 0e1008b..4693dab 100644 --- a/yyetsweb/migration/grafana_test_data.py +++ b/yyetsweb/commands/grafana_test_data.py @@ -10,10 +10,9 @@ __author__ = "Benny " import random from datetime import date, timedelta -import pymongo +from common import Mongo -client = pymongo.MongoClient() -col = client["zimuzu"]["metrics"] +col = Mongo().client["zimuzu"]["metrics"] def generate_date_series(start: str, end: str) -> list: diff --git a/yyetsweb/migration/share_excel.py b/yyetsweb/commands/share_excel.py similarity index 97% rename from yyetsweb/migration/share_excel.py rename to yyetsweb/commands/share_excel.py index f84650a..b8a71a4 100644 --- a/yyetsweb/migration/share_excel.py +++ b/yyetsweb/commands/share_excel.py @@ -7,15 +7,16 @@ __author__ = "Benny " -import openpyxl import pathlib import sys +import openpyxl + web_path = pathlib.Path(__file__).parent.parent.resolve().as_posix() sys.path.append(web_path) from Mongo import Mongo from tqdm import tqdm -from utils import ts_date +from yyetsweb.utils import ts_date wb = openpyxl.open("aliyun.xlsx") diff --git a/yyetsweb/migration/prepare/convert_db.py b/yyetsweb/migration/prepare/convert_db.py deleted file mode 100644 index 8d1577b..0000000 --- a/yyetsweb/migration/prepare/convert_db.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/local/bin/python3 -# coding: utf-8 - -# BagAndDrag - convert_db.py -# 1/12/21 18:24 -# - -__author__ = "Benny " - -# convert to mongodb and con_sqlite - -import json -from typing import List - -import pymongo -import pymysql - -con_mysql = pymysql.Connect(host="127.0.0.1", user="root", password="root", charset="utf8mb4", database="yyets", - cursorclass=pymysql.cursors.DictCursor - ) - -mongo_client = pymongo.MongoClient() - -SIZE = 2000 - - -def clear_mongodb(): - mongo_client.drop_database("zimuzu") - - -def mongodb_insert(data: List[dict]): - db = mongo_client["zimuzu"] - col = db["yyets"] - # deserialize data.data - inserted = [] - for i in data: - api = json.loads(i["data"]) - views = api["data"]["info"]["views"] - api["data"]["info"]["views"] = int(views) - inserted.append(api) - col.insert_many(inserted) - - -def main(): - clear_mongodb() - - mysql_cur = con_mysql.cursor() - - mysql_cur.execute("SELECT * FROM resource") - - while True: - data = mysql_cur.fetchmany(SIZE) - if data: - mongodb_insert(data) - else: - break - - -if __name__ == '__main__': - main() - con_mysql.close() - mongo_client.close() diff --git a/yyetsweb/migration/prepare/dump_kv.py b/yyetsweb/migration/prepare/dump_kv.py deleted file mode 100644 index 0647c55..0000000 --- a/yyetsweb/migration/prepare/dump_kv.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/local/bin/python3 -# coding: utf-8 - -# YYeTsBot - dump_kv.py -# 2/6/21 18:12 -# - -__author__ = "Benny " - -import json -import threading -from concurrent.futures.thread import ThreadPoolExecutor - -import requests - -s = requests.Session() - -with open("index.json", ) as f: - ids = json.load(f) - -chunk = [ids[x:x + 3000] for x in range(0, len(ids), 3000)] - - -def download(c): - print("running batch ", c[0]) - for i in c: - data = s.get("https://yyets.dmesg.app/id={}".format(i)).json() - with open(f"{i}.json", "w") as f: - json.dump(data, f) - - -if __name__ == '__main__': - threads = [] - for part in chunk: - # Create 9 threads counting 10-19, 20-29, ... 90-99. - thread = threading.Thread(target=download, args=(part,)) - threads.append(thread) - - # Start them all - for thread in threads: - thread.start() - - # Wait for all to complete - for thread in threads: - thread.join() diff --git a/yyetsweb/migration/prepare/load_from_kv.py b/yyetsweb/migration/prepare/load_from_kv.py deleted file mode 100644 index f1c47cd..0000000 --- a/yyetsweb/migration/prepare/load_from_kv.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/local/bin/python3 -# coding: utf-8 - -# YYeTsBot - load_from_kv.py -# 2/6/21 18:27 -# - -__author__ = "Benny " - -import json -import os - -import pymongo - -mongo_client = pymongo.MongoClient() - -data_files = [i for i in os.listdir("data/") if i.endswith(".json")] -col = mongo_client["zimuzu"]["yyets"] -for data_file in data_files: - with open(os.path.join("data", data_file)) as f: - d = json.load(f) - views = int(d["data"]["info"]["views"]) - d["data"]["info"]["views"] = views - col.insert_one(d) diff --git a/yyetsweb/server.py b/yyetsweb/server.py index 1eef08a..5bf81b0 100644 --- a/yyetsweb/server.py +++ b/yyetsweb/server.py @@ -22,6 +22,7 @@ from tornado.log import enable_pretty_logging import dump_db from Mongo import OtherMongoResource, ResourceLatestMongoResource +from commands.douban_sync import sync_douban from handler import (AnnouncementHandler, BlacklistHandler, CaptchaHandler, CategoryHandler, CommentChildHandler, CommentHandler, CommentNewestHandler, CommentReactionHandler, @@ -34,7 +35,6 @@ from handler import (AnnouncementHandler, BlacklistHandler, CaptchaHandler, ResourceHandler, ResourceLatestHandler, SpamProcessHandler, TopHandler, TwitterOAuth2LoginHandler, UserEmailHandler, UserHandler) -from migration.douban_sync import sync_douban from utils import Cloudflare enable_pretty_logging()