diff --git a/yyetsweb/common/utils.py b/yyetsweb/common/utils.py index a549b84..d76ce54 100644 --- a/yyetsweb/common/utils.py +++ b/yyetsweb/common/utils.py @@ -8,6 +8,7 @@ __author__ = "Benny " import contextlib +import json import logging import os import pathlib @@ -24,6 +25,8 @@ import requests from akismet import Akismet from jinja2 import Template +from databases.base import Redis + def setup_logger(): coloredlogs.install( @@ -94,7 +97,9 @@ def check_spam(ip, ua, author, content) -> int: return 0 -class Cloudflare: +class Cloudflare(Redis): + key = "cf-blacklist-ip" + def __init__(self): self.account_id = "e8d3ba82fe9e9a41cceb0047c2a2ab4f" self.item_id = "3740654e0b104053b3e5d0a71fe87b33" @@ -103,20 +108,34 @@ class Cloudflare: ) self.session = requests.Session() self.session.headers.update({"Authorization": "Bearer %s" % os.getenv("CF_TOKEN")}) + super().__init__() - def get_old_ips(self) -> list: - data = self.session.get(self.endpoint).json()["result"] - result = [i["ip"] for i in data if data] - return result or [] + def get_old_ips(self) -> dict: + cache = self.r.get(self.key) + if cache: + cache = json.loads(cache) + logging.info("Cache found with %s IPs", len(cache)) + return cache + else: + data = self.session.get(self.endpoint).json() + result = data["result"] + cursor = data.get("result_info", {}).get("cursors", {}).get("after") + while cursor: + logging.info("Fetching next page with cursor %s", cursor) + data = self.session.get(self.endpoint, params={"cursor": cursor}).json() + result.extend(data["result"]) + cursor = data.get("result_info", {}).get("cursors", {}).get("after") + logging.info("Got %s IPs", len(result)) + return result def ban_new_ip(self, ip): + old_ips = [d["ip"] for d in self.get_old_ips()] + old_ips.append(ip) + body = [{"ip": i} for i in set(old_ips)] + self.r.set(self.key, json.dumps(body)) logging.warning("Adding %s to cloudflare managed challenge list", ip) - with contextlib.suppress(Exception): - old_ips = self.get_old_ips() - old_ips.append(ip) - body = [{"ip": i} for i in set(old_ips)] - resp = self.session.put(self.endpoint, json=body) - logging.info(resp.json()) + resp = self.session.put(self.endpoint, json=body) + logging.info(resp.json()) def clear_fw(self): logging.info("Clearing firewall rules") @@ -124,4 +143,5 @@ class Cloudflare: if __name__ == "__main__": - send_mail("benny.think@gmail.com", "主题", {"username": "test123", "text": "测试内容不错"}) + cf = Cloudflare() + cf.ban_new_ip("192.168.1.1") diff --git a/yyetsweb/databases/__init__.py b/yyetsweb/databases/__init__.py index a71650c..908d8a5 100644 --- a/yyetsweb/databases/__init__.py +++ b/yyetsweb/databases/__init__.py @@ -8,21 +8,10 @@ import logging import pathlib import sys -from common.utils import Cloudflare, setup_logger - -cf = Cloudflare() - -setup_logger() - DOUBAN_SEARCH = "https://www.douban.com/search?cat=1002&q={}" DOUBAN_DETAIL = "https://movie.douban.com/subject/{}/" -lib_path = ( - pathlib.Path(__file__) - .parent.parent.parent.joinpath("yyetsbot") - .resolve() - .as_posix() -) +lib_path = pathlib.Path(__file__).parent.parent.parent.joinpath("yyetsbot").resolve().as_posix() sys.path.append(lib_path) from fansub import BD2020, XL720, NewzmzOnline, ZhuixinfanOnline, ZimuxiaOnline diff --git a/yyetsweb/databases/other.py b/yyetsweb/databases/other.py index 89a4a0a..ea42a7b 100644 --- a/yyetsweb/databases/other.py +++ b/yyetsweb/databases/other.py @@ -14,10 +14,11 @@ import requests from bson import ObjectId from captcha.image import ImageCaptcha -from common.utils import ts_date -from databases import cf +from common.utils import Cloudflare, ts_date from databases.base import Mongo, Redis +cf = Cloudflare() + captcha_ex = 60 * 10 predefined_str = re.sub(r"[1l0oOI]", "", string.ascii_letters + string.digits) @@ -77,12 +78,7 @@ class Category(Mongo): query_dict[f"data.info.{key}"] = value logging.info("Query dict %s", query_dict) projection = {"_id": False, "data.list": False} - data = ( - self.db["yyets"] - .find(query_dict, projection=projection) - .limit(size) - .skip((page - 1) * size) - ) + data = self.db["yyets"].find(query_dict, projection=projection).limit(size).skip((page - 1) * size) count = self.db["yyets"].count_documents(query_dict) f = [] for item in data: @@ -91,9 +87,7 @@ class Category(Mongo): {"resourceId": item["data"]["info"]["id"]}, projection=projection ) if douban_data: - douban_data["posterData"] = base64.b64encode( - douban_data["posterData"] - ).decode("u8") + douban_data["posterData"] = base64.b64encode(douban_data["posterData"]).decode("u8") item["data"]["info"]["douban"] = douban_data else: item["data"]["info"]["douban"] = {} @@ -152,11 +146,7 @@ class Other(Mongo): self.db["history"].insert_one(json_data) # save all the views data to history projection = {"_id": False, "data.info.views": True, "data.info.id": True} - data = ( - self.db["yyets"] - .find({}, projection) - .sort("data.info.views", pymongo.DESCENDING) - ) + data = self.db["yyets"].find({}, projection).sort("data.info.views", pymongo.DESCENDING) result = {"date": last_month, "type": "detail"} for datum in data: rid = str(datum["data"]["info"]["id"]) @@ -167,9 +157,7 @@ class Other(Mongo): self.db["yyets"].update_many({}, {"$set": {"data.info.views": 0}}) def import_ban_user(self): - usernames = self.db["users"].find( - {"status.disable": True}, projection={"username": True} - ) + usernames = self.db["users"].find({"status.disable": True}, projection={"username": True}) r = Redis().r r.delete("user_blacklist") logging.info("Importing ban users to redis...%s", usernames) @@ -184,9 +172,7 @@ class Captcha(Redis): image = ImageCaptcha() data = image.generate(chars) self.r.set(captcha_id, chars, ex=captcha_ex) - return ( - f"data:image/png;base64,{base64.b64encode(data.getvalue()).decode('ascii')}" - ) + return f"data:image/png;base64,{base64.b64encode(data.getvalue()).decode('ascii')}" def verify_code(self, user_input, captcha_id) -> dict: correct_code = self.r.get(captcha_id) diff --git a/yyetsweb/handlers/resources.py b/yyetsweb/handlers/resources.py index 5099a03..3efa33e 100644 --- a/yyetsweb/handlers/resources.py +++ b/yyetsweb/handlers/resources.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # coding: utf-8 import logging -import mmap import os from http import HTTPStatus from pathlib import Path @@ -41,9 +40,8 @@ class ResourceHandler(BaseHandler): cf.ban_new_ip(ip) self.set_header("Content-Type", "text/html") self.set_header("Content-Encoding", "gzip") - with open("templates/gift.gzip", "rb") as f: - mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) - return mm.read() + with open("templates/gift.gz", "rb") as f: + return f.read() @run_on_executor() def search_resource(self): diff --git a/yyetsweb/server.py b/yyetsweb/server.py index 9b79979..4a7594c 100644 --- a/yyetsweb/server.py +++ b/yyetsweb/server.py @@ -21,7 +21,7 @@ from tornado.log import enable_pretty_logging from common.dump_db import entry_dump from common.sync import YYSub, sync_douban -from common.utils import setup_logger, Cloudflare +from common.utils import Cloudflare, setup_logger from databases.base import SearchEngine from databases.other import Other from handlers.base import IndexHandler, NotFoundHandler diff --git a/yyetsweb/templates/gift.gz b/yyetsweb/templates/gift.gz new file mode 100644 index 0000000..aee1418 Binary files /dev/null and b/yyetsweb/templates/gift.gz differ