fix cf firewall rules with cursor, remove mmap

yyetsweb/common/utils.py

@@ -8,6 +8,7 @@
 __author__ = "Benny <benny.think@gmail.com>"
 
 import contextlib
+import json
 import logging
 import os
 import pathlib
@@ -24,6 +25,8 @@ import requests
 from akismet import Akismet
 from jinja2 import Template
 
+from databases.base import Redis
+
 
 def setup_logger():
     coloredlogs.install(
@@ -94,7 +97,9 @@ def check_spam(ip, ua, author, content) -> int:
     return 0
 
 
-class Cloudflare:
+class Cloudflare(Redis):
+    key = "cf-blacklist-ip"
+
     def __init__(self):
         self.account_id = "e8d3ba82fe9e9a41cceb0047c2a2ab4f"
         self.item_id = "3740654e0b104053b3e5d0a71fe87b33"
@@ -103,18 +108,32 @@ class Cloudflare:
         )
         self.session = requests.Session()
         self.session.headers.update({"Authorization": "Bearer %s" % os.getenv("CF_TOKEN")})
+        super().__init__()
 
-    def get_old_ips(self) -> list:
+    def get_old_ips(self) -> dict:
-        data = self.session.get(self.endpoint).json()["result"]
+        cache = self.r.get(self.key)
-        result = [i["ip"] for i in data if data]
+        if cache:
-        return result or []
+            cache = json.loads(cache)
+            logging.info("Cache found with %s IPs", len(cache))
+            return cache
+        else:
+            data = self.session.get(self.endpoint).json()
+            result = data["result"]
+            cursor = data.get("result_info", {}).get("cursors", {}).get("after")
+            while cursor:
+                logging.info("Fetching next page with cursor %s", cursor)
+                data = self.session.get(self.endpoint, params={"cursor": cursor}).json()
+                result.extend(data["result"])
+                cursor = data.get("result_info", {}).get("cursors", {}).get("after")
+            logging.info("Got %s IPs", len(result))
+            return result
 
     def ban_new_ip(self, ip):
-        logging.warning("Adding %s to cloudflare managed challenge list", ip)
+        old_ips = [d["ip"] for d in self.get_old_ips()]
-        with contextlib.suppress(Exception):
-            old_ips = self.get_old_ips()
         old_ips.append(ip)
         body = [{"ip": i} for i in set(old_ips)]
+        self.r.set(self.key, json.dumps(body))
+        logging.warning("Adding %s to cloudflare managed challenge list", ip)
         resp = self.session.put(self.endpoint, json=body)
         logging.info(resp.json())
 
@@ -124,4 +143,5 @@ class Cloudflare:
 
 
 if __name__ == "__main__":
-    send_mail("benny.think@gmail.com", "主题", {"username": "test123", "text": "测试内容<b>不错</b>"})
+    cf = Cloudflare()
+    cf.ban_new_ip("192.168.1.1")

yyetsweb/databases/__init__.py

@@ -8,21 +8,10 @@ import logging
 import pathlib
 import sys
 
-from common.utils import Cloudflare, setup_logger
-
-cf = Cloudflare()
-
-setup_logger()
-
 DOUBAN_SEARCH = "https://www.douban.com/search?cat=1002&q={}"
 DOUBAN_DETAIL = "https://movie.douban.com/subject/{}/"
 
-lib_path = (
+lib_path = pathlib.Path(__file__).parent.parent.parent.joinpath("yyetsbot").resolve().as_posix()
-    pathlib.Path(__file__)
-    .parent.parent.parent.joinpath("yyetsbot")
-    .resolve()
-    .as_posix()
-)
 
 sys.path.append(lib_path)
 from fansub import BD2020, XL720, NewzmzOnline, ZhuixinfanOnline, ZimuxiaOnline

yyetsweb/databases/other.py

@@ -14,10 +14,11 @@ import requests
 from bson import ObjectId
 from captcha.image import ImageCaptcha
 
-from common.utils import ts_date
+from common.utils import Cloudflare, ts_date
-from databases import cf
 from databases.base import Mongo, Redis
 
+cf = Cloudflare()
+
 captcha_ex = 60 * 10
 predefined_str = re.sub(r"[1l0oOI]", "", string.ascii_letters + string.digits)
 
@@ -77,12 +78,7 @@ class Category(Mongo):
             query_dict[f"data.info.{key}"] = value
         logging.info("Query dict %s", query_dict)
         projection = {"_id": False, "data.list": False}
-        data = (
+        data = self.db["yyets"].find(query_dict, projection=projection).limit(size).skip((page - 1) * size)
-            self.db["yyets"]
-            .find(query_dict, projection=projection)
-            .limit(size)
-            .skip((page - 1) * size)
-        )
         count = self.db["yyets"].count_documents(query_dict)
         f = []
         for item in data:
@@ -91,9 +87,7 @@ class Category(Mongo):
                     {"resourceId": item["data"]["info"]["id"]}, projection=projection
                 )
                 if douban_data:
-                    douban_data["posterData"] = base64.b64encode(
+                    douban_data["posterData"] = base64.b64encode(douban_data["posterData"]).decode("u8")
-                        douban_data["posterData"]
-                    ).decode("u8")
                     item["data"]["info"]["douban"] = douban_data
                 else:
                     item["data"]["info"]["douban"] = {}
@@ -152,11 +146,7 @@ class Other(Mongo):
         self.db["history"].insert_one(json_data)
         # save all the views data to history
         projection = {"_id": False, "data.info.views": True, "data.info.id": True}
-        data = (
+        data = self.db["yyets"].find({}, projection).sort("data.info.views", pymongo.DESCENDING)
-            self.db["yyets"]
-            .find({}, projection)
-            .sort("data.info.views", pymongo.DESCENDING)
-        )
         result = {"date": last_month, "type": "detail"}
         for datum in data:
             rid = str(datum["data"]["info"]["id"])
@@ -167,9 +157,7 @@ class Other(Mongo):
         self.db["yyets"].update_many({}, {"$set": {"data.info.views": 0}})
 
     def import_ban_user(self):
-        usernames = self.db["users"].find(
+        usernames = self.db["users"].find({"status.disable": True}, projection={"username": True})
-            {"status.disable": True}, projection={"username": True}
-        )
         r = Redis().r
         r.delete("user_blacklist")
         logging.info("Importing ban users to redis...%s", usernames)
@@ -184,9 +172,7 @@ class Captcha(Redis):
         image = ImageCaptcha()
         data = image.generate(chars)
         self.r.set(captcha_id, chars, ex=captcha_ex)
-        return (
+        return f"data:image/png;base64,{base64.b64encode(data.getvalue()).decode('ascii')}"
-            f"data:image/png;base64,{base64.b64encode(data.getvalue()).decode('ascii')}"
-        )
 
     def verify_code(self, user_input, captcha_id) -> dict:
         correct_code = self.r.get(captcha_id)

yyetsweb/handlers/resources.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 # coding: utf-8
 import logging
-import mmap
 import os
 from http import HTTPStatus
 from pathlib import Path
@@ -41,9 +40,8 @@ class ResourceHandler(BaseHandler):
             cf.ban_new_ip(ip)
             self.set_header("Content-Type", "text/html")
             self.set_header("Content-Encoding", "gzip")
-            with open("templates/gift.gzip", "rb") as f:
+            with open("templates/gift.gz", "rb") as f:
-                mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+                return f.read()
-                return mm.read()
 
     @run_on_executor()
     def search_resource(self):

yyetsweb/server.py

@@ -21,7 +21,7 @@ from tornado.log import enable_pretty_logging
 
 from common.dump_db import entry_dump
 from common.sync import YYSub, sync_douban
-from common.utils import setup_logger, Cloudflare
+from common.utils import Cloudflare, setup_logger
 from databases.base import SearchEngine
 from databases.other import Other
 from handlers.base import IndexHandler, NotFoundHandler