injectlib/YYeTsBot
1
0
Fork 0
mirror of https://github.com/tgbot-collection/YYeTsBot.git synced 2025-11-25 11:29:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

finish new banning system

Browse Source
This commit is contained in:
Aura
2022-04-10 13:59:00 +08:00
parent 2a419d832c
commit 9a17fe77dc
4 changed files with 85 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
yyetsweb/Mongo.py
View File

@@ -92,6 +92,15 @@ class OtherMongoResource(OtherResource, Mongo):
# reset
self.db["yyets"].update_many({}, {"$set": {"data.info.views": 0}})
def import_ban_user(self):
usernames = self.db["users"].find({"status.disable": True}, projection={"username": True})
r = Redis().r
r.delete("user_blacklist")
logging.info("Importing ban users to redis...%s", usernames)
for username in [u["username"] for u in usernames]:
r.hset("user_blacklist", username, 100)
r.close()
class AnnouncementMongoResource(AnnouncementResource, Mongo):
def get_announcement(self, page: int, size: int) -> dict:
@@ -507,7 +516,8 @@ class ResourceMongoResource(ResourceResource, Mongo):
{"data.info.id": resource_id},
{'$inc': {'data.info.views': 1}},
{'_id': False})
if not data:
return {}
if username:
user_like_data = self.db["users"].find_one({"username": username})
if user_like_data and resource_id in user_like_data.get("like", []):
@@ -653,13 +663,12 @@ class TopMongoResource(TopResource, Mongo):
def get_top_resource(self) -> dict:
area_dict = dict(ALL={"$regex": ".*"}, US="美国", JP="日本", KR="韩国", UK="英国")
all_data = {}
all_data = {"ALL": "全部"}
for abbr, area in area_dict.items():
data = self.db["yyets"].find({"data.info.area": area, "data.info.id": {"$ne": 233}}, self.projection). \
sort("data.info.views", pymongo.DESCENDING).limit(15)
all_data[abbr] = list(data)
area_dict["ALL"] = "全部"
all_data["class"] = area_dict
return all_data

54
yyetsweb/database.py
View File

@@ -55,60 +55,6 @@ class Redis:
return func
class AntiCrawler:
def __init__(self, instance):
self.tornado = instance
self.redis = Redis()
def execute(self) -> bool:
header_result = self.header_check()
ban_check = self.ban_check()
if header_result or ban_check:
return True
def header_check(self):
referer = self.tornado.request.headers.get("Referer")
resource_id = self.tornado.get_query_argument("id")
uri = self.tornado.request.uri
logging.info("Verifying: Referer:[%s] uri:[%s]", referer, uri)
if referer is None:
return True
if resource_id not in uri:
return True
if resource_id not in referer:
return True
def ban_check(self):
con = self.redis
ip = self.get_real_ip()
str_count = con.r.get(ip)
if str_count and int(str_count) > 10:
return True
def imprisonment(self, ip):
con = self.redis
# don't use incr - we need to set expire time
if con.r.exists(ip):
count_str = con.r.get(ip)
count = int(count_str)
count += 1
else:
count = 1
# ban rule: (count-10)*600
if count > 10:
ex = (count - 10) * 600
else:
ex = None
con.r.set(ip, count, ex)
def get_real_ip(self):
x_real = self.tornado.request.headers.get("X-Real-IP")
remote_ip = self.tornado.request.remote_ip
logging.debug("X-Real-IP:%s, Remote-IP:%s", x_real, remote_ip)
return x_real or remote_ip
class OtherResource():
def reset_top(self):
pass

104
yyetsweb/handler.py
View File

@@ -27,8 +27,7 @@ import zhconv
from tornado import escape, gen, web
from tornado.concurrent import run_on_executor
from database import AntiCrawler, CaptchaResource, Redis
from Mongo import Mongo
from database import CaptchaResource, Redis
escape.json_encode = lambda value: json.dumps(value, ensure_ascii=False)
logging.basicConfig(level=logging.INFO)
@@ -43,7 +42,63 @@ logging.info("%s Running with %s. %s", "#" * 10, adapter, "#" * 10)
index = pathlib.Path(__file__).parent.joinpath("templates", "index.html").as_posix()
class BaseHandler(web.RequestHandler):
class SecurityHandler(web.RequestHandler):
key = "user_blacklist"
def __init__(self, application, request, **kwargs):
super().__init__(application, request, **kwargs)
self.r = Redis().r
def prepare(self):
if self.check_request():
self.set_status(HTTPStatus.FORBIDDEN)
self.finish()
def data_received(self, chunk):
pass
def check_request(self):
ban = self.__ip_check()
user = self.__user_check()
result = ban or user
if result:
self.ban()
return result
def get_real_ip(self):
x_real = self.request.headers.get("X-Real-IP")
remote_ip = self.request.remote_ip
logging.debug("X-Real-IP:%s, Remote-IP:%s", x_real, remote_ip)
return x_real or remote_ip
def ban(self):
ip = self.get_real_ip()
self.r.incr(ip)
count = int(self.r.get(ip))
# ban rule: (count-10)*600
ex = 120 if count <= 10 else (count - 10) * 600
self.r.set(ip, count, ex)
user = self.get_current_user()
if user:
self.r.hincrby(self.key, user)
def get_current_user(self) -> str:
username = self.get_secure_cookie("username") or b""
return username.decode("u8")
def __user_check(self):
count = self.r.hget(self.key, self.get_current_user()) or 0
count = int(count)
if count >= 20:
return True
def __ip_check(self):
d = self.r.get(self.get_real_ip()) or 0
if int(d) >= 10:
return True
class BaseHandler(SecurityHandler):
executor = ThreadPoolExecutor(200)
class_name = f"Fake{adapter}Resource"
adapter_module = importlib.import_module(f"{adapter}")
@@ -54,29 +109,15 @@ class BaseHandler(web.RequestHandler):
with contextlib.suppress(ValueError):
self.json: dict = json.loads(self.request.body)
self.instance = getattr(self.adapter_module, self.class_name)()
self.db = Mongo()
self.ban_yellow_nazi()
def ban_yellow_nazi(self):
if self.db.is_user_blocked(self.get_current_user()):
self.set_status(HTTPStatus.FORBIDDEN, "You don't deserve it.")
real_ip = AntiCrawler(self).get_real_ip()
AntiCrawler(self).imprisonment(real_ip)
def write_error(self, status_code, **kwargs):
if status_code in [HTTPStatus.FORBIDDEN,
HTTPStatus.INTERNAL_SERVER_ERROR,
HTTPStatus.UNAUTHORIZED,
HTTPStatus.NOT_FOUND]:
HTTPStatus.NOT_FOUND,
HTTPStatus.INTERNAL_SERVER_ERROR,
]:
self.write(str(kwargs.get('exc_info')))
def data_received(self, chunk):
pass
def get_current_user(self) -> str:
username = self.get_secure_cookie("username") or b""
return username.decode("u8")
class TopHandler(BaseHandler):
class_name = f"Top{adapter}Resource"
@@ -131,7 +172,7 @@ class UserHandler(BaseHandler):
password = data["password"]
captcha = data.get("captcha")
captcha_id = data.get("captcha_id", "")
ip = AntiCrawler(self).get_real_ip()
ip = self.get_real_ip()
browser = self.request.headers['user-agent']
response = self.instance.login_user(username, password, captcha, captcha_id, ip, browser)
@@ -171,7 +212,7 @@ class UserHandler(BaseHandler):
# everytime we receive a GET request to this api, we'll update last_date and last_ip
username = self.get_current_user()
if username:
now_ip = AntiCrawler(self).get_real_ip()
now_ip = self.get_real_ip()
self.instance.update_user_last(username, now_ip)
@gen.coroutine
@@ -189,20 +230,12 @@ class ResourceHandler(BaseHandler):
@run_on_executor()
def get_resource_data(self):
ban = AntiCrawler(self)
if ban.execute():
logging.warning("%s@%s make you happy:-(", self.request.headers.get("user-agent"), ban.get_real_ip())
self.set_status(HTTPStatus.FORBIDDEN)
return {}
else:
resource_id = int(self.get_query_argument("id"))
username = self.get_current_user()
data = self.instance.get_resource_data(resource_id, username)
if not data:
# not found, dangerous
ip = ban.get_real_ip()
ban.imprisonment(ip)
self.ban()
self.set_status(HTTPStatus.NOT_FOUND)
data = {}
@@ -392,7 +425,7 @@ class CommentHandler(BaseHandler):
resource_id = payload["resource_id"]
comment_id = payload.get("comment_id")
real_ip = AntiCrawler(self).get_real_ip()
real_ip = self.get_real_ip()
username = self.get_current_user()
browser = self.request.headers['user-agent']
@@ -550,7 +583,7 @@ class AnnouncementHandler(BaseHandler):
payload = self.json
content = payload["content"]
real_ip = AntiCrawler(self).get_real_ip()
real_ip = self.get_real_ip()
browser = self.request.headers['user-agent']
self.instance.add_announcement(username, content, real_ip, browser)
@@ -725,6 +758,7 @@ class BlacklistHandler(BaseHandler):
class NotFoundHandler(BaseHandler):
def get(self): # for react app
self.ban()
self.render(index)
@@ -941,7 +975,7 @@ class SpamProcessHandler(BaseHandler):
obj_id = self.json.get("obj_id")
token = self.json.get("token")
ua = self.request.headers['user-agent']
ip = AntiCrawler(self).get_real_ip()
ip = self.get_real_ip()
logging.info("Authentication %s(%s) for spam API now...", ua, ip)
if token == os.getenv("TOKEN"):
return getattr(self.instance, method)(obj_id)

2
yyetsweb/server.py
View File

@@ -105,7 +105,9 @@ if __name__ == "__main__":
scheduler.add_job(sync_douban, trigger=CronTrigger.from_crontab("1 1 1 * *"))
scheduler.add_job(entry_dump, trigger=CronTrigger.from_crontab("2 2 * * *"))
scheduler.add_job(ResourceLatestMongoResource().refresh_latest_resource, 'interval', hours=1)
scheduler.add_job(OtherMongoResource().import_ban_user, 'interval', seconds=300)
scheduler.start()
options.define("p", default=8888, help="running port", type=int)
options.define("h", default='127.0.0.1', help="listen address", type=str)
options.parse_command_line()