avatar fix for phone number, continue GlitchTip fix

requirements.txt

@@ -1,4 +1,4 @@
-requests==2.28.2
+requests==2.29.0
 pytelegrambotapi==4.11.0
 beautifulsoup4==4.12.2
 tgbot-ping==1.0.7

yyetsweb/common/utils.py

@@ -12,12 +12,14 @@ import json
 import logging
 import os
 import pathlib
+import re
 import smtplib
 import time
 from datetime import datetime
 from email.header import Header
 from email.mime.text import MIMEText
 from email.utils import formataddr, parseaddr
+from hashlib import sha256
 
 import coloredlogs
 import pytz
@@ -36,6 +38,18 @@ def setup_logger():
     )
 
 
+def hide_phone(data: list):
+    for item in data:
+        if item["username"].isdigit() and len(item["username"]) == 11:
+            item["hash"] = sha256(item["username"].encode("u8")).hexdigest()
+            item["username"] = mask_phone(item["username"])
+    return data
+
+
+def mask_phone(num):
+    return re.sub(r"(\d{3})\d{4}(\d{4})", r"\g<1>****\g<2>", num)
+
+
 def ts_date(ts=None):
     # Let's always set the timezone to CST
     timestamp = ts or time.time()

yyetsweb/databases/base.py

@@ -116,7 +116,7 @@ class SearchEngine(Mongo):
         "comment": "$content",
         "commentID": {"$toString": "$_id"},
         "origin": "comment",
-        "hasAvatar": {"$toBool": "$avatar"},
+        "hasAvatar": "yes",
         "resourceID": "$resource_id",
         "resourceName": {"$first": "$resource.data.info.cnname"},
         "_id": {"$toString": "$_id"},

yyetsweb/databases/comment.py

@@ -74,16 +74,11 @@ class Comment(Mongo):
             cid = comment.get("id") or comment.get("_id")
             cid = str(cid)
             reactions = (
-                self.db["reactions"].find_one(
-                    {"comment_id": cid}, projection={"_id": False, "comment_id": False}
-                )
-                or {}
+                self.db["reactions"].find_one({"comment_id": cid}, projection={"_id": False, "comment_id": False}) or {}
             )
             for verb, users in reactions.items():
                 if users:
-                    comment.setdefault("reactions", []).append(
-                        {"verb": verb, "users": users}
-                    )
+                    comment.setdefault("reactions", []).append({"verb": verb, "users": users})
 
     def get_comment(self, resource_id: int, page: int, size: int, **kwargs) -> dict:
         self.inner_page = kwargs.get("inner_page", 1)
@@ -136,10 +131,7 @@ class Comment(Mongo):
     ) -> dict:
         user_data = self.db["users"].find_one({"username": username})
         # old user is allowed to comment without verification
-        if (
-            not self.is_old_user(username)
-            and user_data.get("email", {}).get("verified", False) is False
-        ):
+        if not self.is_old_user(username) and user_data.get("email", {}).get("verified", False) is False:
             return {
                 "status_code": HTTPStatus.TEMPORARY_REDIRECT,
                 "message": "你需要验证邮箱才能评论，请到个人中心进行验证",
@@ -228,16 +220,12 @@ class Comment(Mongo):
                 upsert=True,
             )
             # send email
-            parent_comment = self.db["comment"].find_one(
-                {"_id": ObjectId(parent_comment_id)}
-            )
+            parent_comment = self.db["comment"].find_one({"_id": ObjectId(parent_comment_id)})
             if resource_id == 233:
                 link = f"https://yyets.dmesg.app/discuss#{parent_comment_id}"
             else:
                 link = f"https://yyets.dmesg.app/resource?id={resource_id}#{parent_comment_id}"
-            user_info = self.db["users"].find_one(
-                {"username": parent_comment["username"], "email.verified": True}
-            )
+            user_info = self.db["users"].find_one({"username": parent_comment["username"], "email.verified": True})
             if user_info:
                 subject = "[人人影视下载分享站] 你的评论有了新的回复"
                 pt_content = content.split("</reply>")[-1]
@@ -299,14 +287,10 @@ class CommentReaction(Mongo):
             }
 
         if method == "POST":
-            self.db["reactions"].update_one(
-                {"comment_id": comment_id}, {"$addToSet": {verb: username}}, upsert=True
-            )
+            self.db["reactions"].update_one({"comment_id": comment_id}, {"$addToSet": {verb: username}}, upsert=True)
             code = HTTPStatus.CREATED
         elif method == "DELETE":
-            self.db["reactions"].update_one(
-                {"comment_id": comment_id}, {"$pull": {verb: username}}
-            )
+            self.db["reactions"].update_one({"comment_id": comment_id}, {"$pull": {verb: username}})
             code = HTTPStatus.ACCEPTED
         else:
             code = HTTPStatus.BAD_REQUEST
@@ -405,9 +389,7 @@ class CommentSearch(CommentNewest):
         for item in data:
             child_id: "list" = item.get("children", [])
             children = list(
-                self.db["comment"]
-                .find({"_id": {"$in": child_id}}, self.projection)
-                .sort("_id", pymongo.DESCENDING)
+                self.db["comment"].find({"_id": {"$in": child_id}}, self.projection).sort("_id", pymongo.DESCENDING)
             )
             self.convert_objectid(children)
             self.get_user_group(children)
@@ -418,9 +400,7 @@ class CommentSearch(CommentNewest):
 class Notification(Mongo):
     def get_notification(self, username, page, size):
         # .sort("_id", pymongo.DESCENDING).limit(size).skip((page - 1) * size)
-        notify = self.db["notification"].find_one(
-            {"username": username}, projection={"_id": False}
-        )
+        notify = self.db["notification"].find_one({"username": username}, projection={"_id": False})
         if not notify:
             return {
                 "username": username,
@@ -453,9 +433,7 @@ class Notification(Mongo):
     def get_content(self, id_list):
         comments = (
             self.db["comment"]
-            .find(
-                {"_id": {"$in": id_list}}, projection={"ip": False, "parent_id": False}
-            )
+            .find({"_id": {"$in": id_list}}, projection={"ip": False, "parent_id": False})
             .sort("_id", pymongo.DESCENDING)
         )
         comments = list(comments)
@@ -467,7 +445,7 @@ class Notification(Mongo):
                 {"_id": ObjectId(reply_to_id)},
                 projection={"content": True, "_id": False},
             )
-            comment["reply_to_content"] = rtc["content"]
+            comment["reply_to_content"] = getattr(rtc, "content", "")
 
         return comments
 

yyetsweb/databases/oauth.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 # coding: utf-8
+from hashlib import sha256
+
 from common.utils import ts_date
 from databases.base import Mongo
 
@@ -27,6 +29,7 @@ class OAuthRegister(Mongo):
                     "oldUser": True,
                     "source": source,
                     "uid": uid,
+                    "hash": sha256(username.encode("u8")).hexdigest(),
                 }
             )
             return {
@@ -40,17 +43,17 @@ class GitHubOAuth2Login(OAuthRegister):
     pass
 
 
-class MSOAuth2Login:
+class MSOAuth2Login(OAuthRegister):
     pass
 
 
-class GoogleOAuth2Login:
+class GoogleOAuth2Login(OAuthRegister):
     pass
 
 
-class TwitterOAuth2Login:
+class TwitterOAuth2Login(OAuthRegister):
     pass
 
 
-class FacebookAuth2Login:
+class FacebookAuth2Login(OAuthRegister):
     pass

yyetsweb/databases/other.py

@@ -8,6 +8,7 @@ import random
 import re
 import string
 import time
+from hashlib import sha256
 
 import pymongo
 import requests
@@ -165,6 +166,15 @@ class Other(Mongo):
             r.hset("user_blacklist", username, 100)
         r.close()
 
+    def fill_user_hash(self):
+        users = self.db["users"].find({"hash": {"$exists": False}}, projection={"username": True})
+        # do it old school
+        for user in users:
+            logging.info("Filling hash for %s", user)
+            username = user["username"]
+            hash_value = sha256(username.encode("u8")).hexdigest()
+            self.db["users"].update_one({"username": username}, {"$set": {"hash": hash_value}})
+
 
 class Captcha(Redis):
     def get_captcha(self, captcha_id):

yyetsweb/databases/resources.py

@@ -11,7 +11,7 @@ import requests
 import zhconv
 from tqdm import tqdm
 
-from common.utils import ts_date
+from common.utils import hide_phone, ts_date
 from databases.base import Mongo, Redis, SearchEngine
 from databases.comment import CommentSearch
 
@@ -52,7 +52,7 @@ class Resource(SearchEngine):
         returned = {"data": [], "comment": [], "extra": []}
         if search_type == "default":
             yyets = self.search_yyets(keyword)
-            comment = self.search_comment(keyword)
+            comment = hide_phone(self.search_comment(keyword))
             returned["data"] = yyets
             returned["comment"] = comment
             return returned
@@ -98,12 +98,13 @@ class Resource(SearchEngine):
             zimuzu_data.append(item["data"]["info"])
 
         # get comment
-        r = CommentSearch().get_comment(1, 2**10, keyword)
         c_search = []
-        for c in r.get("data", []):
+        comments = CommentSearch().get_comment(1, 2**10, keyword)
+        hide_phone(comments.get("data", []))
+        for c in comments.get("data", []):
             comment_rid = c["resource_id"]
-            d = self.db["yyets"].find_one({"data.info.id": comment_rid}, projection={"data.info": True})
-            if d:
+            res = self.db["yyets"].find_one({"data.info.id": comment_rid}, projection={"data.info": True})
+            if res:
                 c_search.append(
                     {
                         "username": c["username"],
@@ -111,9 +112,10 @@ class Resource(SearchEngine):
                         "comment": c["content"],
                         "commentID": c["id"],
                         "resourceID": comment_rid,
-                        "resourceName": d["data"]["info"]["cnname"],
+                        "resourceName": res["data"]["info"]["cnname"],
                         "origin": "comment",
                         "hasAvatar": c["hasAvatar"],
+                        "hash": c["hash"],
                     }
                 )
         # zimuzu -> comment -> extra

yyetsweb/databases/user.py

@@ -3,10 +3,12 @@
 import os
 import random
 import re
+from hashlib import md5, sha256
 from http import HTTPStatus
 
 import filetype
 import pymongo
+import requests
 from passlib.handlers.pbkdf2 import pbkdf2_sha256
 
 from common.utils import send_mail, ts_date
@@ -101,6 +103,7 @@ class User(Mongo, Redis):
                         date=ts_date(),
                         ip=ip,
                         browser=browser,
+                        hash=sha256(username.encode("u8")).hexdigest(),
                     )
                 )
                 returned_value["status_code"] = HTTPStatus.CREATED
@@ -177,11 +180,23 @@ class UserAvatar(User, Mongo):
 
         return {"status_code": HTTPStatus.CREATED, "message": "头像上传成功"}
 
-    def get_avatar(self, username):
+    def get_avatar(self, username, user_hash=None):
+        if user_hash:
+            user = self.db["users"].find_one({"hash": user_hash})
+        else:
             user = self.db["users"].find_one({"username": username})
+        if user:
             img = user.get("avatar", b"")
             mime = filetype.guess_mime(img)
             return {"image": img, "content_type": mime}
+        elif "@" in username:
+            # fallback to gravatar
+            url = f"https://gravatar.webp.se/avatar/{md5(username.encode('u8')).hexdigest()}"
+            img = requests.get(url).content
+            mime = filetype.guess_mime(img)
+            return {"image": img, "content_type": mime}
+        else:
+            return {"image": None, "content_type": None}
 
 
 class UserEmail(Mongo):

yyetsweb/handlers/comment.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 # coding: utf-8
-import re
 from http import HTTPStatus
 from pathlib import Path
 
 from tornado import gen, web
 from tornado.concurrent import run_on_executor
 
+from common.utils import hide_phone
 from handlers.base import BaseHandler
 
 filename = Path(__file__).name.split(".")[0]
@@ -15,13 +15,6 @@ filename = Path(__file__).name.split(".")[0]
 class CommentHandler(BaseHandler):
     filename = filename
 
-    @staticmethod
-    def hide_phone(data: list):
-        for item in data:
-            if item["username"].isdigit() and len(item["username"]) == 11:
-                item["username"] = re.sub(r"(\d{3})\d{4}(\d{4})", r"\g<1>****\g<2>", item["username"])
-        return data
-
     @run_on_executor()
     def get_comment(self):
         query_id = self.get_argument("resource_id", "0")
@@ -44,7 +37,7 @@ class CommentHandler(BaseHandler):
             inner_page=inner_page,
             comment_id=comment_id,
         )
-        self.hide_phone((comment_data["data"]))
+        hide_phone((comment_data["data"]))
         return comment_data
 
     @run_on_executor()
@@ -144,7 +137,7 @@ class CommentChildHandler(CommentHandler):
             self.set_status(HTTPStatus.BAD_REQUEST)
             return {"status": False, "message": "请提供 parent_id"}
         comment_data = self.instance.get_comment(parent_id, page, size)
-        self.hide_phone((comment_data["data"]))
+        hide_phone((comment_data["data"]))
         return comment_data
 
     @gen.coroutine
@@ -162,7 +155,7 @@ class CommentNewestHandler(CommentHandler):
         page = int(self.get_argument("page", "1"))
 
         comment_data = self.instance.get_comment(page, size)
-        self.hide_phone((comment_data["data"]))
+        hide_phone((comment_data["data"]))
         return comment_data
 
     @gen.coroutine
@@ -180,7 +173,7 @@ class CommentSearchHandler(CommentHandler):
         page = int(self.get_argument("page", "1"))
         keyword = self.get_argument("keyword", "")
         comment_data = self.instance.get_comment(page, size, keyword)
-        self.hide_phone((comment_data["data"]))
+        hide_phone((comment_data["data"]))
         return comment_data
 
     @gen.coroutine

yyetsweb/handlers/resources.py

@@ -19,7 +19,8 @@ class ResourceHandler(BaseHandler):
 
     @run_on_executor()
     def get_resource_data(self):
-        resource_id = int(self.get_query_argument("id"))
+        query = self.get_query_argument("id", None)
+        resource_id = int(query) if query.isdigit() else 0
         username = self.get_current_user()
         if str(resource_id) in os.getenv("HIDDEN_RESOURCE", "").split(","):
             self.set_status(HTTPStatus.NOT_FOUND)

yyetsweb/handlers/user.py

@@ -27,9 +27,7 @@ class UserHandler(BaseHandler):
         ip = self.get_real_ip()
         browser = self.request.headers["user-agent"]
 
-        response = self.instance.login_user(
-            username, password, captcha, captcha_id, ip, browser
-        )
+        response = self.instance.login_user(username, password, captcha, captcha_id, ip, browser)
         if response["status_code"] in (HTTPStatus.CREATED, HTTPStatus.OK):
             self.set_login(username)
         else:
@@ -96,7 +94,8 @@ class UserAvatarHandler(BaseHandler):
 
     @run_on_executor()
     def get_avatar(self, username):
-        data = self.instance.get_avatar(username)
+        user_hash = self.get_query_argument("hash", None)
+        data = self.instance.get_avatar(username, user_hash)
         if data["image"]:
             self.set_header("Content-Type", data["content_type"])
             return data["image"]

yyetsweb/server.py

@@ -167,6 +167,7 @@ if __name__ == "__main__":
     scheduler.add_job(sync_douban, trigger=CronTrigger.from_crontab("1 1 1 * *"))
     scheduler.add_job(entry_dump, trigger=CronTrigger.from_crontab("2 2 1 * *"))
     scheduler.add_job(Other().import_ban_user, "interval", seconds=300)
+    scheduler.add_job(Other().fill_user_hash, "interval", seconds=60)
     scheduler.add_job(Cloudflare().clear_fw, trigger=CronTrigger.from_crontab("0 0 */3 * *"))
     scheduler.add_job(YYSub().run, trigger=CronTrigger.from_crontab("0 1 * * *"))