injectlib/YYeTsBot
1
0
Fork 0
mirror of https://github.com/tgbot-collection/YYeTsBot.git synced 2025-11-26 03:44:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor and add newzmz

Browse Source 
add userinfo in comment, fix docker cache
This commit is contained in:
BennyThink
2021-06-26 15:00:01 +08:00
parent 07186e0bf8
commit 7d33d27ea4
9 changed files with 278 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
API.md
View File

@@ -1,3 +1,19 @@
# 需求与待开发功能
## FE
- [ ] group为admin特殊显示，评论接口已返回group信息
- [ ] 评论楼中楼
- [ ] 评论通知（浏览器通知）
- [ ] 最新评论（id为-1）
- [ ] 联合搜索，当本地数据库搜索不到数据时，会返回extra字段
# BE
- [ ] 添加资源API
- [x] 联合搜索：字幕侠、new字幕组、追新番
- [ ] 评论通知，需要新接口
- [ ] grafana面板
- [ ] 豆瓣接口
- [ ] 用户体系（添加邮箱，邮件支持，找回密码）
# 资源
## 获取指定id的资源
@@ -75,13 +91,14 @@
}
```
当数据库搜索不到资源时，会尝试从追新番和字幕侠搜索，返回如下
当数据库搜索不到资源时，会尝试从字幕侠、new字幕组和追新番搜索，返回如下
```json
{
"data": [],
"extra": {
"女人不杀生": "https://www.zimuxia.cn/portfolio/%e5%a5%b3%e4%ba%ba%e4%b8%8d%e6%9d%80%e7%94%9f"
"name": "东城梦魇",
"url": "https://www.zimuxia.cn/portfolio/%e4%b8%9c%e5%9f%8e%e6%a2%a6%e9%ad%87"
}
}
```
@@ -196,9 +213,9 @@
{
"username": "Benny",
"date": "2021-03-12 11:11:11",
"last_date": "2021-03-15 13:11:18",
"lastDate": "2021-03-15 13:11:18",
"ip": "1.1.1.1",
"last_ip": "2.2.2.2",
"lastIP": "2.2.2.2",
"browser": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.85 Safari/537.36",
"like": [
11133
@@ -287,7 +304,7 @@
* inner_size: 内嵌评论数量，默认5
* inner_page: 内嵌评论当前页，默认1
返回 楼中楼评论
返回 楼中楼评论，group表示用户所属组，admin是管理员，user是普通用户
```json
{
@@ -299,7 +316,10 @@
"content": "父评论benny",
"resource_id": 233,
"type": "parent",
"id": "60d1bae2d87ce6e9a2934a0f"
"id": "60d1bae2d87ce6e9a2934a0f",
"group": [
"admin"
]
},
{
"username": "Benny",
@@ -308,16 +328,21 @@
"content": "父评论benny",
"resource_id": 233,
"type": "parent",
"ack": false,
"group": [
"admin"
],
"children": [
{
"username": "admin",
"username": "test",
"date": "2021-06-22 18:25:12",
"browser": "PostmanRuntime/7.28.0",
"content": "admin子评2论2",
"resource_id": 233,
"type": "child",
"id": "60d1ba88d87ce6e9a2934a0c"
"id": "60d1ba88d87ce6e9a2934a0c",
"group": [
"user"
]
},
{
"username": "admin",
@@ -326,7 +351,10 @@
"content": "admin子评论2",
"resource_id": 233,
"type": "child",
"id": "60d1ba84d87ce6e9a2934a0a"
"id": "60d1ba84d87ce6e9a2934a0a",
"group": [
"user"
]
}
],
"id": "60d1ba6cd87ce6e9a2934a08"

3
Dockerfile
View File

@@ -18,7 +18,8 @@ RUN apk add git
COPY YYeTsFE/package.json /YYeTsBot/YYeTsFE/
COPY YYeTsFE/yarn.lock /YYeTsBot/YYeTsFE/
RUN yarn
COPY . /YYeTsBot/
COPY YYeTsFE /YYeTsBot/YYeTsFE/
COPY .git/modules /YYeTsBot/.git/modules/
RUN echo "gitdir: ../.git/modules/YYeTsFE" > .git
RUN yarn run release

2
Makefile
View File

@@ -4,12 +4,10 @@ default:
dev:
rm -f YYeTsFE/.env
git checkout docker-compose.yml
git pull
git submodule update --remote
cp .env YYeTsFE/.env
docker build -t bennythink/yyetsbot .
cp ../docker-compose.yml ./docker-compose.yml
docker-compose up -d
clean:

10
README.md
View File

@@ -31,9 +31,8 @@ help - 帮助
credits - 致谢
ping - 运行状态
settings - 获取公告
zimuxia_offline - 字幕侠离线数据
zimuxia_online - 字幕侠在线数据
yyets_online - 人人影视在线数据
newzmz_online - new字幕组在线数据
yyets_offline - 人人影视离线数据
```
@@ -54,7 +53,7 @@ yyets_offline - 人人影视离线数据
## 指定字幕组搜索
目前只支持YYeTsOffline和ZimuxiaOnline
目前只支持YYeTsOffline、ZimuxiaOnline和NewzmzOnline
![](assets/3.png)
@@ -76,6 +75,7 @@ yyets_offline - 人人影视离线数据
* Windows用户双击 `start_windows.bat`
* macOS 在终端中执行 `bash start_unix.sh`
* Linux用户：没你的份
打开浏览器 http://127.0.0.1:8888 就可以看到熟悉的搜索界面啦！
@@ -86,9 +86,9 @@ yyets_offline - 人人影视离线数据
# Credits
* [人人影视](http://www.zmz2019.com/)
* [追新番](http://www.zhuixinfan.com/main.php)
* [磁力下载站](http://oabt005.com/home.html)
* [追新番](http://www.fanxinzhui.com/)
* [FIX字幕侠](https://www.zimuxia.cn/)
* [new字幕组](https://newzmz.com/)
# 支持我

4
requirements.txt
View File

@@ -1,5 +1,5 @@
requests==2.24.0
pytelegrambotapi==3.7.7
pytelegrambotapi==3.7.9
beautifulsoup4==4.9.1
tgbot-ping
redis==3.5.3
@@ -11,3 +11,5 @@ captcha==0.3
passlib==1.7.4
fakeredis==1.5.0
pytz==2021.1
requests[socks]

30
yyetsbot/config.py
View File

@@ -17,26 +17,32 @@ SHARE_URL = "http://www.rrys2020.com/resource/ushare"
SHARE_WEB = "http://got002.com/resource.html?code={code}"
# http://got002.com/api/v1/static/resource/detail?code=9YxN91
SHARE_API = "http://got002.com/api/v1/static/resource/detail?code={code}"
# fix
# fix zimuxia
FIX_RESOURCE = "https://www.zimuxia.cn/portfolio/{name}"
FIX_SEARCH = "https://www.zimuxia.cn/?s={kw}"
# zhuixinfan
ZHUIXINFAN_SEARCH = "http://www.fanxinzhui.com/list?k={}"
ZHUIXINFAN_RESOURCE = "http://www.fanxinzhui.com{}"
# cloudflare worker
# yyets website
WORKERS = "https://yyets.dmesg.app/resource.html?id={id}"
# new zmz
NEWZMZ_SEARCH = "https://newzmz.com/subres/index/getres.html?keyword={}"
NEWZMZ_RESOURCE = "https://ysfx.tv/view/{}"
# authentication config
TOKEN = os.environ.get("TOKEN") or "TOKEN"
USERNAME = os.environ.get("USERNAME") or "USERNAME"
PASSWORD = os.environ.get("PASSWORD") or "password"
TOKEN = os.getenv("TOKEN") or "TOKEN"
USERNAME = os.getenv("USERNAME") or "USERNAME"
PASSWORD = os.getenv("PASSWORD") or "password"
# network and server config
PROXY = os.environ.get("PROXY")
REDIS = os.environ.get("REDIS") or "redis"
MONGO = os.environ.get("MONGO") or "mongo"
PROXY = os.getenv("PROXY")
REDIS = os.getenv("REDIS") or "redis"
MONGO = os.getenv("MONGO") or "mongo"
# other
MAINTAINER = os.environ.get("MAINTAINER")
REPORT = os.environ.get("REPORT") or False
# This name must match class name, other wise this bot won't functional.
FANSUB_ORDER: str = os.environ.get("ORDER") or 'YYeTsOffline,ZimuxiaOnline,ZhuixinfanOnline'
MAINTAINER = os.getenv("MAINTAINER")
REPORT = os.getenv("REPORT") or False
# This name must match class name, other wise this bot won't running.
FANSUB_ORDER: str = os.getenv("ORDER") or 'YYeTsOffline,ZimuxiaOnline,NewzmzOnline,ZhuixinfanOnline'

392
yyetsbot/fansub.py
View File

@@ -11,17 +11,16 @@ import sys
import json
import hashlib
import contextlib
import re
import requests
import pymongo
import redis
import fakeredis
from bs4 import BeautifulSoup
from config import (YYETS_SEARCH_URL, GET_USER, BASE_URL, SHARE_WEB,
SHARE_URL, WORKERS, SHARE_API, USERNAME, PASSWORD,
AJAX_LOGIN, REDIS, FANSUB_ORDER, FIX_SEARCH, MONGO,
ZHUIXINFAN_SEARCH, ZHUIXINFAN_RESOURCE)
import redis
from config import (WORKERS, REDIS, FANSUB_ORDER, FIX_SEARCH, MONGO,
ZHUIXINFAN_SEARCH, ZHUIXINFAN_RESOURCE, NEWZMZ_SEARCH, NEWZMZ_RESOURCE)
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(filename)s [%(levelname)s]: %(message)s')
@@ -32,6 +31,81 @@ session.headers.update({"User-Agent": ua})
this_module = sys.modules[__name__]
class Redis:
def __init__(self):
if os.getenv("DISABLE_REDIS"):
self.r = fakeredis.FakeStrictRedis()
else:
self.r = redis.StrictRedis(host=REDIS, decode_responses=True)
def __del__(self):
self.r.close()
@classmethod
def preview_cache(cls, timeout):
def func(fun):
def inner(*args, **kwargs):
search_text = args[1]
cache_value = cls().r.get(search_text)
if cache_value:
logging.info('🎉 Preview cache hit for %s %s', fun, search_text)
return json.loads(cache_value)
else:
logging.info('😱 Preview cache expired. Running %s %s', fun, search_text)
res = fun(*args, **kwargs)
# if res len is 1, then it means we have no search result at all.
if len(res) != 1:
json_str = json.dumps(res, ensure_ascii=False)
cls().r.set(search_text, json_str, ex=timeout)
# save hash->url mapping
res_copy = res.copy()
res_copy.pop("class")
for url_hash, value in res_copy.items():
cls().r.hset(url_hash, mapping=value)
return res
return inner
return func
@classmethod
def result_cache(cls, timeout):
def func(fun):
def inner(*args, **kwargs):
# this method will convert hash to url
url_or_hash = args[1]
if re.findall(r"http[s]?://", url_or_hash):
# means this is a url
url_hash = hashlib.sha1(url_or_hash.encode('u8')).hexdigest()
cls().r.hset(url_hash, mapping={"url": url_or_hash})
else:
# this is cache, retrieve real url from redis
url_or_hash = cls().r.hget(url_or_hash, "url")
if not url_or_hash:
url_or_hash = ""
url = url_or_hash
del url_or_hash
cache_value = cls().r.hgetall(url)
if cache_value:
logging.info('🎉 Result cache hit for %s %s', fun, url)
return cache_value
else:
logging.info('😱 Result cache expired. Running %s %s', fun, url)
new_args = (args[0], url)
res = fun(*new_args, **kwargs)
# we must have an result for it,
cls().r.hset(url, mapping=res)
cls().r.expire(url, timeout)
return res
return inner
return func
class BaseFansub:
"""
all the subclass should implement three kinds of methods:
@@ -40,26 +114,24 @@ class BaseFansub:
3. search_result as this is critical for bot to draw markup
"""
label = None
cookie_file = None
def __init__(self):
self.data = None
self.url = None
self.redis = redis.StrictRedis(host=REDIS, decode_responses=True)
# how to store data in redis:
# either online or offline, only these two type of data will be stored
# http://z.cn --> <html><head>.... valid for 12 hours, cache purpose
# 1273hda_hash_of_url - {url:http://z.cn,class:xxclass} forever
self.redis = Redis().r
@property
def id(self):
# implement how to get the unique id for this resource
return None
def __get_search_html__(self, kw: str) -> str:
def get_html(self, link: str, encoding=None) -> str:
# return html text of search page
pass
logging.info("[%s] Searching for %s", self.__class__.__name__, link)
with session.get(link) as r:
if encoding is not None:
r.encoding = encoding
html = r.text
return html
def search_preview(self, search_text: str) -> dict:
# try to retrieve critical information from html
@@ -67,26 +139,18 @@ class BaseFansub:
# {"url1": "name1", "url2": "name2", "source":"yyets"}
pass
def search_result(self, resource_url: str) -> dict:
def search_result(self, url_or_hash: str) -> dict:
"""
This will happen when user click one of the button, only by then we can know the resource link
From the information above, try to get a detail dict structure.
This method should check cache first if applicable
This method should set self.link and self.data
This method should call __execute_online_search
:param resource_url: in entry method, this variable is hash. Otherwise it's plain url
:param url_or_hash: url or hash.
:return: {"all": dict_result, "share": share_link, "cnname": cnname}
"""
pass
def __execute_search_result__(self) -> dict:
"""
Do the real search job, without any cache mechanism
:return: {"all": rss_result, "share": share_link, "cnname": cnname}
"""
pass
def __login_check(self):
pass
@@ -101,118 +165,17 @@ class BaseFansub:
with open(self.cookie_file, 'rb') as f:
return pickle.load(f)
def __get_from_cache__(self, url: str, method_name: str) -> dict:
logging.info("[%s] Reading data from cache %s", self.label, url)
data = self.redis.get(url)
if data:
logging.info("😄 Cache hit")
return json.loads(data)
else:
logging.info("😱 Cache miss")
result_method = getattr(self, method_name)
self.__save_to_cache__(url, result_method())
return self.__get_from_cache__(url, method_name)
def __save_to_cache__(self, url: str, value: dict, ex=3600 * 12) -> None:
data = json.dumps(value, ensure_ascii=False)
self.redis.set(url, data, ex=ex)
class YYeTsBase(BaseFansub):
@property
def id(self):
# implement how to get the unique id for this resource
rid = self.url.split('/')[-1]
return rid
class YYeTsOnline(YYeTsBase):
label = "yyets online"
cookie_file = os.path.join("data", "cookies.dump")
def __get_search_html__(self, kw: str) -> str:
# don't have to login here
logging.info("[%s] Searching for %s", self.label, kw)
r = session.get(YYETS_SEARCH_URL.format(kw=kw))
r.close()
return r.text
def search_preview(self, search_text: str) -> dict:
# yyets online
html_text = self.__get_search_html__(search_text)
logging.info('[%s] Parsing html...', self.label)
soup = BeautifulSoup(html_text, 'html.parser')
link_list = soup.find_all("div", class_="clearfix search-item")
dict_result = {}
for block in link_list:
name = block.find_all('a')[-1].text
url = BASE_URL + block.find_all('a')[-1].attrs['href']
url_hash = hashlib.sha1(url.encode('u8')).hexdigest()
dict_result[url_hash] = name
self.redis.hset(url_hash, mapping={"class": self.__class__.__name__, "url": url})
dict_result["source"] = self.label
return dict_result
def search_result(self, resource_url: str) -> dict:
# yyets online
self.url = resource_url
self.data = self.__get_from_cache__(self.url, self.__execute_search_result__.__name__)
return self.data
def __execute_search_result__(self) -> dict:
logging.info("[%s] Loading detail page %s", self.label, self.url)
share_link, api_res = self.__get_share_page()
cnname = api_res["data"]["info"]["cnname"]
self.data = {"all": api_res, "share": share_link, "cnname": cnname}
return self.data
def __login_check(self):
logging.debug("[%s] Checking login status...", self.label)
if not os.path.exists(self.cookie_file):
logging.warning("[%s] Cookie file not found", self.label)
self.__manual_login()
r = session.get(GET_USER, cookies=self.__load_cookies__())
if not r.json()['status'] == 1:
self.__manual_login()
def __manual_login(self):
data = {"account": USERNAME, "password": PASSWORD, "remember": 1}
logging.info("[%s] Login in as %s", self.label, data)
r = requests.post(AJAX_LOGIN, data=data)
resp = r.json()
if resp.get('status') == 1:
logging.debug("Login success! %s", r.cookies)
self.__save_cookies__(r.cookies)
else:
logging.error("Login failed! %s", resp)
sys.exit(1)
r.close()
def __get_share_page(self):
self.__login_check()
rid = self.id
res = session.post(SHARE_URL, data={"rid": rid}, cookies=self.__load_cookies__()).json()
share_code = res['data'].split('/')[-1]
share_url = SHARE_WEB.format(code=share_code)
logging.info("[%s] Share url is %s", self.label, share_url)
# get api response
api_response = session.get(SHARE_API.format(code=share_code)).json()
return share_url, api_response
class YYeTsOffline(YYeTsBase):
label = "yyets offline"
class YYeTsOffline(BaseFansub):
def __init__(self, db="zimuzu", col="yyets"):
super().__init__()
self.mongo = pymongo.MongoClient(host=MONGO)
self.collection = self.mongo[db][col]
@Redis.preview_cache(3600)
def search_preview(self, search_text: str) -> dict:
logging.info("[%s] Loading offline data from MongoDB...", self.label)
logging.info("[%s] Loading offline data from MongoDB...", self.__class__.__name__)
projection = {'_id': False, 'data.info': True}
data = self.collection.find({
@@ -226,43 +189,39 @@ class YYeTsOffline(YYeTsBase):
results = {}
for item in data:
info = item["data"]["info"]
fake_url = "http://www.rrys2020.com/resource/{}".format(info["id"])
url_hash = hashlib.sha1(fake_url.encode('u8')).hexdigest()
results[url_hash] = info["cnname"] + info["enname"] + info["aliasname"]
self.redis.hset(url_hash, mapping={"class": self.__class__.__name__, "url": fake_url})
url = "https://yyets.dmesg.app/resource.html?id={}".format(info["id"])
url_hash = hashlib.sha1(url.encode('u8')).hexdigest()
all_name = info["cnname"] + info["enname"] + info["aliasname"]
results[url_hash] = {
"name": all_name,
"url": url,
"class": self.__class__.__name__
}
logging.info("[%s] Offline search complete", self.label)
results["source"] = self.label
logging.info("[%s] Offline search complete", self.__class__.__name__)
results["class"] = self.__class__.__name__
return results
@Redis.result_cache(600)
def search_result(self, resource_url) -> dict:
# yyets offline
self.url = resource_url
# http://www.rrys2020.com/resource/10017
rid = self.url.split("/resource/")[1]
# https://yyets.dmesg.app/resource.html?id=37089
rid = resource_url.split("id=")[1]
data: dict = self.collection.find_one({"data.info.id": int(rid)}, {'_id': False})
name = data["data"]["info"]["cnname"]
self.data = {"all": data, "share": WORKERS.format(id=rid), "cnname": name}
return self.data
return {"all": json.dumps(data, ensure_ascii=False), "share": WORKERS.format(id=rid), "cnname": name}
def __del__(self):
self.mongo.close()
class ZimuxiaOnline(BaseFansub):
label = "zimuxia online"
def __get_search_html__(self, kw: str) -> str:
# don't have to login here
logging.info("[%s] Searching for %s", self.label, kw)
r = session.get(FIX_SEARCH.format(kw=kw))
r.close()
return r.text
@Redis.preview_cache(3600)
def search_preview(self, search_text: str) -> dict:
# zimuxia online
html_text = self.__get_search_html__(search_text)
logging.info('[%s] Parsing html...', self.label)
search_url = FIX_SEARCH.format(kw=search_text)
html_text = self.get_html(search_url)
logging.info('[%s] Parsing html...', self.__class__.__name__)
soup = BeautifulSoup(html_text, 'html.parser')
link_list = soup.find_all("h2", class_="post-title")
@@ -272,53 +231,32 @@ class ZimuxiaOnline(BaseFansub):
url = link.a['href']
url_hash = hashlib.sha1(url.encode('u8')).hexdigest()
name = link.a.text
dict_result[url_hash] = name
self.redis.hset(url_hash, mapping={"class": self.__class__.__name__, "url": url})
dict_result["source"] = self.label
dict_result[url_hash] = {
"url": url,
"name": name,
"class": self.__class__.__name__
}
dict_result["class"] = self.__class__.__name__
return dict_result
@Redis.result_cache(600)
def search_result(self, resource_url: str) -> dict:
# zimuxia online
self.url = resource_url
self.data = self.__get_from_cache__(self.url, self.__execute_search_result__.__name__)
return self.data
def __execute_search_result__(self) -> dict:
logging.info("[%s] Loading detail page %s", self.label, self.url)
cnname, html_text = self.obtain_all_response()
self.data = {"all": html_text, "share": self.url, "cnname": cnname}
return self.data
def obtain_all_response(self) -> (str, str):
r = session.get(self.url)
soup = BeautifulSoup(r.text, 'html.parser')
logging.info("[%s] Loading detail page %s", self.__class__.__name__, resource_url)
html = self.get_html(resource_url)
soup = BeautifulSoup(html, 'html.parser')
cnname = soup.title.text.split("|")[0]
return cnname, dict(html=r.text)
class ZimuxiaOffline(BaseFansub):
label = "zimuxia offline"
def search_preview(self, search_text: str) -> dict:
pass
def search_result(self, resource_url) -> dict:
pass
return {"all": html, "share": resource_url, "cnname": cnname}
class ZhuixinfanOnline(BaseFansub):
label = "zhuixinfan online"
def __get_search_html__(self, kw: str) -> str:
logging.info("[%s] Searching for %s", self.label, kw)
r = session.get(ZHUIXINFAN_SEARCH.format(kw))
r.close()
return r.text
@Redis.preview_cache(3600)
def search_preview(self, search_text: str) -> dict:
# zhuixinfan online
html_text = self.__get_search_html__(search_text)
logging.info('[%s] Parsing html...', self.label)
search_link = ZHUIXINFAN_SEARCH.format(search_text)
html_text = self.get_html(search_link)
logging.info('[%s] Parsing html...', self.__class__.__name__)
soup = BeautifulSoup(html_text, 'html.parser')
link_list = soup.find_all("ul", class_="resource_list")
@@ -329,46 +267,75 @@ class ZhuixinfanOnline(BaseFansub):
name = link.dd.text
url = ZHUIXINFAN_RESOURCE.format(link.dd.a["href"])
url_hash = hashlib.sha1(url.encode('u8')).hexdigest()
dict_result[url_hash] = name
self.redis.hset(url_hash, mapping={"class": self.__class__.__name__, "url": url, "name": name})
dict_result["source"] = self.label
dict_result[url_hash] = {
"url": url,
"name": name,
"class": self.__class__.__name__
}
dict_result["class"] = self.__class__.__name__
return dict_result
def search_result(self, resource_url: str) -> dict:
@Redis.result_cache(1800)
def search_result(self, url: str) -> dict:
# zhuixinfan online
self.url = resource_url
self.data = self.__execute_search_result__()
return self.data
# {"all": dict_result, "share": share_link, "cnname": cnname}
# don't worry, url_hash will become real url
logging.info("[%s] Loading detail page %s", self.__class__.__name__, url)
html = self.get_html(url, "utf-8")
# 解析获得cnname等信息
soup = BeautifulSoup(html, 'html.parser')
cnname = soup.title.text.split("_")[0]
return {"all": html, "share": url, "cnname": cnname}
def __execute_search_result__(self) -> dict:
logging.info("[%s] Loading detail page %s", self.label, self.url)
url_hash = hashlib.sha1(self.url.encode('u8')).hexdigest()
cnname = self.redis.hget(url_hash, "name")
# TODO
self.data = {"all": "不好意思，还没做呢……", "share": self.url, "cnname": cnname}
return self.data
class NewzmzOnline(BaseFansub):
@Redis.preview_cache(3600)
def search_preview(self, search_text: str) -> dict:
# zhuixinfan online
search_link = NEWZMZ_SEARCH.format(search_text)
html_text = self.get_html(search_link)
search_response = json.loads(html_text)
dict_result = {}
for item in search_response["data"]:
url = NEWZMZ_RESOURCE.format(item["link_url"].split("-")[1])
url_hash = hashlib.sha1(url.encode('u8')).hexdigest()
dict_result[url_hash] = {
"url": url,
"name": item["name"] + item["name_eng"],
"class": self.__class__.__name__
}
dict_result["class"] = self.__class__.__name__
return dict_result
@Redis.result_cache(1800)
def search_result(self, url: str) -> dict:
logging.info("[%s] Loading detail page %s", self.__class__.__name__, url)
html = self.get_html(url)
# 解析获得cnname等信息
soup = BeautifulSoup(html, 'html.parser')
cnname = soup.title.text.split("-")[0]
return {"all": html, "share": url, "cnname": cnname}
class FansubEntrance(BaseFansub):
order = FANSUB_ORDER.split(",")
def search_preview(self, search_text: str) -> dict:
source = "聪明机智温柔可爱善良的Benny"
class_ = "聪明机智温柔可爱善良的Benny"
for sub_str in self.order:
logging.info("Looping from %s", sub_str)
fc = globals().get(sub_str)
result = fc().search_preview(search_text)
# this result contains source:sub, so we'll pop and add it
source = result.pop("source")
class_ = result.pop("class")
if result:
logging.info("Result hit in %s %s", sub_str, fc)
FansubEntrance.fansub_class = fc
result["source"] = source
result["class"] = class_
return result
return dict(source=source)
return {"class": class_}
def search_result(self, resource_url_hash: str) -> dict:
# entrance
@@ -403,6 +370,9 @@ for sub_name in globals().copy():
vars()[cmd_name] = m
if __name__ == '__main__':
a = ZimuxiaOnline()
v = a.search_preview("女人为何")
print(v)
sub = NewzmzOnline()
# search = sub.search_preview("法")
# print(search)
uh = "914a549bc15e11a610293779761c5dd3f047ceb0"
result = sub.search_result(uh)
print(json.dumps(result, ensure_ascii=False))

19
yyetsbot/yyetsbot.py
View File

@@ -25,7 +25,7 @@ from config import PROXY, TOKEN, YYETS_SEARCH_URL, MAINTAINER, REPORT, FANSUB_OR
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(filename)s [%(levelname)s]: %(message)s')
if PROXY:
apihelper.proxy = {'http': PROXY}
apihelper.proxy = {'https': PROXY}
bot = telebot.TeleBot(TOKEN, num_threads=100)
angry_count = 0
@@ -127,13 +127,11 @@ for sub_name in dir(fansub):
tv_name: str = re.findall(r"/.*line\s*(\S*)", message.text)[0]
class_name: str = re.findall(r"/(.*line)", message.text)[0]
class_ = getattr(fansub, class_name)
if class_name not in ("zimuxia_online", "yyets_offline"):
bot.send_message(message.chat.id, f"{class_.label}: under dev")
return
if not tv_name:
bot.send_message(message.chat.id, f"{class_.label}: 请附加你要搜索的剧集名称，如 `/{class_name} 逃避可耻`",
bot.send_message(message.chat.id, f"{class_.__name__}: 请附加你要搜索的剧集名称，如 `/{class_name} 逃避可耻`",
parse_mode='markdown')
return
else:
setattr(message, "text", tv_name)
@@ -211,11 +209,10 @@ def base_send_search(message, instance=None):
markup = types.InlineKeyboardMarkup()
source = result.get("source")
result.pop("source")
for url, detail in result.items():
# we don't need to save which fansub class we used here, because we saved an url and that's good enough.
btn = types.InlineKeyboardButton(detail, callback_data="choose%s" % url)
source = result.get("class")
result.pop("class")
for url_hash, detail in result.items():
btn = types.InlineKeyboardButton(detail["name"], callback_data="choose%s" % url_hash)
markup.add(btn)
if result:
@@ -262,7 +259,7 @@ def magic_recycle(fan, call, url_hash):
def choose_link(call):
fan = fansub.FansubEntrance()
bot.send_chat_action(call.message.chat.id, 'typing')
# call.data is url, with sha1, http://www.rrys2020.com/resource/36588
# call.data is url_hash, with sha1, http://www.rrys2020.com/resource/36588
resource_url_hash = re.findall(r"choose(\S*)", call.data)[0]
if magic_recycle(fan, call, resource_url_hash):
return

42
yyetsweb/Mongo.py
View File

@@ -21,7 +21,7 @@ from database import (AnnouncementResource, BlacklistResource, CommentResource,
GrafanaQueryResource, MetricsResource, NameResource, OtherResource,
TopResource, UserLikeResource, UserResource, CaptchaResource, Redis)
from utils import ts_date
from fansub import ZhuixinfanOnline, ZimuxiaOnline
from fansub import ZhuixinfanOnline, ZimuxiaOnline, NewzmzOnline
mongo_host = os.getenv("mongo") or "localhost"
@@ -126,10 +126,18 @@ class CommentMongoResource(CommentResource, Mongo):
children_data = self.db["comment"].find(condition, self.projection) \
.sort("_id", pymongo.DESCENDING).limit(self.inner_size).skip((self.inner_page - 1) * self.inner_size)
children_data = list(children_data)
self.get_user_group(children_data)
if children_data:
item["children"] = []
item["children"].extend(children_data)
def get_user_group(self, data):
for comment in data:
username = comment["username"]
user = self.db["users"].find_one({"username": username})
group = user.get("group", ["user"])
comment["group"] = group
def get_comment(self, resource_id: int, page: int, size: int, **kwargs) -> dict:
self.inner_page = kwargs.get("inner_page", 1)
self.inner_size = kwargs.get("inner_size", 5)
@@ -143,6 +151,7 @@ class CommentMongoResource(CommentResource, Mongo):
data = list(data)
self.find_children(data)
self.convert_objectid(data)
self.get_user_group(data)
return {
"data": data,
"count": count,
@@ -294,25 +303,15 @@ class NameMongoResource(NameResource, Mongo):
class ResourceMongoResource(ResourceResource, Mongo):
redis = Redis().r
def zhuixinfan_search(self, kw):
# export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
result = ZhuixinfanOnline().search_preview(kw)
result.pop("source")
json_result = {} # name as key, url as value
def fansub_search(self, class_name: str, kw: str):
class_ = globals().get(class_name)
result = class_().search_preview(kw)
result.pop("class")
json_result = {} # name as key, url_hash as value
if result:
# this means we have search result, get it from redis cache with real name
for key, name in result.items():
json_result[name] = self.redis.hget(key, "url")
return json_result
def zimuxia_search(self, kw):
result = ZimuxiaOnline().search_preview(kw)
result.pop("source")
json_result = {} # name as key, url as value
if result:
# this means we have search result, get it from redis cache with real name
for key, name in result.items():
json_result[name] = self.redis.hget(key, "url")
for values in result.values():
json_result = {"name": values["name"], "url": values["url"]}
return json_result
def get_resource_data(self, resource_id: int, username: str) -> dict:
@@ -348,7 +347,10 @@ class ResourceMongoResource(ResourceResource, Mongo):
returned = dict(data=data)
returned["extra"] = []
else:
extra = self.zhuixinfan_search(keyword) or self.zimuxia_search(keyword)
extra = self.fansub_search(ZimuxiaOnline.__name__, keyword) or \
self.fansub_search(NewzmzOnline.__name__, keyword) or \
self.fansub_search(ZhuixinfanOnline.__name__, keyword)
returned["data"] = []
returned["extra"] = extra
@@ -444,5 +446,5 @@ class UserMongoResource(UserResource, Mongo):
def update_user_last(self, username: str, now_ip: str) -> None:
self.db["users"].update_one({"username": username},
{"$set": {"last_date": (ts_date()), "last_ip": now_ip}}
{"$set": {"lastDate": (ts_date()), "lastIP": now_ip}}
)