From e094644e8ba07f1e6120aca34d6c9c59db4e8c56 Mon Sep 17 00:00:00 2001 From: "www.xueximeng.com" Date: Tue, 28 Oct 2025 22:44:18 +0800 Subject: [PATCH] update --- plugin/gying/gying.go | 3176 ++++++++++++++--------------------------- 1 file changed, 1093 insertions(+), 2083 deletions(-) diff --git a/plugin/gying/gying.go b/plugin/gying/gying.go index ee308ff..1ef1139 100644 --- a/plugin/gying/gying.go +++ b/plugin/gying/gying.go @@ -1,2160 +1,1170 @@ -package gying +package plugin import ( - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "crypto/sha256" - "encoding/base64" - "encoding/hex" "fmt" - "io" - "io/ioutil" "net/http" - "net/url" - "os" - "path/filepath" - "reflect" - "regexp" - "sort" - "strconv" "strings" "sync" + "sync/atomic" "time" - "unsafe" "github.com/gin-gonic/gin" + "pansou/config" "pansou/model" - "pansou/plugin" - "pansou/util/json" - - cloudscraper "github.com/Advik-B/cloudscraper/lib" ) -// 插件配置参数 -const ( - MaxConcurrentUsers = 10 // 最多使用的用户数 - MaxConcurrentDetails = 50 // 最大并发详情请求数 - DebugLog = true // 调试日志开关（排查问题时改为true） +// ============================================================ +// 第一部分：接口定义和类型 +// ============================================================ + +// AsyncSearchPlugin 异步搜索插件接口 +type AsyncSearchPlugin interface { + // Name 返回插件名称 + Name() string + + // Priority 返回插件优先级 + Priority() int + + // AsyncSearch 异步搜索方法 + AsyncSearch(keyword string, searchFunc func(*http.Client, string, map[string]interface{}) ([]model.SearchResult, error), mainCacheKey string, ext map[string]interface{}) ([]model.SearchResult, error) + + // SetMainCacheKey 设置主缓存键 + SetMainCacheKey(key string) + + // SetCurrentKeyword 设置当前搜索关键词（用于日志显示） + SetCurrentKeyword(keyword string) + + // Search 兼容性方法（内部调用AsyncSearch） + Search(keyword string, ext map[string]interface{}) ([]model.SearchResult, error) + + // SkipServiceFilter 返回是否跳过Service层的关键词过滤 + // 对于磁力搜索等需要宽泛结果的插件，应返回true + SkipServiceFilter() bool +} + +// PluginWithWebHandler 支持Web路由的插件接口 +// 插件可以选择实现此接口来注册自定义的HTTP路由 +type PluginWithWebHandler interface { + AsyncSearchPlugin // 继承搜索插件接口 + + // RegisterWebRoutes 注册Web路由 + // router: gin的路由组，插件可以在此注册自己的路由 + RegisterWebRoutes(router *gin.RouterGroup) +} + +// ============================================================ +// 第二部分：全局变量和注册表 +// ============================================================ + +// 全局异步插件注册表 +var ( + globalRegistry = make(map[string]AsyncSearchPlugin) + globalRegistryLock sync.RWMutex ) -// 默认账户配置（可通过Web界面添加更多账户） -// 用户数据会保存到文件，重启后自动恢复 -var DefaultAccounts = []struct { - Username string - Password string -}{ - // 请使用 Web 接口添加用户： - // POST /gying/add_user?username=xxx&password=xxx -} - -// 存储目录 -var StorageDir string - -// 初始化存储目录 -func init() { - cachePath := os.Getenv("CACHE_PATH") - if cachePath == "" { - cachePath = "./cache" - } +// 工作池和统计相关变量 +var ( + // API响应缓存，键为关键词，值为缓存的响应（仅内存，不持久化） + apiResponseCache = sync.Map{} - StorageDir = filepath.Join(cachePath, "gying_users") + // 工作池相关变量 + backgroundWorkerPool chan struct{} + backgroundTasksCount int32 = 0 - if err := os.MkdirAll(StorageDir, 0755); err != nil { - fmt.Printf("⚠️ 警告: 无法创建Gying存储目录 %s: %v\n", StorageDir, err) - } else { - fmt.Printf("✓ Gying存储目录: %s\n", StorageDir) - } -} - -// HTML模板 -const HTMLTemplate = ` - - - - - PanSou Gying搜索配置 - - - -
-
-

🔍 PanSou Gying搜索

-

配置你的专属搜索服务

-

- 🔗 当前地址: HASH_PLACEHOLDER -

-
- -
-
🔐 登录状态
- - - - -
- -
-
🔍 测试搜索(限制返回10条数据)
- -
- - -
- - -
- -
-
📖 API调用说明
- -

你可以通过API程序化管理：

- -
- 登录 -
curl -X POST https://your-domain.com/gying/HASH_PLACEHOLDER \ - -H "Content-Type: application/json" \ - -d '{"action": "login", "username": "user", "password": "pass"}'
-
-
-
- - - -` - -// GyingPlugin 插件结构 -type GyingPlugin struct { - *plugin.BaseAsyncPlugin - users sync.Map // 内存缓存：hash -> *User - scrapers sync.Map // cloudscraper实例缓存：hash -> *cloudscraper.Scraper - mu sync.RWMutex -} - -// User 用户数据结构 -type User struct { - Hash string `json:"hash"` - Username string `json:"username"` // 原始用户名（存储） - UsernameMasked string `json:"username_masked"` // 脱敏用户名（显示） - EncryptedPassword string `json:"encrypted_password"` // 加密后的密码（用于重启恢复） - Cookie string `json:"cookie"` // 登录Cookie字符串（仅供参考） - Status string `json:"status"` // pending/active/expired - CreatedAt time.Time `json:"created_at"` - LoginAt time.Time `json:"login_at"` - ExpireAt time.Time `json:"expire_at"` - LastAccessAt time.Time `json:"last_access_at"` -} - -// SearchData 搜索页面JSON数据结构 -type SearchData struct { - Q string `json:"q"` // 搜索关键词 - WD []string `json:"wd"` // 分词 - N string `json:"n"` // 结果数量 - L struct { - Title []string `json:"title"` // 标题数组 - Year []int `json:"year"` // 年份数组 - D []string `json:"d"` // 类型数组（mv/ac/tv） - I []string `json:"i"` // 资源ID数组 - Info []string `json:"info"` // 信息数组 - Daoyan []string `json:"daoyan"` // 导演数组 - Zhuyan []string `json:"zhuyan"` // 主演数组 - } `json:"l"` -} - -// DetailData 详情接口JSON数据结构 -type DetailData struct { - Code int `json:"code"` - WP bool `json:"wp"` - Panlist struct { - ID []string `json:"id"` - Name []string `json:"name"` - P []string `json:"p"` // 提取码数组 - URL []string `json:"url"` // 链接数组 - Type []int `json:"type"` // 类型标识 - User []string `json:"user"` // 分享用户 - Time []string `json:"time"` // 分享时间 - TName []string `json:"tname"` // 网盘类型名称 - } `json:"panlist"` -} - -func init() { - p := &GyingPlugin{ - BaseAsyncPlugin: plugin.NewBaseAsyncPlugin("gying", 3), - } - - // 初始化存储目录 - if err := os.MkdirAll(StorageDir, 0755); err != nil { - fmt.Printf("[Gying] 创建存储目录失败: %v\n", err) - return - } - - // 加载所有用户到内存 - p.loadAllUsers() - - // 异步初始化默认账户（不阻塞启动） - go func() { - // 延迟1秒，等待主程序完全启动 - time.Sleep(1 * time.Second) - p.initDefaultAccounts() - }() - - // 启动定期清理任务 - go p.startCleanupTask() - - plugin.RegisterGlobalPlugin(p) -} - -// ============ 插件接口实现 ============ - -// RegisterWebRoutes 注册Web路由 -func (p *GyingPlugin) RegisterWebRoutes(router *gin.RouterGroup) { - gying := router.Group("/gying") - gying.GET("/:param", p.handleManagePage) - gying.POST("/:param", p.handleManagePagePOST) + // 统计数据 (仅用于内部监控) + cacheHits int64 = 0 + cacheMisses int64 = 0 + asyncCompletions int64 = 0 - fmt.Printf("[Gying] Web路由已注册: /gying/:param\n") + // 初始化标志 + initialized bool = false + initLock sync.Mutex + + // 默认配置值 + defaultAsyncResponseTimeout = 4 * time.Second + defaultPluginTimeout = 30 * time.Second + defaultCacheTTL = 1 * time.Hour // 恢复但仅用于内存缓存 + defaultMaxBackgroundWorkers = 20 + defaultMaxBackgroundTasks = 100 + + // 缓存访问频率记录 + cacheAccessCount = sync.Map{} + + // 缓存清理相关变量 + lastCleanupTime = time.Now() + cleanupMutex sync.Mutex +) + +// 全局序列化器引用（由主程序设置） +var globalCacheSerializer interface { + Serialize(interface{}) ([]byte, error) + Deserialize([]byte, interface{}) error } -// Search 执行搜索并返回结果 -func (p *GyingPlugin) Search(keyword string, ext map[string]interface{}) ([]model.SearchResult, error) { - result, err := p.SearchWithResult(keyword, ext) - if err != nil { - return nil, err - } - return result.Results, nil +// 缓存响应结构（仅内存，不持久化到磁盘） +type cachedResponse struct { + Results []model.SearchResult `json:"results"` + Timestamp time.Time `json:"timestamp"` + Complete bool `json:"complete"` + LastAccess time.Time `json:"last_access"` + AccessCount int `json:"access_count"` } -// SearchWithResult 执行搜索并返回包含IsFinal标记的结果 -func (p *GyingPlugin) SearchWithResult(keyword string, ext map[string]interface{}) (model.PluginSearchResult, error) { - if DebugLog { - fmt.Printf("[Gying] ========== 开始搜索: %s ==========\n", keyword) - } +// ============================================================ +// 第三部分：插件注册和管理 +// ============================================================ - // 1. 获取所有有效用户 - users := p.getActiveUsers() - if DebugLog { - fmt.Printf("[Gying] 找到 %d 个有效用户\n", len(users)) - } - - if len(users) == 0 { - if DebugLog { - fmt.Printf("[Gying] 没有有效用户，返回空结果\n") - } - return model.PluginSearchResult{Results: []model.SearchResult{}, IsFinal: true}, nil - } - - // 2. 限制用户数量 - if len(users) > MaxConcurrentUsers { - sort.Slice(users, func(i, j int) bool { - return users[i].LastAccessAt.After(users[j].LastAccessAt) - }) - users = users[:MaxConcurrentUsers] - } - - // 3. 并发执行搜索 - results := p.executeSearchTasks(users, keyword) - if DebugLog { - fmt.Printf("[Gying] 搜索完成，获得 %d 条结果\n", len(results)) - } - - return model.PluginSearchResult{ - Results: results, - IsFinal: true, - }, nil -} - -// ============ 用户管理 ============ - -// loadAllUsers 加载所有用户到内存（包括用户名、加密密码等） -// 注意：只加载用户数据，scraper实例将在initDefaultAccounts中使用密码重新登录获取 -func (p *GyingPlugin) loadAllUsers() { - files, err := ioutil.ReadDir(StorageDir) - if err != nil { - return - } - - count := 0 - for _, file := range files { - if file.IsDir() || filepath.Ext(file.Name()) != ".json" { - continue - } - - filePath := filepath.Join(StorageDir, file.Name()) - data, err := ioutil.ReadFile(filePath) - if err != nil { - continue - } - - var user User - if err := json.Unmarshal(data, &user); err != nil { - continue - } - - // 只存储用户数据（包括用户名和加密密码） - // scraper实例将在initDefaultAccounts中通过重新登录获取 - p.users.Store(user.Hash, &user) - count++ - - if DebugLog { - hasPassword := "无" - if user.EncryptedPassword != "" { - hasPassword = "有" - } - fmt.Printf("[Gying] 已加载用户 %s (密码:%s, 将在初始化时登录)\n", user.UsernameMasked, hasPassword) - } - } - - fmt.Printf("[Gying] 已加载 %d 个用户到内存\n", count) -} - -// initDefaultAccounts 初始化所有账户（异步执行，不阻塞启动） -// 包括：1. DefaultAccounts（代码配置） 2. 从文件加载的用户（使用加密密码重新登录） -func (p *GyingPlugin) initDefaultAccounts() { - fmt.Printf("[Gying] ========== 异步初始化所有账户 ==========\n") - - // 步骤1：处理DefaultAccounts（代码中配置的默认账户） - for i, account := range DefaultAccounts { - if DebugLog { - fmt.Printf("[Gying] [默认账户 %d/%d] 处理: %s\n", i+1, len(DefaultAccounts), account.Username) - } - - p.initOrRestoreUser(account.Username, account.Password, "default") - } - - // 步骤2：遍历所有已加载的用户，恢复没有scraper的用户 - var usersToRestore []*User - p.users.Range(func(key, value interface{}) bool { - user := value.(*User) - // 检查scraper是否存在 - _, scraperExists := p.scrapers.Load(user.Hash) - if !scraperExists && user.EncryptedPassword != "" { - usersToRestore = append(usersToRestore, user) - } - return true - }) - - if len(usersToRestore) > 0 { - fmt.Printf("[Gying] 发现 %d 个需要恢复的用户（使用加密密码重新登录）\n", len(usersToRestore)) - for i, user := range usersToRestore { - if DebugLog { - fmt.Printf("[Gying] [恢复用户 %d/%d] 处理: %s\n", i+1, len(usersToRestore), user.UsernameMasked) - } - - // 解密密码 - password, err := p.decryptPassword(user.EncryptedPassword) - if err != nil { - fmt.Printf("[Gying] ❌ 用户 %s 解密密码失败: %v\n", user.UsernameMasked, err) - continue - } - - p.initOrRestoreUser(user.Username, password, "restore") - } - } - - fmt.Printf("[Gying] ========== 所有账户初始化完成 ==========\n") -} - -// initOrRestoreUser 初始化或恢复单个用户（登录并保存） -func (p *GyingPlugin) initOrRestoreUser(username, password, source string) { - hash := p.generateHash(username) - - // 检查scraper是否已存在 - _, scraperExists := p.scrapers.Load(hash) - if scraperExists { - if DebugLog { - fmt.Printf("[Gying] 用户 %s scraper已存在，跳过\n", p.maskUsername(username)) - } +// RegisterGlobalPlugin 注册异步插件到全局注册表 +func RegisterGlobalPlugin(plugin AsyncSearchPlugin) { + if plugin == nil { return } - // 登录 - if DebugLog { - fmt.Printf("[Gying] 开始登录账户: %s\n", username) - } - scraper, cookie, err := p.doLogin(username, password) - if err != nil { - fmt.Printf("[Gying] ❌ 账户 %s 登录失败: %v\n", username, err) - return - } - - if DebugLog { - fmt.Printf("[Gying] 登录成功，已获取cloudscraper实例\n") - } - - // 加密密码 - encryptedPassword, err := p.encryptPassword(password) - if err != nil { - fmt.Printf("[Gying] ❌ 加密密码失败: %v\n", err) + globalRegistryLock.Lock() + defer globalRegistryLock.Unlock() + + name := plugin.Name() + if name == "" { return } - // 保存用户 - user := &User{ - Hash: hash, - Username: username, - UsernameMasked: p.maskUsername(username), - EncryptedPassword: encryptedPassword, - Cookie: cookie, - Status: "active", - CreatedAt: time.Now(), - LoginAt: time.Now(), - ExpireAt: time.Now().AddDate(0, 4, 0), // 121天有效期 - LastAccessAt: time.Now(), + globalRegistry[name] = plugin +} + +// GetRegisteredPlugins 获取所有已注册的异步插件 +func GetRegisteredPlugins() []AsyncSearchPlugin { + globalRegistryLock.RLock() + defer globalRegistryLock.RUnlock() + + plugins := make([]AsyncSearchPlugin, 0, len(globalRegistry)) + for _, plugin := range globalRegistry { + plugins = append(plugins, plugin) } - // 保存scraper实例到内存 - p.scrapers.Store(hash, scraper) + return plugins +} + +// GetPluginByName 根据名称获取已注册的插件 +func GetPluginByName(name string) (AsyncSearchPlugin, bool) { + globalRegistryLock.RLock() + defer globalRegistryLock.RUnlock() - if err := p.saveUser(user); err != nil { - fmt.Printf("[Gying] ❌ 保存账户失败: %v\n", err) - return + plugin, exists := globalRegistry[name] + return plugin, exists +} + +// PluginManager 异步插件管理器 +type PluginManager struct { + plugins []AsyncSearchPlugin +} + +// NewPluginManager 创建新的异步插件管理器 +func NewPluginManager() *PluginManager { + return &PluginManager{ + plugins: make([]AsyncSearchPlugin, 0), } - - fmt.Printf("[Gying] ✅ 账户 %s 初始化成功 (来源:%s)\n", user.UsernameMasked, source) } -// getUserByHash 获取用户 -func (p *GyingPlugin) getUserByHash(hash string) (*User, bool) { - value, ok := p.users.Load(hash) - if !ok { - return nil, false +// RegisterPlugin 注册异步插件 +func (pm *PluginManager) RegisterPlugin(plugin AsyncSearchPlugin) { + pm.plugins = append(pm.plugins, plugin) +} + +// RegisterAllGlobalPlugins 注册所有全局异步插件 +func (pm *PluginManager) RegisterAllGlobalPlugins() { + allPlugins := GetRegisteredPlugins() + for _, plugin := range allPlugins { + pm.RegisterPlugin(plugin) } - return value.(*User), true } -// saveUser 保存用户 -func (p *GyingPlugin) saveUser(user *User) error { - p.users.Store(user.Hash, user) - return p.persistUser(user) -} - -// persistUser 持久化用户到文件 -func (p *GyingPlugin) persistUser(user *User) error { - filePath := filepath.Join(StorageDir, user.Hash+".json") - data, err := json.MarshalIndent(user, "", " ") - if err != nil { - return err - } - return ioutil.WriteFile(filePath, data, 0644) -} - -// deleteUser 删除用户 -func (p *GyingPlugin) deleteUser(hash string) error { - p.users.Delete(hash) - filePath := filepath.Join(StorageDir, hash+".json") - return os.Remove(filePath) -} - -// getActiveUsers 获取有效用户 -func (p *GyingPlugin) getActiveUsers() []*User { - var users []*User +// RegisterGlobalPluginsWithFilter 根据过滤器注册全局异步插件 +// enabledPlugins: nil表示未设置（不启用任何插件），空切片表示设置为空（不启用任何插件），具体列表表示启用指定插件 +func (pm *PluginManager) RegisterGlobalPluginsWithFilter(enabledPlugins []string) { + allPlugins := GetRegisteredPlugins() - p.users.Range(func(key, value interface{}) bool { - user := value.(*User) - if user.Status == "active" && user.Cookie != "" { - users = append(users, user) - } - return true - }) - - return users -} - -// ============ HTTP路由处理 ============ - -// handleManagePage GET路由处理 -func (p *GyingPlugin) handleManagePage(c *gin.Context) { - param := c.Param("param") - - // 判断是用户名还是hash - if len(param) == 64 && p.isHexString(param) { - html := strings.ReplaceAll(HTMLTemplate, "HASH_PLACEHOLDER", param) - c.Data(200, "text/html; charset=utf-8", []byte(html)) - } else { - hash := p.generateHash(param) - c.Redirect(302, "/gying/"+hash) - } -} - -// handleManagePagePOST POST路由处理 -func (p *GyingPlugin) handleManagePagePOST(c *gin.Context) { - hash := c.Param("param") - - var reqData map[string]interface{} - if err := c.ShouldBindJSON(&reqData); err != nil { - respondError(c, "无效的请求格式: "+err.Error()) - return - } - - action, ok := reqData["action"].(string) - if !ok || action == "" { - respondError(c, "缺少action字段") - return - } - - switch action { - case "get_status": - p.handleGetStatus(c, hash) - case "login": - p.handleLogin(c, hash, reqData) - case "logout": - p.handleLogout(c, hash) - case "test_search": - p.handleTestSearch(c, hash, reqData) - default: - respondError(c, "未知的操作类型: "+action) - } -} - -// handleGetStatus 获取状态 -func (p *GyingPlugin) handleGetStatus(c *gin.Context, hash string) { - user, exists := p.getUserByHash(hash) - - if !exists { - user = &User{ - Hash: hash, - Status: "pending", - CreatedAt: time.Now(), - LastAccessAt: time.Now(), - } - p.saveUser(user) - } else { - user.LastAccessAt = time.Now() - p.saveUser(user) - } - - loggedIn := false - if user.Status == "active" && user.Cookie != "" { - loggedIn = true - } - - expiresInDays := 0 - if !user.ExpireAt.IsZero() { - expiresInDays = int(time.Until(user.ExpireAt).Hours() / 24) - if expiresInDays < 0 { - expiresInDays = 0 - } - } - - respondSuccess(c, "获取成功", gin.H{ - "hash": hash, - "logged_in": loggedIn, - "status": user.Status, - "username_masked": user.UsernameMasked, - "login_time": user.LoginAt.Format("2006-01-02 15:04:05"), - "expire_time": user.ExpireAt.Format("2006-01-02 15:04:05"), - "expires_in_days": expiresInDays, - }) -} - -// handleLogin 处理登录 -func (p *GyingPlugin) handleLogin(c *gin.Context, hash string, reqData map[string]interface{}) { - username, _ := reqData["username"].(string) - password, _ := reqData["password"].(string) - - if username == "" || password == "" { - respondError(c, "缺少用户名或密码") - return - } - - // 执行登录 - scraper, cookie, err := p.doLogin(username, password) - if err != nil { - respondError(c, "登录失败: "+err.Error()) - return - } - - // 保存scraper实例到内存 - p.scrapers.Store(hash, scraper) - - // 加密密码 - encryptedPassword, err := p.encryptPassword(password) - if err != nil { - respondError(c, "加密密码失败: "+err.Error()) + // nil 表示未设置环境变量，不启用任何插件 + if enabledPlugins == nil { return } - // 保存用户 - user := &User{ - Hash: hash, - Username: username, - UsernameMasked: p.maskUsername(username), - EncryptedPassword: encryptedPassword, - Cookie: cookie, - Status: "active", - LoginAt: time.Now(), - ExpireAt: time.Now().AddDate(0, 4, 0), // 121天 - LastAccessAt: time.Now(), - } - - if _, exists := p.getUserByHash(hash); !exists { - user.CreatedAt = time.Now() - } - - if err := p.saveUser(user); err != nil { - respondError(c, "保存失败: "+err.Error()) - return - } - - respondSuccess(c, "登录成功", gin.H{ - "status": "active", - "username_masked": user.UsernameMasked, - }) -} - -// handleLogout 退出登录 -func (p *GyingPlugin) handleLogout(c *gin.Context, hash string) { - user, exists := p.getUserByHash(hash) - if !exists { - respondError(c, "用户不存在") - return - } - - user.Cookie = "" - user.Status = "pending" - - if err := p.saveUser(user); err != nil { - respondError(c, "退出失败") - return - } - - respondSuccess(c, "已退出登录", gin.H{ - "status": "pending", - }) -} - -// handleTestSearch 测试搜索 -func (p *GyingPlugin) handleTestSearch(c *gin.Context, hash string, reqData map[string]interface{}) { - keyword, ok := reqData["keyword"].(string) - if !ok || keyword == "" { - respondError(c, "缺少keyword字段") - return - } - - user, exists := p.getUserByHash(hash) - if !exists || user.Cookie == "" { - respondError(c, "请先登录") - return - } - - // 获取scraper实例 - scraperVal, exists := p.scrapers.Load(hash) - if !exists { - respondError(c, "用户scraper实例不存在，请重新登录") + // 空切片表示设置为空字符串，也不启用任何插件 + if len(enabledPlugins) == 0 { return } - scraper, ok := scraperVal.(*cloudscraper.Scraper) - if !ok || scraper == nil { - respondError(c, "scraper实例无效，请重新登录") - return + // 创建启用插件名称的映射表，用于快速查找 + enabledMap := make(map[string]bool) + for _, name := range enabledPlugins { + enabledMap[name] = true } - // 执行搜索（带403自动重新登录） - results, err := p.searchWithScraperWithRetry(keyword, scraper, user) - if err != nil { - respondError(c, "搜索失败: "+err.Error()) - return - } - - // 限制返回数量 - maxResults := 10 - if len(results) > maxResults { - results = results[:maxResults] - } - - // 转换为前端格式 - frontendResults := make([]gin.H, 0, len(results)) - for _, r := range results { - links := make([]gin.H, 0, len(r.Links)) - for _, link := range r.Links { - links = append(links, gin.H{ - "type": link.Type, - "url": link.URL, - "password": link.Password, - }) + // 只注册在启用列表中的插件 + for _, plugin := range allPlugins { + if enabledMap[plugin.Name()] { + pm.RegisterPlugin(plugin) } - - frontendResults = append(frontendResults, gin.H{ - "title": r.Title, - "links": links, - }) - } - - respondSuccess(c, fmt.Sprintf("找到 %d 条结果", len(frontendResults)), gin.H{ - "keyword": keyword, - "total_results": len(frontendResults), - "results": frontendResults, - }) -} - -// ============ 密码加密/解密 ============ - -// encryptPassword 使用AES加密密码 -func (p *GyingPlugin) encryptPassword(password string) (string, error) { - // 使用固定密钥（实际应用中可以使用配置或环境变量） - key := []byte("gying-secret-key-32bytes-long!!!") // 32字节密钥用于AES-256 - - block, err := aes.NewCipher(key) - if err != nil { - return "", err - } - - // 创建GCM模式 - gcm, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - // 生成随机nonce - nonce := make([]byte, gcm.NonceSize()) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return "", err - } - - // 加密 - ciphertext := gcm.Seal(nonce, nonce, []byte(password), nil) - - // 返回base64编码的密文 - return base64.StdEncoding.EncodeToString(ciphertext), nil -} - -// decryptPassword 解密密码 -func (p *GyingPlugin) decryptPassword(encrypted string) (string, error) { - // 使用与加密相同的密钥 - key := []byte("gying-secret-key-32bytes-long!!!") - - // base64解码 - ciphertext, err := base64.StdEncoding.DecodeString(encrypted) - if err != nil { - return "", err - } - - block, err := aes.NewCipher(key) - if err != nil { - return "", err - } - - gcm, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - nonceSize := gcm.NonceSize() - if len(ciphertext) < nonceSize { - return "", fmt.Errorf("ciphertext too short") - } - - nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] - plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) - if err != nil { - return "", err - } - - return string(plaintext), nil -} - -// ============ Cookie管理 ============ - -// createScraperWithCookies 创建一个带有指定cookies的cloudscraper实例 -// 使用反射访问内部的http.Client并设置cookies到cookiejar -// 关键：禁用session refresh以防止cookies被清空 -func (p *GyingPlugin) createScraperWithCookies(cookieStr string) (*cloudscraper.Scraper, error) { - // 创建cloudscraper实例，配置以保护cookies不被刷新 - scraper, err := cloudscraper.New( - cloudscraper.WithSessionConfig( - false, // refreshOn403 = false，禁用403时自动刷新 - 365*24*time.Hour, // interval = 1年，基本不刷新 - 0, // maxRetries = 0 - ), - ) - if err != nil { - return nil, fmt.Errorf("创建cloudscraper失败: %w", err) - } - - // 如果有保存的cookies，使用反射设置到scraper的内部http.Client - if cookieStr != "" { - cookies := parseCookieString(cookieStr) - - if DebugLog { - fmt.Printf("[Gying] 正在恢复 %d 个cookie到scraper实例\n", len(cookies)) - } - - // 使用反射访问scraper的unexported client字段 - scraperValue := reflect.ValueOf(scraper).Elem() - clientField := scraperValue.FieldByName("client") - - if clientField.IsValid() && !clientField.IsNil() { - // 使用反射访问client (需要使用Elem()因为是指针) - clientValue := reflect.NewAt(clientField.Type(), unsafe.Pointer(clientField.UnsafeAddr())).Elem() - client, ok := clientValue.Interface().(*http.Client) - - if ok && client != nil && client.Jar != nil { - // 将cookies设置到cookiejar - // 注意：必须使用正确的URL和cookie属性 - gyingURL, _ := url.Parse("https://www.gying.net") - var httpCookies []*http.Cookie - - for name, value := range cookies { - cookie := &http.Cookie{ - Name: name, - Value: value, - // 不设置Domain和Path，让cookiejar根据URL自动推导 - // cookiejar.SetCookies会根据提供的URL自动设置正确的Domain和Path - } - httpCookies = append(httpCookies, cookie) - - if DebugLog { - fmt.Printf("[Gying] 准备恢复Cookie: %s=%s\n", - cookie.Name, cookie.Value[:min(10, len(cookie.Value))]) - } - } - - client.Jar.SetCookies(gyingURL, httpCookies) - - // 验证cookies是否被正确设置 - if DebugLog { - storedCookies := client.Jar.Cookies(gyingURL) - fmt.Printf("[Gying] ✅ 成功恢复 %d 个cookie到scraper的cookiejar\n", len(cookies)) - fmt.Printf("[Gying] 验证: cookiejar中现有 %d 个cookie\n", len(storedCookies)) - - // 详细打印每个cookie以便调试 - for i, c := range storedCookies { - fmt.Printf("[Gying] 设置后Cookie[%d]: %s=%s (Domain:%s, Path:%s)\n", - i, c.Name, c.Value[:min(10, len(c.Value))], c.Domain, c.Path) - } - } - } else { - if DebugLog { - fmt.Printf("[Gying] ⚠️ 无法获取http.Client或其Jar\n") - } - } - } else { - if DebugLog { - fmt.Printf("[Gying] ⚠️ 无法通过反射访问client字段\n") - } - } - } - - return scraper, nil -} - -// parseCookieString 解析cookie字符串为map -func parseCookieString(cookieStr string) map[string]string { - cookies := make(map[string]string) - parts := strings.Split(cookieStr, ";") - - for _, part := range parts { - part = strings.TrimSpace(part) - if idx := strings.Index(part, "="); idx > 0 { - name := part[:idx] - value := part[idx+1:] - cookies[name] = value - } - } - - return cookies -} - -// ============ 登录逻辑 ============ - -// doLogin 执行登录，返回scraper实例和cookie字符串 -// -// 登录流程（3步）： -// 1. GET登录页 (https://www.gying.net/user/login/) → 获取PHPSESSID -// 2. POST登录 (https://www.gying.net/user/login) → 获取BT_auth、BT_cookietime等认证cookies -// 3. GET详情页 (https://www.gying.net/mv/wkMn) → 触发防爬cookies (vrg_sc、vrg_go等) -// -// 返回: (*cloudscraper.Scraper, cookie字符串, error) -func (p *GyingPlugin) doLogin(username, password string) (*cloudscraper.Scraper, string, error) { - if DebugLog { - fmt.Printf("[Gying] ========== 开始登录 ==========\n") - fmt.Printf("[Gying] 用户名: %s\n", username) - fmt.Printf("[Gying] 密码长度: %d\n", len(password)) - } - - // 创建cloudscraper实例（每个用户独立的实例） - // 关键配置：禁用403自动刷新,防止cookie被清空 - scraper, err := cloudscraper.New( - cloudscraper.WithSessionConfig( - false, // refreshOn403 = false，禁用403时自动刷新（重要！） - 365*24*time.Hour, // interval = 1年，基本不刷新 - 0, // maxRetries = 0 - ), - ) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 创建cloudscraper失败: %v\n", err) - } - return nil, "", fmt.Errorf("创建cloudscraper失败: %w", err) - } - - if DebugLog { - fmt.Printf("[Gying] cloudscraper创建成功（已禁用403自动刷新）\n") - } - - // 创建cookieMap用于收集所有cookies - cookieMap := make(map[string]string) - - // ========== 步骤1: GET登录页 (获取初始PHPSESSID) ========== - loginPageURL := "https://www.gying.net/user/login/" - if DebugLog { - fmt.Printf("[Gying] 步骤1: 访问登录页面: %s\n", loginPageURL) - } - - getResp, err := scraper.Get(loginPageURL) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 访问登录页面失败: %v\n", err) - } - return nil, "", fmt.Errorf("访问登录页面失败: %w", err) - } - defer getResp.Body.Close() - ioutil.ReadAll(getResp.Body) // 读取body - - if DebugLog { - fmt.Printf("[Gying] 登录页面状态码: %d\n", getResp.StatusCode) - } - - // 从登录页响应中收集cookies - for _, setCookie := range getResp.Header["Set-Cookie"] { - parts := strings.Split(setCookie, ";") - if len(parts) > 0 { - cookiePart := strings.TrimSpace(parts[0]) - if idx := strings.Index(cookiePart, "="); idx > 0 { - name := cookiePart[:idx] - value := cookiePart[idx+1:] - cookieMap[name] = value - if DebugLog { - displayValue := value - if len(displayValue) > 20 { - displayValue = displayValue[:20] + "..." - } - fmt.Printf("[Gying] 登录页Cookie: %s=%s\n", name, displayValue) - } - } - } - } - - // ========== 步骤2: POST登录 (获取认证cookies) ========== - loginURL := "https://www.gying.net/user/login" - postData := fmt.Sprintf("code=&siteid=1&dosubmit=1&cookietime=10506240&username=%s&password=%s", - url.QueryEscape(username), - url.QueryEscape(password)) - - if DebugLog { - fmt.Printf("[Gying] 步骤2: POST登录\n") - fmt.Printf("[Gying] 登录URL: %s\n", loginURL) - fmt.Printf("[Gying] POST数据: %s\n", postData) - } - - resp, err := scraper.Post(loginURL, "application/x-www-form-urlencoded", strings.NewReader(postData)) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 登录POST请求失败: %v\n", err) - } - return nil, "", fmt.Errorf("登录POST请求失败: %w", err) - } - defer resp.Body.Close() - - if DebugLog { - fmt.Printf("[Gying] 响应状态码: %d\n", resp.StatusCode) - } - - // 从POST登录响应中收集cookies - for _, setCookie := range resp.Header["Set-Cookie"] { - parts := strings.Split(setCookie, ";") - if len(parts) > 0 { - cookiePart := strings.TrimSpace(parts[0]) - if idx := strings.Index(cookiePart, "="); idx > 0 { - name := cookiePart[:idx] - value := cookiePart[idx+1:] - cookieMap[name] = value - if DebugLog { - displayValue := value - if len(displayValue) > 20 { - displayValue = displayValue[:20] + "..." - } - fmt.Printf("[Gying] POST登录Cookie: %s=%s\n", name, displayValue) - } - } - } - } - - // 读取响应 - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 读取响应失败: %v\n", err) - } - return nil, "", fmt.Errorf("读取响应失败: %w", err) - } - - if DebugLog { - fmt.Printf("[Gying] 响应内容: %s\n", string(body)) - } - - var loginResp map[string]interface{} - if err := json.Unmarshal(body, &loginResp); err != nil { - if DebugLog { - fmt.Printf("[Gying] JSON解析失败: %v\n", err) - } - return nil, "", fmt.Errorf("JSON解析失败: %w, 响应内容: %s", err, string(body)) - } - - if DebugLog { - fmt.Printf("[Gying] 解析后的响应: %+v\n", loginResp) - fmt.Printf("[Gying] code字段类型: %T, 值: %v\n", loginResp["code"], loginResp["code"]) - } - - // 检查登录结果（兼容多种类型：int、float64、json.Number、string） - var codeValue int - codeInterface := loginResp["code"] - - switch v := codeInterface.(type) { - case int: - codeValue = v - case float64: - codeValue = int(v) - case int64: - codeValue = int(v) - default: - // 尝试转换为字符串再解析 - codeStr := fmt.Sprintf("%v", codeInterface) - parsed, err := strconv.Atoi(codeStr) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 无法解析code字段: %T, 值: %v, 错误: %v\n", codeInterface, codeInterface, err) - } - return nil, "", fmt.Errorf("无法解析code字段，类型: %T, 值: %v", codeInterface, codeInterface) - } - codeValue = parsed - } - - if DebugLog { - fmt.Printf("[Gying] 解析后的code值: %d\n", codeValue) - } - - if codeValue != 200 { - if DebugLog { - fmt.Printf("[Gying] 登录失败: code=%d (期望200)\n", codeValue) - } - return nil, "", fmt.Errorf("登录失败: code=%d, 响应=%s", codeValue, string(body)) - } - - // ========== 步骤3: GET详情页 (触发防爬cookies如vrg_sc、vrg_go等) ========== - if DebugLog { - fmt.Printf("[Gying] 步骤3: GET详情页收集完整Cookie\n") - } - - detailResp, err := scraper.Get("https://www.gying.net/mv/wkMn") - if err == nil { - defer detailResp.Body.Close() - ioutil.ReadAll(detailResp.Body) - - if DebugLog { - fmt.Printf("[Gying] 详情页状态码: %d\n", detailResp.StatusCode) - } - - // 从详情页响应中收集cookies - for _, setCookie := range detailResp.Header["Set-Cookie"] { - parts := strings.Split(setCookie, ";") - if len(parts) > 0 { - cookiePart := strings.TrimSpace(parts[0]) - if idx := strings.Index(cookiePart, "="); idx > 0 { - name := cookiePart[:idx] - value := cookiePart[idx+1:] - cookieMap[name] = value - if DebugLog { - displayValue := value - if len(displayValue) > 30 { - displayValue = displayValue[:30] + "..." - } - fmt.Printf("[Gying] 详情页Cookie: %s=%s\n", name, displayValue) - } - } - } - } - } - - // 构建cookie字符串 - var cookieParts []string - for name, value := range cookieMap { - cookieParts = append(cookieParts, fmt.Sprintf("%s=%s", name, value)) - } - cookieStr := strings.Join(cookieParts, "; ") - - if DebugLog { - fmt.Printf("[Gying] ✅ 登录成功！提取到 %d 个Cookie\n", len(cookieMap)) - fmt.Printf("[Gying] Cookie字符串长度: %d\n", len(cookieStr)) - for name, value := range cookieMap { - displayValue := value - if len(displayValue) > 30 { - displayValue = displayValue[:30] + "..." - } - fmt.Printf("[Gying] %s=%s (len:%d)\n", name, displayValue, len(value)) - } - fmt.Printf("[Gying] ========== 登录完成 ==========\n") - } - - // 返回scraper实例和实际的cookie字符串 - return scraper, cookieStr, nil -} - -// min 辅助函数 -func min(a, b int) int { - if a < b { - return a - } - return b -} - -// ============ 重新登录逻辑 ============ - -// reloginUser 重新登录指定用户 -func (p *GyingPlugin) reloginUser(user *User) error { - if DebugLog { - fmt.Printf("[Gying] 🔄 开始重新登录用户: %s\n", user.UsernameMasked) - } - - // 解密密码 - password, err := p.decryptPassword(user.EncryptedPassword) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] ❌ 解密密码失败: %v\n", err) - } - return fmt.Errorf("解密密码失败: %w", err) - } - - // 执行登录 - scraper, cookie, err := p.doLogin(user.Username, password) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] ❌ 重新登录失败: %v\n", err) - } - return fmt.Errorf("重新登录失败: %w", err) - } - - // 更新scraper实例 - p.scrapers.Store(user.Hash, scraper) - - // 更新用户信息 - user.Cookie = cookie - user.LoginAt = time.Now() - user.ExpireAt = time.Now().AddDate(0, 4, 0) - user.Status = "active" - - if err := p.saveUser(user); err != nil { - if DebugLog { - fmt.Printf("[Gying] ⚠️ 保存用户失败: %v\n", err) - } - } - - if DebugLog { - fmt.Printf("[Gying] ✅ 用户 %s 重新登录成功\n", user.UsernameMasked) - } - - return nil -} - -// ============ 搜索逻辑 ============ - -// executeSearchTasks 并发执行搜索任务 -func (p *GyingPlugin) executeSearchTasks(users []*User, keyword string) []model.SearchResult { - var allResults []model.SearchResult - var mu sync.Mutex - var wg sync.WaitGroup - - for _, user := range users { - wg.Add(1) - go func(u *User) { - defer wg.Done() - - // 获取用户的scraper实例 - scraperVal, exists := p.scrapers.Load(u.Hash) - var scraper *cloudscraper.Scraper - - if !exists { - if DebugLog { - fmt.Printf("[Gying] 用户 %s 没有scraper实例，尝试使用已保存的cookie创建\n", u.UsernameMasked) - } - - // 为用户创建新的cloudscraper实例（禁用403自动刷新） - newScraper, err := cloudscraper.New( - cloudscraper.WithSessionConfig( - false, // refreshOn403 = false - 365*24*time.Hour, // interval = 1年 - 0, // maxRetries = 0 - ), - ) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 为用户 %s 创建scraper失败: %v\n", u.UsernameMasked, err) - } - return - } - - // 存储新创建的scraper实例 - p.scrapers.Store(u.Hash, newScraper) - scraper = newScraper - - if DebugLog { - fmt.Printf("[Gying] 已为用户 %s 创建新的scraper实例（已禁用403刷新）\n", u.UsernameMasked) - } - } else { - var ok bool - scraper, ok = scraperVal.(*cloudscraper.Scraper) - if !ok || scraper == nil { - if DebugLog { - fmt.Printf("[Gying] 用户 %s scraper实例无效，跳过\n", u.UsernameMasked) - } - return - } - } - - results, err := p.searchWithScraperWithRetry(keyword, scraper, u) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 用户 %s 搜索失败（已重试）: %v\n", u.UsernameMasked, err) - } - return - } - - mu.Lock() - allResults = append(allResults, results...) - mu.Unlock() - }(user) - } - - wg.Wait() - - // 去重 - return p.deduplicateResults(allResults) -} - -// searchWithScraperWithRetry 使用scraper搜索（带403自动重新登录重试） -func (p *GyingPlugin) searchWithScraperWithRetry(keyword string, scraper *cloudscraper.Scraper, user *User) ([]model.SearchResult, error) { - results, err := p.searchWithScraper(keyword, scraper) - - // 检测是否为403错误 - if err != nil && strings.Contains(err.Error(), "403") { - if DebugLog { - fmt.Printf("[Gying] ⚠️ 检测到403错误，尝试重新登录用户 %s\n", user.UsernameMasked) - } - - // 尝试重新登录 - if reloginErr := p.reloginUser(user); reloginErr != nil { - if DebugLog { - fmt.Printf("[Gying] ❌ 重新登录失败: %v\n", reloginErr) - } - return nil, fmt.Errorf("403错误且重新登录失败: %w", reloginErr) - } - - // 获取新的scraper实例 - scraperVal, exists := p.scrapers.Load(user.Hash) - if !exists { - return nil, fmt.Errorf("重新登录后未找到scraper实例") - } - - newScraper, ok := scraperVal.(*cloudscraper.Scraper) - if !ok || newScraper == nil { - return nil, fmt.Errorf("重新登录后scraper实例无效") - } - - // 使用新scraper重试搜索 - if DebugLog { - fmt.Printf("[Gying] 🔄 使用新登录状态重试搜索\n") - } - results, err = p.searchWithScraper(keyword, newScraper) - if err != nil { - return nil, fmt.Errorf("重新登录后搜索仍然失败: %w", err) - } - } - - return results, err -} - -// searchWithScraper 使用scraper搜索 -func (p *GyingPlugin) searchWithScraper(keyword string, scraper *cloudscraper.Scraper) ([]model.SearchResult, error) { - if DebugLog { - fmt.Printf("[Gying] ---------- searchWithScraper 开始 ----------\n") - fmt.Printf("[Gying] 关键词: %s\n", keyword) - } - - // 1. 使用cloudscraper请求搜索页面 - searchURL := fmt.Sprintf("https://www.gying.net/s/1---1/%s", url.QueryEscape(keyword)) - - if DebugLog { - fmt.Printf("[Gying] 搜索URL: %s\n", searchURL) - fmt.Printf("[Gying] 使用cloudscraper发送请求\n") - } - - resp, err := scraper.Get(searchURL) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 搜索请求失败: %v\n", err) - } - return nil, err - } - defer resp.Body.Close() - - if DebugLog { - fmt.Printf("[Gying] 搜索响应状态码: %d\n", resp.StatusCode) - } - - // 读取响应body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 读取响应失败: %v\n", err) - } - return nil, err - } - - if DebugLog { - fmt.Printf("[Gying] 响应Body长度: %d 字节\n", len(body)) - if len(body) > 0 { - // 打印前500字符 - preview := string(body) - if len(preview) > 500 { - preview = preview[:500] + "..." - } - fmt.Printf("[Gying] 响应预览: %s\n", preview) - } - } - - // 检查403错误 - if resp.StatusCode == 403 { - if DebugLog { - fmt.Printf("[Gying] ❌ 收到403 Forbidden - Cookie可能已过期或被网站拒绝\n") - if len(body) > 0 { - preview := string(body) - if len(preview) > 300 { - preview = preview[:300] + "..." - } - fmt.Printf("[Gying] 403响应内容: %s\n", preview) - } - } - return nil, fmt.Errorf("HTTP 403 Forbidden - 可能需要重新登录") - } - - // 2. 提取 _obj.search JSON - re := regexp.MustCompile(`_obj\.search=(\{.*?\});`) - matches := re.FindSubmatch(body) - - if DebugLog { - fmt.Printf("[Gying] 正则匹配结果: 找到 %d 个匹配\n", len(matches)) - } - - if len(matches) < 2 { - if DebugLog { - fmt.Printf("[Gying] ❌ 未找到 _obj.search JSON数据\n") - // 尝试查找是否有其他模式 - if strings.Contains(string(body), "_obj.search") { - fmt.Printf("[Gying] 但是Body中包含 '_obj.search' 字符串\n") - } else { - fmt.Printf("[Gying] Body中不包含 '_obj.search' 字符串\n") - } - } - return nil, fmt.Errorf("未找到搜索结果数据") - } - - if DebugLog { - jsonStr := string(matches[1]) - if len(jsonStr) > 200 { - jsonStr = jsonStr[:200] + "..." - } - fmt.Printf("[Gying] 提取的JSON数据: %s\n", jsonStr) - } - - var searchData SearchData - if err := json.Unmarshal(matches[1], &searchData); err != nil { - if DebugLog { - fmt.Printf("[Gying] JSON解析失败: %v\n", err) - fmt.Printf("[Gying] 原始JSON: %s\n", string(matches[1])) - } - return nil, fmt.Errorf("解析搜索数据失败: %w", err) - } - - if DebugLog { - fmt.Printf("[Gying] 搜索数据解析成功:\n") - fmt.Printf("[Gying] - 关键词: %s\n", searchData.Q) - fmt.Printf("[Gying] - 结果数量字符串: %s\n", searchData.N) - fmt.Printf("[Gying] - 资源ID数组长度: %d\n", len(searchData.L.I)) - fmt.Printf("[Gying] - 标题数组长度: %d\n", len(searchData.L.Title)) - if len(searchData.L.I) > 0 { - fmt.Printf("[Gying] - 前3个资源ID: %v\n", searchData.L.I[:min(3, len(searchData.L.I))]) - fmt.Printf("[Gying] - 前3个标题: %v\n", searchData.L.Title[:min(3, len(searchData.L.Title))]) - } - } - - // 3. 并发请求详情接口 - results, err := p.fetchAllDetails(&searchData, scraper) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] fetchAllDetails 失败: %v\n", err) - fmt.Printf("[Gying] ---------- searchWithScraper 结束 ----------\n") - } - return nil, err - } - - if DebugLog { - fmt.Printf("[Gying] fetchAllDetails 返回 %d 条结果\n", len(results)) - fmt.Printf("[Gying] ---------- searchWithScraper 结束 ----------\n") - } - - return results, nil -} - -// fetchAllDetails 并发获取所有详情 -func (p *GyingPlugin) fetchAllDetails(searchData *SearchData, scraper *cloudscraper.Scraper) ([]model.SearchResult, error) { - if DebugLog { - fmt.Printf("[Gying] >>> fetchAllDetails 开始\n") - fmt.Printf("[Gying] 需要获取 %d 个详情\n", len(searchData.L.I)) - } - - var results []model.SearchResult - var mu sync.Mutex - var wg sync.WaitGroup - - semaphore := make(chan struct{}, MaxConcurrentDetails) - errChan := make(chan error, 1) // 用于接收403错误 - - successCount := 0 - failCount := 0 - has403 := false - - for i := 0; i < len(searchData.L.I); i++ { - wg.Add(1) - go func(index int) { - defer wg.Done() - - semaphore <- struct{}{} - defer func() { <-semaphore }() - - // 检查是否已经遇到403错误 - mu.Lock() - if has403 { - mu.Unlock() - return - } - mu.Unlock() - - if DebugLog { - fmt.Printf("[Gying] [%d/%d] 获取详情: ID=%s, Type=%s\n", - index+1, len(searchData.L.I), searchData.L.I[index], searchData.L.D[index]) - } - - detail, err := p.fetchDetail(searchData.L.I[index], searchData.L.D[index], scraper) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] [%d/%d] ❌ 获取详情失败: %v\n", index+1, len(searchData.L.I), err) - } - - // 检查是否是403错误 - if strings.Contains(err.Error(), "403") { - mu.Lock() - if !has403 { - has403 = true - select { - case errChan <- err: - default: - } - } - mu.Unlock() - } - - mu.Lock() - failCount++ - mu.Unlock() - return - } - - result := p.buildResult(detail, searchData, index) - if result.Title != "" && len(result.Links) > 0 { - if DebugLog { - fmt.Printf("[Gying] [%d/%d] ✅ 成功: %s (%d个链接)\n", - index+1, len(searchData.L.I), result.Title, len(result.Links)) - } - mu.Lock() - results = append(results, result) - successCount++ - mu.Unlock() - } else { - if DebugLog { - fmt.Printf("[Gying] [%d/%d] ⚠️ 跳过: 标题或链接为空 (标题:%s, 链接数:%d)\n", - index+1, len(searchData.L.I), result.Title, len(result.Links)) - } - } - }(i) - } - - wg.Wait() - - // 检查是否有403错误 - select { - case err := <-errChan: - if DebugLog { - fmt.Printf("[Gying] <<< fetchAllDetails 检测到403错误，需要重新登录\n") - } - return nil, err - default: - } - - if DebugLog { - fmt.Printf("[Gying] <<< fetchAllDetails 完成: 成功=%d, 失败=%d, 总计=%d\n", - successCount, failCount, len(searchData.L.I)) - } - - return results, nil -} - -// fetchDetail 获取详情 -func (p *GyingPlugin) fetchDetail(resourceID, resourceType string, scraper *cloudscraper.Scraper) (*DetailData, error) { - detailURL := fmt.Sprintf("https://www.gying.net/res/downurl/%s/%s", resourceType, resourceID) - - if DebugLog { - fmt.Printf("[Gying] fetchDetail: %s\n", detailURL) - } - - // 使用cloudscraper发送请求（自动管理Cookie和绕过反爬虫） - resp, err := scraper.Get(detailURL) - - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 请求失败: %v\n", err) - } - return nil, err - } - defer resp.Body.Close() - - if DebugLog { - fmt.Printf("[Gying] 响应状态码: %d\n", resp.StatusCode) - } - - // 检查403错误 - if resp.StatusCode == 403 { - if DebugLog { - fmt.Printf("[Gying] ❌ 详情接口返回403 - Cookie可能已过期\n") - } - return nil, fmt.Errorf("HTTP 403 Forbidden") - } - - if resp.StatusCode != 200 { - if DebugLog { - fmt.Printf("[Gying] ❌ HTTP错误: %d\n", resp.StatusCode) - } - return nil, fmt.Errorf("HTTP %d", resp.StatusCode) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - if DebugLog { - fmt.Printf("[Gying] 读取响应失败: %v\n", err) - } - return nil, err - } - - if DebugLog { - fmt.Printf("[Gying] 响应长度: %d 字节\n", len(body)) - } - - var detail DetailData - if err := json.Unmarshal(body, &detail); err != nil { - if DebugLog { - fmt.Printf("[Gying] JSON解析失败: %v\n", err) - // 打印前200字符 - preview := string(body) - if len(preview) > 200 { - preview = preview[:200] + "..." - } - fmt.Printf("[Gying] 响应内容: %s\n", preview) - } - return nil, err - } - - if DebugLog { - fmt.Printf("[Gying] 详情Code: %d, 网盘链接数: %d\n", detail.Code, len(detail.Panlist.URL)) - } - - // 检查JSON响应中的code字段（关键！） - if detail.Code == 403 { - if DebugLog { - fmt.Printf("[Gying] ❌ 详情接口返回Code=403 - 登录状态可能已失效\n") - } - return nil, fmt.Errorf("Detail API returned code 403 - authentication may have expired") - } - - return &detail, nil -} - -// buildResult 构建SearchResult -func (p *GyingPlugin) buildResult(detail *DetailData, searchData *SearchData, index int) model.SearchResult { - if index >= len(searchData.L.Title) { - return model.SearchResult{} - } - - title := searchData.L.Title[index] - resourceType := searchData.L.D[index] - resourceID := searchData.L.I[index] - - // 构建描述 - var contentParts []string - if index < len(searchData.L.Info) && searchData.L.Info[index] != "" { - contentParts = append(contentParts, searchData.L.Info[index]) - } - if index < len(searchData.L.Daoyan) && searchData.L.Daoyan[index] != "" { - contentParts = append(contentParts, fmt.Sprintf("导演: %s", searchData.L.Daoyan[index])) - } - if index < len(searchData.L.Zhuyan) && searchData.L.Zhuyan[index] != "" { - contentParts = append(contentParts, fmt.Sprintf("主演: %s", searchData.L.Zhuyan[index])) - } - - // 提取网盘链接 - links := p.extractPanLinks(detail) - - // 构建标签 - var tags []string - if index < len(searchData.L.Year) && searchData.L.Year[index] > 0 { - tags = append(tags, fmt.Sprintf("%d", searchData.L.Year[index])) - } - - return model.SearchResult{ - UniqueID: fmt.Sprintf("gying-%s-%s", resourceType, resourceID), - Title: title, - Content: strings.Join(contentParts, " | "), - Links: links, - Tags: tags, - Channel: "", // 插件搜索结果Channel为空 - Datetime: time.Now(), } } -// extractPanLinks 提取网盘链接 -func (p *GyingPlugin) extractPanLinks(detail *DetailData) []model.Link { - var links []model.Link - seen := make(map[string]bool) - - for i := 0; i < len(detail.Panlist.URL); i++ { - linkURL := strings.TrimSpace(detail.Panlist.URL[i]) - - // 去除URL中的访问码标记 - linkURL = regexp.MustCompile(`（访问码：.*?）`).ReplaceAllString(linkURL, "") - linkURL = regexp.MustCompile(`\(访问码：.*?\)`).ReplaceAllString(linkURL, "") - linkURL = strings.TrimSpace(linkURL) - - if linkURL == "" || seen[linkURL] { - continue - } - seen[linkURL] = true - - // 识别网盘类型 - linkType := p.determineLinkType(linkURL) - if linkType == "others" { - continue - } - - // 提取提取码 - password := "" - if i < len(detail.Panlist.P) && detail.Panlist.P[i] != "" { - password = detail.Panlist.P[i] - } - - // 从URL提取提取码（优先） - if urlPwd := p.extractPasswordFromURL(linkURL); urlPwd != "" { - password = urlPwd - } - - links = append(links, model.Link{ - Type: linkType, - URL: linkURL, - Password: password, - }) - } - - return links +// GetPlugins 获取所有注册的异步插件 +func (pm *PluginManager) GetPlugins() []AsyncSearchPlugin { + return pm.plugins } -// determineLinkType 识别网盘类型 -func (p *GyingPlugin) determineLinkType(linkURL string) string { - switch { - case strings.Contains(linkURL, "pan.quark.cn"): - return "quark" - case strings.Contains(linkURL, "drive.uc.cn"): - return "uc" - case strings.Contains(linkURL, "pan.baidu.com"): - return "baidu" - case strings.Contains(linkURL, "aliyundrive.com") || strings.Contains(linkURL, "alipan.com"): - return "aliyun" - case strings.Contains(linkURL, "pan.xunlei.com"): - return "xunlei" - case strings.Contains(linkURL, "cloud.189.cn"): - return "tianyi" - case strings.Contains(linkURL, "115.com"): - return "115" - case strings.Contains(linkURL, "123pan.com"): - return "123" - default: - return "others" - } -} +// ============================================================ +// 第四部分：工具函数 +// ============================================================ -// extractPasswordFromURL 从URL提取提取码 -func (p *GyingPlugin) extractPasswordFromURL(linkURL string) string { - // 百度网盘: ?pwd=xxxx - if strings.Contains(linkURL, "?pwd=") { - re := regexp.MustCompile(`\?pwd=([a-zA-Z0-9]+)`) - if matches := re.FindStringSubmatch(linkURL); len(matches) > 1 { - return matches[1] - } +// FilterResultsByKeyword 根据关键词过滤搜索结果的全局辅助函数 +func FilterResultsByKeyword(results []model.SearchResult, keyword string) []model.SearchResult { + if keyword == "" { + return results } - // 115网盘: ?password=xxxx - if strings.Contains(linkURL, "?password=") { - re := regexp.MustCompile(`\?password=([a-zA-Z0-9]+)`) - if matches := re.FindStringSubmatch(linkURL); len(matches) > 1 { - return matches[1] - } - } + // 预估过滤后会保留80%的结果 + filteredResults := make([]model.SearchResult, 0, len(results)*8/10) - return "" -} + // 将关键词转为小写，用于不区分大小写的比较 + lowerKeyword := strings.ToLower(keyword) -// deduplicateResults 去重 -func (p *GyingPlugin) deduplicateResults(results []model.SearchResult) []model.SearchResult { - seen := make(map[string]bool) - var deduplicated []model.SearchResult + // 将关键词按空格分割，用于支持多关键词搜索 + keywords := strings.Fields(lowerKeyword) for _, result := range results { - if !seen[result.UniqueID] { - seen[result.UniqueID] = true - deduplicated = append(deduplicated, result) + // 将标题和内容转为小写 + lowerTitle := strings.ToLower(result.Title) + lowerContent := strings.ToLower(result.Content) + + // 检查每个关键词是否在标题或内容中 + matched := true + for _, kw := range keywords { + // 对于所有关键词，检查是否在标题或内容中 + if !strings.Contains(lowerTitle, kw) && !strings.Contains(lowerContent, kw) { + matched = false + break + } + } + + if matched { + filteredResults = append(filteredResults, result) } } - return deduplicated + return filteredResults } -// ============ 工具函数 ============ +// ============================================================ +// 第五部分：异步插件基础设施（初始化、工作池、缓存） +// ============================================================ -// generateHash 生成hash -func (p *GyingPlugin) generateHash(username string) string { - salt := os.Getenv("GYING_HASH_SALT") - if salt == "" { - salt = "pansou_gying_secret_2025" - } - data := username + salt - hash := sha256.Sum256([]byte(data)) - return hex.EncodeToString(hash[:]) -} - -// maskUsername 脱敏用户名 -func (p *GyingPlugin) maskUsername(username string) string { - if len(username) <= 2 { - return username - } - if len(username) <= 4 { - return username[:1] + "**" + username[len(username)-1:] - } - return username[:2] + "****" + username[len(username)-2:] -} - -// isHexString 判断是否为十六进制 -func (p *GyingPlugin) isHexString(s string) bool { - for _, c := range s { - if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { - return false - } - } - return true -} - -// respondSuccess 成功响应 -func respondSuccess(c *gin.Context, message string, data interface{}) { - c.JSON(200, gin.H{ - "success": true, - "message": message, - "data": data, - }) -} - -// respondError 错误响应 -func respondError(c *gin.Context, message string) { - c.JSON(200, gin.H{ - "success": false, - "message": message, - "data": nil, - }) -} - -// ============ Cookie加密（可选） ============ - -func getEncryptionKey() []byte { - key := os.Getenv("GYING_ENCRYPTION_KEY") - if key == "" { - key = "default-32-byte-key-change-me!" - } - return []byte(key)[:32] -} - -func encryptCookie(plaintext string) (string, error) { - key := getEncryptionKey() - block, err := aes.NewCipher(key) - if err != nil { - return "", err - } - - gcm, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - nonce := make([]byte, gcm.NonceSize()) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return "", err - } - - ciphertext := gcm.Seal(nonce, nonce, []byte(plaintext), nil) - return base64.StdEncoding.EncodeToString(ciphertext), nil -} - -func decryptCookie(encrypted string) (string, error) { - key := getEncryptionKey() - ciphertext, err := base64.StdEncoding.DecodeString(encrypted) - if err != nil { - return "", err - } - - block, err := aes.NewCipher(key) - if err != nil { - return "", err - } - - gcm, err := cipher.NewGCM(block) - if err != nil { - return "", err - } - - nonceSize := gcm.NonceSize() - if len(ciphertext) < nonceSize { - return "", fmt.Errorf("ciphertext too short") - } - - nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] - plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) - if err != nil { - return "", err - } - - return string(plaintext), nil -} - -// ============ 定期清理 ============ - -func (p *GyingPlugin) startCleanupTask() { - ticker := time.NewTicker(24 * time.Hour) - for range ticker.C { - deleted := p.cleanupExpiredUsers() - marked := p.markInactiveUsers() - - if deleted > 0 || marked > 0 { - fmt.Printf("[Gying] 清理任务完成: 删除 %d 个过期用户, 标记 %d 个不活跃用户\n", deleted, marked) - } - } -} - -func (p *GyingPlugin) cleanupExpiredUsers() int { - deletedCount := 0 +// cleanupExpiredApiCache 清理过期API缓存的函数 +func cleanupExpiredApiCache() { + cleanupMutex.Lock() + defer cleanupMutex.Unlock() + now := time.Now() - expireThreshold := now.AddDate(0, 0, -30) - - p.users.Range(func(key, value interface{}) bool { - user := value.(*User) - if user.Status == "expired" && user.LastAccessAt.Before(expireThreshold) { - if err := p.deleteUser(user.Hash); err == nil { - deletedCount++ + // 只有距离上次清理超过30分钟才执行 + if now.Sub(lastCleanupTime) < 30*time.Minute { + return + } + + cleanedCount := 0 + totalCount := 0 + deletedKeys := make([]string, 0) + + // 清理已过期的缓存（基于实际TTL + 合理的宽限期） + apiResponseCache.Range(func(key, value interface{}) bool { + totalCount++ + if cached, ok := value.(cachedResponse); ok { + // 使用默认TTL + 30分钟宽限期，避免过于激进的清理 + expireThreshold := defaultCacheTTL + 30*time.Minute + if now.Sub(cached.Timestamp) > expireThreshold { + keyStr := key.(string) + apiResponseCache.Delete(key) + deletedKeys = append(deletedKeys, keyStr) + cleanedCount++ } } return true }) - - return deletedCount + + // 清理访问计数缓存中对应的项 + for _, key := range deletedKeys { + cacheAccessCount.Delete(key) + } + + lastCleanupTime = now + + // 记录清理日志（仅在有清理时输出） + if cleanedCount > 0 { + fmt.Printf("[Cache] 清理过期缓存: 删除 %d/%d 项，释放内存\n", cleanedCount, totalCount) + } } -func (p *GyingPlugin) markInactiveUsers() int { - markedCount := 0 +// initAsyncPlugin 初始化异步插件配置 +func initAsyncPlugin() { + initLock.Lock() + defer initLock.Unlock() + + if initialized { + return + } + + // 如果配置已加载，则从配置读取工作池大小 + maxWorkers := defaultMaxBackgroundWorkers + if config.AppConfig != nil { + maxWorkers = config.AppConfig.AsyncMaxBackgroundWorkers + } + + backgroundWorkerPool = make(chan struct{}, maxWorkers) + + // 异步插件本地缓存系统已移除，现在只依赖主缓存系统 + + initialized = true +} + +// InitAsyncPluginSystem 导出的初始化函数，用于确保异步插件系统初始化 +func InitAsyncPluginSystem() { + initAsyncPlugin() +} + +// acquireWorkerSlot 尝试获取工作槽 +func acquireWorkerSlot() bool { + // 获取最大任务数 + maxTasks := int32(defaultMaxBackgroundTasks) + if config.AppConfig != nil { + maxTasks = int32(config.AppConfig.AsyncMaxBackgroundTasks) + } + + // 检查总任务数 + if atomic.LoadInt32(&backgroundTasksCount) >= maxTasks { + return false + } + + // 尝试获取工作槽 + select { + case backgroundWorkerPool <- struct{}{}: + atomic.AddInt32(&backgroundTasksCount, 1) + return true + default: + return false + } +} + +// releaseWorkerSlot 释放工作槽 +func releaseWorkerSlot() { + <-backgroundWorkerPool + atomic.AddInt32(&backgroundTasksCount, -1) +} + +// recordCacheHit 记录缓存命中 (内部使用) +func recordCacheHit() { + atomic.AddInt64(&cacheHits, 1) +} + +// recordCacheMiss 记录缓存未命中 (内部使用) +func recordCacheMiss() { + atomic.AddInt64(&cacheMisses, 1) +} + +// recordAsyncCompletion 记录异步完成 (内部使用) +func recordAsyncCompletion() { + atomic.AddInt64(&asyncCompletions, 1) +} + +// recordCacheAccess 记录缓存访问次数，用于智能缓存策略（仅内存） +func recordCacheAccess(key string) { + // 更新缓存项的访问时间和计数 + if cached, ok := apiResponseCache.Load(key); ok { + cachedItem := cached.(cachedResponse) + cachedItem.LastAccess = time.Now() + cachedItem.AccessCount++ + apiResponseCache.Store(key, cachedItem) + } + + // 更新全局访问计数 + if count, ok := cacheAccessCount.Load(key); ok { + cacheAccessCount.Store(key, count.(int) + 1) + } else { + cacheAccessCount.Store(key, 1) + } + + // 触发定期清理（异步执行，不阻塞当前操作） + go cleanupExpiredApiCache() +} + +// ============================================================ +// 第六部分：BaseAsyncPlugin 结构和构造函数 +// ============================================================ + +// BaseAsyncPlugin 基础异步插件结构 +type BaseAsyncPlugin struct { + name string + priority int + client *http.Client // 用于短超时的客户端 + backgroundClient *http.Client // 用于长超时的客户端 + cacheTTL time.Duration // 内存缓存有效期 + mainCacheUpdater func(string, []model.SearchResult, time.Duration, bool, string) error // 主缓存更新函数（支持IsFinal参数，接收原始数据，最后参数为关键词） + MainCacheKey string // 主缓存键，导出字段 + currentKeyword string // 当前搜索的关键词，用于日志显示 + finalUpdateTracker map[string]bool // 追踪已更新的最终结果缓存 + finalUpdateMutex sync.RWMutex // 保护finalUpdateTracker的并发访问 + skipServiceFilter bool // 是否跳过Service层的关键词过滤 +} + +// NewBaseAsyncPlugin 创建基础异步插件 +func NewBaseAsyncPlugin(name string, priority int) *BaseAsyncPlugin { + // 确保异步插件已初始化 + if !initialized { + initAsyncPlugin() + } + + // 确定超时和缓存时间 + responseTimeout := defaultAsyncResponseTimeout + processingTimeout := defaultPluginTimeout + cacheTTL := defaultCacheTTL + + // 如果配置已初始化，则使用配置中的值 + if config.AppConfig != nil { + responseTimeout = config.AppConfig.AsyncResponseTimeoutDur + processingTimeout = config.AppConfig.PluginTimeout + cacheTTL = time.Duration(config.AppConfig.AsyncCacheTTLHours) * time.Hour + } + + return &BaseAsyncPlugin{ + name: name, + priority: priority, + client: &http.Client{ + Timeout: responseTimeout, + }, + backgroundClient: &http.Client{ + Timeout: processingTimeout, + }, + cacheTTL: cacheTTL, + finalUpdateTracker: make(map[string]bool), // 初始化缓存更新追踪器 + skipServiceFilter: false, // 默认不跳过Service层过滤 + } +} + +// NewBaseAsyncPluginWithFilter 创建基础异步插件（支持设置Service层过滤参数） +func NewBaseAsyncPluginWithFilter(name string, priority int, skipServiceFilter bool) *BaseAsyncPlugin { + // 确保异步插件已初始化 + if !initialized { + initAsyncPlugin() + } + + // 确定超时和缓存时间 + responseTimeout := defaultAsyncResponseTimeout + processingTimeout := defaultPluginTimeout + cacheTTL := defaultCacheTTL + + // 如果配置已初始化，则使用配置中的值 + if config.AppConfig != nil { + responseTimeout = config.AppConfig.AsyncResponseTimeoutDur + processingTimeout = config.AppConfig.PluginTimeout + cacheTTL = time.Duration(config.AppConfig.AsyncCacheTTLHours) * time.Hour + } + + return &BaseAsyncPlugin{ + name: name, + priority: priority, + client: &http.Client{ + Timeout: responseTimeout, + }, + backgroundClient: &http.Client{ + Timeout: processingTimeout, + }, + cacheTTL: cacheTTL, + finalUpdateTracker: make(map[string]bool), // 初始化缓存更新追踪器 + skipServiceFilter: skipServiceFilter, // 使用传入的过滤设置 + } +} + +// ============================================================ +// 第七部分：BaseAsyncPlugin 接口实现方法 +// ============================================================ + +// SetMainCacheKey 设置主缓存键 +func (p *BaseAsyncPlugin) SetMainCacheKey(key string) { + p.MainCacheKey = key +} + +// SetCurrentKeyword 设置当前搜索关键词（用于日志显示） +func (p *BaseAsyncPlugin) SetCurrentKeyword(keyword string) { + p.currentKeyword = keyword +} + +// SetMainCacheUpdater 设置主缓存更新函数（修复后的签名，增加关键词参数） +func (p *BaseAsyncPlugin) SetMainCacheUpdater(updater func(string, []model.SearchResult, time.Duration, bool, string) error) { + p.mainCacheUpdater = updater +} + +// Name 返回插件名称 +func (p *BaseAsyncPlugin) Name() string { + return p.name +} + +// Priority 返回插件优先级 +func (p *BaseAsyncPlugin) Priority() int { + return p.priority +} + +// SkipServiceFilter 返回是否跳过Service层的关键词过滤 +func (p *BaseAsyncPlugin) SkipServiceFilter() bool { + return p.skipServiceFilter +} + +// GetClient 返回短超时客户端 +func (p *BaseAsyncPlugin) GetClient() *http.Client { + return p.client +} + +// ============================================================ +// 第八部分：异步搜索核心逻辑 +// ============================================================ + +// AsyncSearch 异步搜索基础方法 +func (p *BaseAsyncPlugin) AsyncSearch( + keyword string, + searchFunc func(*http.Client, string, map[string]interface{}) ([]model.SearchResult, error), + mainCacheKey string, + ext map[string]interface{}, +) ([]model.SearchResult, error) { + // 确保ext不为nil + if ext == nil { + ext = make(map[string]interface{}) + } + now := time.Now() - inactiveThreshold := now.AddDate(0, 0, -90) - - p.users.Range(func(key, value interface{}) bool { - user := value.(*User) - if user.LastAccessAt.Before(inactiveThreshold) && user.Status != "expired" { - user.Status = "expired" - user.Cookie = "" - - if err := p.saveUser(user); err == nil { - markedCount++ + + // 修改缓存键，确保包含插件名称 + pluginSpecificCacheKey := fmt.Sprintf("%s:%s", p.name, keyword) + + // 检查是否有refresh参数，如果为true则跳过缓存 + skipCache := false + if refreshValue, ok := ext["refresh"]; ok { + if refresh, ok := refreshValue.(bool); ok && refresh { + skipCache = true + } + } + + // 检查缓存（除非refresh=true） + if !skipCache { + if cachedItems, ok := apiResponseCache.Load(pluginSpecificCacheKey); ok { + cachedResult := cachedItems.(cachedResponse) + + // 缓存完全有效（未过期且完整） + if time.Since(cachedResult.Timestamp) < p.cacheTTL && cachedResult.Complete { + recordCacheHit() + recordCacheAccess(pluginSpecificCacheKey) + + // 如果缓存接近过期（已用时间超过TTL的80%），在后台刷新缓存 + if time.Since(cachedResult.Timestamp) > (p.cacheTTL * 4 / 5) { + go p.refreshCacheInBackground(keyword, pluginSpecificCacheKey, searchFunc, cachedResult, mainCacheKey, ext) + } + + return cachedResult.Results, nil + } + + // 缓存已过期但有结果，启动后台刷新，同时返回旧结果 + if len(cachedResult.Results) > 0 { + recordCacheHit() + recordCacheAccess(pluginSpecificCacheKey) + + // 标记为部分过期 + if time.Since(cachedResult.Timestamp) >= p.cacheTTL { + // 在后台刷新缓存 + go p.refreshCacheInBackground(keyword, pluginSpecificCacheKey, searchFunc, cachedResult, mainCacheKey, ext) + + // 日志记录 + fmt.Printf("[%s] 缓存已过期，后台刷新中: %s (已过期: %v)\n", + p.name, pluginSpecificCacheKey, time.Since(cachedResult.Timestamp)) + } + + return cachedResult.Results, nil } } - return true - }) - - return markedCount + } + + recordCacheMiss() + + // 创建通道 + resultChan := make(chan []model.SearchResult, 1) + errorChan := make(chan error, 1) + doneChan := make(chan struct{}) + + // 启动后台处理 + go func() { + // 尝试获取工作槽 + if !acquireWorkerSlot() { + // 工作池已满，使用快速响应客户端直接处理 + results, err := searchFunc(p.client, keyword, ext) + if err != nil { + select { + case errorChan <- err: + default: + } + return + } + + select { + case resultChan <- results: + default: + } + + // 缓存结果 + apiResponseCache.Store(pluginSpecificCacheKey, cachedResponse{ + Results: results, + Timestamp: now, + Complete: true, + LastAccess: now, + AccessCount: 1, + }) + + // 🔧 工作池满时短超时(默认4秒)内完成，这是完整结果 + p.updateMainCacheWithFinal(mainCacheKey, results, true) + + return + } + defer releaseWorkerSlot() + + // 执行搜索 + results, err := searchFunc(p.backgroundClient, keyword, ext) + + // 检查是否已经响应 + select { + case <-doneChan: + // 已经响应，只更新缓存 + if err == nil { + // 检查是否存在旧缓存 + var accessCount int = 1 + var lastAccess time.Time = now + + if oldCache, ok := apiResponseCache.Load(pluginSpecificCacheKey); ok { + oldCachedResult := oldCache.(cachedResponse) + accessCount = oldCachedResult.AccessCount + lastAccess = oldCachedResult.LastAccess + + // 合并结果（新结果优先） + if len(oldCachedResult.Results) > 0 { + // 创建合并结果集 + mergedResults := make([]model.SearchResult, 0, len(results) + len(oldCachedResult.Results)) + + // 创建已有结果ID的映射 + existingIDs := make(map[string]bool) + for _, r := range results { + existingIDs[r.UniqueID] = true + mergedResults = append(mergedResults, r) + } + + // 添加旧结果中不存在的项 + for _, r := range oldCachedResult.Results { + if !existingIDs[r.UniqueID] { + mergedResults = append(mergedResults, r) + } + } + + // 使用合并结果 + results = mergedResults + } + } + + apiResponseCache.Store(pluginSpecificCacheKey, cachedResponse{ + Results: results, + Timestamp: now, + Complete: true, + LastAccess: lastAccess, + AccessCount: accessCount, + }) + recordAsyncCompletion() + + // 异步插件后台完成时更新主缓存（标记为最终结果） + p.updateMainCacheWithFinal(mainCacheKey, results, true) + + // 异步插件本地缓存系统已移除 + } + default: + // 尚未响应，发送结果 + if err != nil { + select { + case errorChan <- err: + default: + } + } else { + // 检查是否存在旧缓存用于合并 + if oldCache, ok := apiResponseCache.Load(pluginSpecificCacheKey); ok { + oldCachedResult := oldCache.(cachedResponse) + if len(oldCachedResult.Results) > 0 { + // 创建合并结果集 + mergedResults := make([]model.SearchResult, 0, len(results) + len(oldCachedResult.Results)) + + // 创建已有结果ID的映射 + existingIDs := make(map[string]bool) + for _, r := range results { + existingIDs[r.UniqueID] = true + mergedResults = append(mergedResults, r) + } + + // 添加旧结果中不存在的项 + for _, r := range oldCachedResult.Results { + if !existingIDs[r.UniqueID] { + mergedResults = append(mergedResults, r) + } + } + + // 使用合并结果 + results = mergedResults + } + } + + select { + case resultChan <- results: + default: + } + + // 更新缓存 + apiResponseCache.Store(pluginSpecificCacheKey, cachedResponse{ + Results: results, + Timestamp: now, + Complete: true, + LastAccess: now, + AccessCount: 1, + }) + + // 🔧 短超时(默认4秒)内正常完成，这是完整的最终结果 + p.updateMainCacheWithFinal(mainCacheKey, results, true) + + // 异步插件本地缓存系统已移除 + } + } + }() + + // 获取响应超时时间 + responseTimeout := defaultAsyncResponseTimeout + if config.AppConfig != nil { + responseTimeout = config.AppConfig.AsyncResponseTimeoutDur + } + + // 等待响应超时或结果 + select { + case results := <-resultChan: + close(doneChan) + return results, nil + case err := <-errorChan: + close(doneChan) + return nil, err + case <-time.After(responseTimeout): + // 插件响应超时，后台继续处理（优化完成，日志简化） + + // 响应超时，返回空结果，后台继续处理 + go func() { + defer close(doneChan) + }() + + // 检查是否有部分缓存可用 + if cachedItems, ok := apiResponseCache.Load(pluginSpecificCacheKey); ok { + cachedResult := cachedItems.(cachedResponse) + if len(cachedResult.Results) > 0 { + // 有部分缓存可用，记录访问并返回 + recordCacheAccess(pluginSpecificCacheKey) + fmt.Printf("[%s] 响应超时，返回部分缓存: %s (项目数: %d)\n", + p.name, pluginSpecificCacheKey, len(cachedResult.Results)) + return cachedResult.Results, nil + } + } + + // 创建空的临时缓存，以便后台处理完成后可以更新 + apiResponseCache.Store(pluginSpecificCacheKey, cachedResponse{ + Results: []model.SearchResult{}, + Timestamp: now, + Complete: false, // 标记为不完整 + LastAccess: now, + AccessCount: 1, + }) + + // 🔧 修复：4秒超时时也要更新主缓存，标记为部分结果（空结果） + p.updateMainCacheWithFinal(mainCacheKey, []model.SearchResult{}, false) + + // fmt.Printf("[%s] 响应超时，后台继续处理: %s\n", p.name, pluginSpecificCacheKey) + return []model.SearchResult{}, nil + } } +// AsyncSearchWithResult 异步搜索方法，返回PluginSearchResult +func (p *BaseAsyncPlugin) AsyncSearchWithResult( + keyword string, + searchFunc func(*http.Client, string, map[string]interface{}) ([]model.SearchResult, error), + mainCacheKey string, + ext map[string]interface{}, +) (model.PluginSearchResult, error) { + // 确保ext不为nil + if ext == nil { + ext = make(map[string]interface{}) + } + + now := time.Now() + + // 修改缓存键，确保包含插件名称 + pluginSpecificCacheKey := fmt.Sprintf("%s:%s", p.name, keyword) + + // 检查是否有refresh参数，如果为true则跳过缓存 + skipCache := false + if refreshValue, ok := ext["refresh"]; ok { + if refresh, ok := refreshValue.(bool); ok && refresh { + skipCache = true + } + } + + // 检查缓存（除非refresh=true） + if !skipCache { + if cachedItems, ok := apiResponseCache.Load(pluginSpecificCacheKey); ok { + cachedResult := cachedItems.(cachedResponse) + + // 缓存完全有效（未过期且完整） + if time.Since(cachedResult.Timestamp) < p.cacheTTL && cachedResult.Complete { + recordCacheHit() + recordCacheAccess(pluginSpecificCacheKey) + + // 如果缓存接近过期（已用时间超过TTL的80%），在后台刷新缓存 + if time.Since(cachedResult.Timestamp) > (p.cacheTTL * 4 / 5) { + go p.refreshCacheInBackground(keyword, pluginSpecificCacheKey, searchFunc, cachedResult, mainCacheKey, ext) + } + + return model.PluginSearchResult{ + Results: cachedResult.Results, + IsFinal: cachedResult.Complete, + Timestamp: cachedResult.Timestamp, + Source: p.name, + Message: "从缓存获取", + }, nil + } + + // 缓存已过期但有结果，启动后台刷新，同时返回旧结果 + if len(cachedResult.Results) > 0 { + recordCacheHit() + recordCacheAccess(pluginSpecificCacheKey) + + // 标记为部分过期 + if time.Since(cachedResult.Timestamp) >= p.cacheTTL { + // 在后台刷新缓存 + go p.refreshCacheInBackground(keyword, pluginSpecificCacheKey, searchFunc, cachedResult, mainCacheKey, ext) + } + + return model.PluginSearchResult{ + Results: cachedResult.Results, + IsFinal: false, // 🔥 过期数据标记为非最终结果 + Timestamp: cachedResult.Timestamp, + Source: p.name, + Message: "缓存已过期，后台刷新中", + }, nil + } + } + } + + recordCacheMiss() + + // 创建通道 + resultChan := make(chan []model.SearchResult, 1) + errorChan := make(chan error, 1) + doneChan := make(chan struct{}) + + // 启动后台处理 + go func() { + defer func() { + select { + case <-doneChan: + default: + close(doneChan) + } + }() + + // 尝试获取工作槽 + if !acquireWorkerSlot() { + // 工作池已满，使用快速响应客户端直接处理 + results, err := searchFunc(p.client, keyword, ext) + if err != nil { + select { + case errorChan <- err: + default: + } + return + } + + select { + case resultChan <- results: + default: + } + return + } + defer releaseWorkerSlot() + + // 使用长超时客户端进行搜索 + results, err := searchFunc(p.backgroundClient, keyword, ext) + if err != nil { + select { + case errorChan <- err: + default: + } + } else { + select { + case resultChan <- results: + default: + } + } + }() + + // 等待结果或超时 + responseTimeout := defaultAsyncResponseTimeout + if config.AppConfig != nil { + responseTimeout = config.AppConfig.AsyncResponseTimeoutDur + } + + select { + case results := <-resultChan: + // 不直接关闭，让defer处理 + + // 缓存结果 + apiResponseCache.Store(pluginSpecificCacheKey, cachedResponse{ + Results: results, + Timestamp: now, + Complete: true, // 🔥 及时完成，标记为完整结果 + LastAccess: now, + AccessCount: 1, + }) + + // 🔧 恢复主缓存更新：使用统一的GOB序列化 + // 传递原始数据，由主程序负责序列化 + if mainCacheKey != "" && p.mainCacheUpdater != nil { + err := p.mainCacheUpdater(mainCacheKey, results, p.cacheTTL, true, p.currentKeyword) + if err != nil { + fmt.Printf("❌ [%s] 及时完成缓存更新失败: %s | 错误: %v\n", p.name, mainCacheKey, err) + } + } + + return model.PluginSearchResult{ + Results: results, + IsFinal: true, // 🔥 及时完成，最终结果 + Timestamp: now, + Source: p.name, + Message: "搜索完成", + }, nil + + case err := <-errorChan: + // 不直接关闭，让defer处理 + return model.PluginSearchResult{}, err + + case <-time.After(responseTimeout): + // 🔥 超时处理：返回空结果，后台继续处理 + go p.completeSearchInBackground(keyword, searchFunc, pluginSpecificCacheKey, mainCacheKey, doneChan, ext) + + // 存储临时缓存（标记为不完整） + apiResponseCache.Store(pluginSpecificCacheKey, cachedResponse{ + Results: []model.SearchResult{}, + Timestamp: now, + Complete: false, // 🔥 标记为不完整 + LastAccess: now, + AccessCount: 1, + }) + + return model.PluginSearchResult{ + Results: []model.SearchResult{}, + IsFinal: false, // 🔥 超时返回，非最终结果 + Timestamp: now, + Source: p.name, + Message: "处理中，后台继续...", + }, nil + } +} + +// completeSearchInBackground 后台完成搜索 +func (p *BaseAsyncPlugin) completeSearchInBackground( + keyword string, + searchFunc func(*http.Client, string, map[string]interface{}) ([]model.SearchResult, error), + pluginCacheKey string, + mainCacheKey string, + doneChan chan struct{}, + ext map[string]interface{}, +) { + defer func() { + select { + case <-doneChan: + default: + close(doneChan) + } + }() + + // 执行完整搜索 + results, err := searchFunc(p.backgroundClient, keyword, ext) + if err != nil { + return + } + + // 更新插件缓存 + now := time.Now() + apiResponseCache.Store(pluginCacheKey, cachedResponse{ + Results: results, + Timestamp: now, + Complete: true, // 🔥 标记为完整结果 + LastAccess: now, + AccessCount: 1, + }) + + // 🔧 恢复主缓存更新：使用统一的GOB序列化 + // 传递原始数据，由主程序负责序列化 + if mainCacheKey != "" && p.mainCacheUpdater != nil { + err := p.mainCacheUpdater(mainCacheKey, results, p.cacheTTL, true, p.currentKeyword) + if err != nil { + fmt.Printf("❌ [%s] 后台完成缓存更新失败: %s | 错误: %v\n", p.name, mainCacheKey, err) + } + } +} + +// refreshCacheInBackground 在后台刷新缓存 +func (p *BaseAsyncPlugin) refreshCacheInBackground( + keyword string, + cacheKey string, + searchFunc func(*http.Client, string, map[string]interface{}) ([]model.SearchResult, error), + oldCache cachedResponse, + originalCacheKey string, + ext map[string]interface{}, +) { + // 确保ext不为nil + if ext == nil { + ext = make(map[string]interface{}) + } + + // 注意：这里的cacheKey已经是插件特定的了，因为是从AsyncSearch传入的 + + // 检查是否有足够的工作槽 + if !acquireWorkerSlot() { + return + } + defer releaseWorkerSlot() + + // 记录刷新开始时间 + refreshStart := time.Now() + + // 执行搜索 + results, err := searchFunc(p.backgroundClient, keyword, ext) + if err != nil || len(results) == 0 { + return + } + + // 创建合并结果集 + mergedResults := make([]model.SearchResult, 0, len(results) + len(oldCache.Results)) + + // 创建已有结果ID的映射 + existingIDs := make(map[string]bool) + for _, r := range results { + existingIDs[r.UniqueID] = true + mergedResults = append(mergedResults, r) + } + + // 添加旧结果中不存在的项 + for _, r := range oldCache.Results { + if !existingIDs[r.UniqueID] { + mergedResults = append(mergedResults, r) + } + } + + // 更新缓存 + apiResponseCache.Store(cacheKey, cachedResponse{ + Results: mergedResults, + Timestamp: time.Now(), + Complete: true, + LastAccess: oldCache.LastAccess, + AccessCount: oldCache.AccessCount, + }) + + // 🔥 异步插件后台刷新完成时更新主缓存（标记为最终结果） + p.updateMainCacheWithFinal(originalCacheKey, mergedResults, true) + + // 记录刷新时间 + refreshTime := time.Since(refreshStart) + fmt.Printf("[%s] 后台刷新完成: %s (耗时: %v, 新项目: %d, 合并项目: %d)\n", + p.name, cacheKey, refreshTime, len(results), len(mergedResults)) + + // 异步插件本地缓存系统已移除 +} + +// ============================================================ +// 第九部分：缓存管理 +// ============================================================ + +// updateMainCache 更新主缓存系统（兼容性方法，默认IsFinal=true） +func (p *BaseAsyncPlugin) updateMainCache(cacheKey string, results []model.SearchResult) { + p.updateMainCacheWithFinal(cacheKey, results, true) +} + +// updateMainCacheWithFinal 更新主缓存系统，支持IsFinal参数 +func (p *BaseAsyncPlugin) updateMainCacheWithFinal(cacheKey string, results []model.SearchResult, isFinal bool) { + // 如果主缓存更新函数为空或缓存键为空，直接返回 + if p.mainCacheUpdater == nil || cacheKey == "" { + return + } + + // 🚀 优化：如果新结果为空，跳过缓存更新（避免无效操作） + if len(results) == 0 { + return + } + + // 🔥 增强防重复更新机制 - 使用数据哈希确保真正的去重 + // 生成结果数据的简单哈希标识 + dataHash := fmt.Sprintf("%d_%d", len(results), results[0].UniqueID) + if len(results) > 1 { + dataHash += fmt.Sprintf("_%d", results[len(results)-1].UniqueID) + } + updateKey := fmt.Sprintf("final_%s_%s_%s_%t", p.name, cacheKey, dataHash, isFinal) + + // 检查是否已经处理过相同的数据 + if p.hasUpdatedFinalCache(updateKey) { + return + } + + // 标记已更新 + p.markFinalCacheUpdated(updateKey) + + // 🔧 恢复异步插件缓存更新，使用修复后的统一序列化 + // 传递原始数据，由主程序负责GOB序列化 + if p.mainCacheUpdater != nil { + err := p.mainCacheUpdater(cacheKey, results, p.cacheTTL, isFinal, p.currentKeyword) + if err != nil { + fmt.Printf("❌ [%s] 主缓存更新失败: %s | 错误: %v\n", p.name, cacheKey, err) + } + } +} + +// hasUpdatedFinalCache 检查是否已经更新过指定的最终结果缓存 +func (p *BaseAsyncPlugin) hasUpdatedFinalCache(updateKey string) bool { + p.finalUpdateMutex.RLock() + defer p.finalUpdateMutex.RUnlock() + return p.finalUpdateTracker[updateKey] +} + +// markFinalCacheUpdated 标记已更新指定的最终结果缓存 +func (p *BaseAsyncPlugin) markFinalCacheUpdated(updateKey string) { + p.finalUpdateMutex.Lock() + defer p.finalUpdateMutex.Unlock() + p.finalUpdateTracker[updateKey] = true +} + +// ============================================================ +// 第十部分：序列化器 +// ============================================================ + +// SetGlobalCacheSerializer 设置全局缓存序列化器（由主程序调用） +func SetGlobalCacheSerializer(serializer interface { + Serialize(interface{}) ([]byte, error) + Deserialize([]byte, interface{}) error +}) { + globalCacheSerializer = serializer +} + +// getEnhancedCacheSerializer 获取增强缓存的序列化器 +func getEnhancedCacheSerializer() interface { + Serialize(interface{}) ([]byte, error) + Deserialize([]byte, interface{}) error +} { + return globalCacheSerializer +}