mirror of
https://github.com/fish2018/pansou.git
synced 2025-11-25 03:14:59 +08:00
优化内存管理
This commit is contained in:
@@ -12,7 +12,7 @@ PanSou是一个高性能的网盘资源搜索API服务,支持TG搜索和自定
|
||||
- ✅ **高可用性**: 长时间运行无故障
|
||||
|
||||
|
||||
## 特性
|
||||
## 特性([详情见PanSou系统开发设计文档](docs/PanSou%E7%B3%BB%E7%BB%9F%E5%BC%80%E5%8F%91%E8%AE%BE%E8%AE%A1%E6%96%87%E6%A1%A3.md))
|
||||
|
||||
- **高性能搜索**:并发搜索多个Telegram频道,显著提升搜索速度;工作池设计,高效管理并发任务
|
||||
- **网盘类型分类**:自动识别多种网盘链接,按类型归类展示
|
||||
|
||||
@@ -105,7 +105,7 @@ func (p *MyPlugin) searchImpl(client *http.Client, keyword string, ext map[strin
|
||||
|
||||
// 3. 发送HTTP请求
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
@@ -39,6 +39,10 @@ var (
|
||||
|
||||
// 缓存访问频率记录
|
||||
cacheAccessCount = sync.Map{}
|
||||
|
||||
// 🔥 新增:缓存清理相关变量
|
||||
lastCleanupTime = time.Now()
|
||||
cleanupMutex sync.Mutex
|
||||
)
|
||||
|
||||
// 缓存响应结构(仅内存,不持久化到磁盘)
|
||||
@@ -50,6 +54,50 @@ type cachedResponse struct {
|
||||
AccessCount int `json:"access_count"`
|
||||
}
|
||||
|
||||
// 🔥 新增:清理过期API缓存的函数
|
||||
func cleanupExpiredApiCache() {
|
||||
cleanupMutex.Lock()
|
||||
defer cleanupMutex.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
// 只有距离上次清理超过30分钟才执行
|
||||
if now.Sub(lastCleanupTime) < 30*time.Minute {
|
||||
return
|
||||
}
|
||||
|
||||
cleanedCount := 0
|
||||
totalCount := 0
|
||||
deletedKeys := make([]string, 0)
|
||||
|
||||
// 清理已过期的缓存(基于实际TTL + 合理的宽限期)
|
||||
apiResponseCache.Range(func(key, value interface{}) bool {
|
||||
totalCount++
|
||||
if cached, ok := value.(cachedResponse); ok {
|
||||
// 使用默认TTL + 30分钟宽限期,避免过于激进的清理
|
||||
expireThreshold := defaultCacheTTL + 30*time.Minute
|
||||
if now.Sub(cached.Timestamp) > expireThreshold {
|
||||
keyStr := key.(string)
|
||||
apiResponseCache.Delete(key)
|
||||
deletedKeys = append(deletedKeys, keyStr)
|
||||
cleanedCount++
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
// 清理访问计数缓存中对应的项
|
||||
for _, key := range deletedKeys {
|
||||
cacheAccessCount.Delete(key)
|
||||
}
|
||||
|
||||
lastCleanupTime = now
|
||||
|
||||
// 记录清理日志(仅在有清理时输出)
|
||||
if cleanedCount > 0 {
|
||||
fmt.Printf("[Cache] 清理过期缓存: 删除 %d/%d 项,释放内存\n", cleanedCount, totalCount)
|
||||
}
|
||||
}
|
||||
|
||||
// initAsyncPlugin 初始化异步插件配置
|
||||
func initAsyncPlugin() {
|
||||
initLock.Lock()
|
||||
@@ -143,6 +191,9 @@ func recordCacheAccess(key string) {
|
||||
} else {
|
||||
cacheAccessCount.Store(key, 1)
|
||||
}
|
||||
|
||||
// 🔥 新增:触发定期清理(异步执行,不阻塞当前操作)
|
||||
go cleanupExpiredApiCache()
|
||||
}
|
||||
|
||||
// BaseAsyncPlugin 基础异步插件结构(保留内存缓存,移除磁盘持久化)
|
||||
|
||||
2634
plugin/labi/1.txt
Normal file
2634
plugin/labi/1.txt
Normal file
File diff suppressed because it is too large
Load Diff
@@ -99,7 +99,7 @@ type SusuAsyncPlugin struct {
|
||||
// NewSusuAsyncPlugin 创建新的SuSu搜索异步插件
|
||||
func NewSusuAsyncPlugin() *SusuAsyncPlugin {
|
||||
return &SusuAsyncPlugin{
|
||||
BaseAsyncPlugin: plugin.NewBaseAsyncPlugin("susu", 4), // 高优先级
|
||||
BaseAsyncPlugin: plugin.NewBaseAsyncPlugin("susu", 1), // 高优先级
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ type XuexizhinanPlugin struct {
|
||||
// NewXuexizhinanPlugin 创建新的4K指南搜索异步插件
|
||||
func NewXuexizhinanPlugin() *XuexizhinanPlugin {
|
||||
return &XuexizhinanPlugin{
|
||||
BaseAsyncPlugin: plugin.NewBaseAsyncPlugin("xuexizhinan", 2), // 中等优先级
|
||||
BaseAsyncPlugin: plugin.NewBaseAsyncPlugin("xuexizhinan", 1), // 中等优先级
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
16
util/cache/sharded_disk_cache.go
vendored
16
util/cache/sharded_disk_cache.go
vendored
@@ -148,14 +148,16 @@ func (c *ShardedDiskCache) cleanExpired() {
|
||||
}
|
||||
}
|
||||
|
||||
// StartCleanupTask 启动定期清理任务
|
||||
// CleanExpired 公开的清理方法,符合cleanupTarget接口
|
||||
func (c *ShardedDiskCache) CleanExpired() {
|
||||
c.cleanExpired()
|
||||
}
|
||||
|
||||
// StartCleanupTask 启动定期清理任务(修改为使用单例模式)
|
||||
func (c *ShardedDiskCache) StartCleanupTask() {
|
||||
ticker := time.NewTicker(10 * time.Minute)
|
||||
go func() {
|
||||
for range ticker.C {
|
||||
c.cleanExpired()
|
||||
}
|
||||
}()
|
||||
// 使用与内存缓存相同的全局清理系统
|
||||
registerForCleanup(c)
|
||||
startGlobalCleanupTask()
|
||||
}
|
||||
|
||||
// GetShards 获取所有分片(用于测试和调试)
|
||||
|
||||
50
util/cache/sharded_memory_cache.go
vendored
50
util/cache/sharded_memory_cache.go
vendored
@@ -8,6 +8,19 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// 🔥 全局清理任务相关变量(单例模式)
|
||||
var (
|
||||
globalCleanupTicker *time.Ticker
|
||||
globalCleanupOnce sync.Once
|
||||
registeredCaches []cleanupTarget
|
||||
cacheRegistryMutex sync.RWMutex
|
||||
)
|
||||
|
||||
// 🔥 清理目标接口
|
||||
type cleanupTarget interface {
|
||||
CleanExpired()
|
||||
}
|
||||
|
||||
// 分片内存缓存项
|
||||
type shardedMemoryCacheItem struct {
|
||||
data []byte
|
||||
@@ -289,14 +302,37 @@ func (c *ShardedMemoryCache) Clear() {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// 启动定期清理
|
||||
// 🔥 启动全局清理任务(单例模式)
|
||||
func startGlobalCleanupTask() {
|
||||
globalCleanupOnce.Do(func() {
|
||||
globalCleanupTicker = time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
for range globalCleanupTicker.C {
|
||||
cacheRegistryMutex.RLock()
|
||||
caches := make([]cleanupTarget, len(registeredCaches))
|
||||
copy(caches, registeredCaches)
|
||||
cacheRegistryMutex.RUnlock()
|
||||
|
||||
// 并行清理所有注册的缓存
|
||||
for _, cache := range caches {
|
||||
go cache.CleanExpired()
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// 🔥 注册缓存到全局清理任务
|
||||
func registerForCleanup(cache cleanupTarget) {
|
||||
cacheRegistryMutex.Lock()
|
||||
defer cacheRegistryMutex.Unlock()
|
||||
registeredCaches = append(registeredCaches, cache)
|
||||
}
|
||||
|
||||
// 启动定期清理(修改为使用单例模式)
|
||||
func (c *ShardedMemoryCache) StartCleanupTask() {
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
for range ticker.C {
|
||||
c.CleanExpired()
|
||||
}
|
||||
}()
|
||||
registerForCleanup(c)
|
||||
startGlobalCleanupTask()
|
||||
}
|
||||
|
||||
// SetDiskCacheReference 设置磁盘缓存引用
|
||||
|
||||
Reference in New Issue
Block a user