injectlib/pansou
1
0
Fork 0
mirror of https://github.com/fish2018/pansou.git synced 2025-11-25 03:14:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

移除监控,优化GC参数

Browse Source
This commit is contained in:
www.xueximeng.com
2025-09-10 20:05:35 +08:00
parent 50c9ac9cad
commit 440f17df0f
21 changed files with 114 additions and 4558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -122,7 +122,7 @@ export ENABLED_PLUGINS=hunhepan,jikepan,panwiki,pansearch,panta,qupansou,susu,th
| CACHE_WRITE_STRATEGY | 缓存写入策略(immediate/hybrid) | `hybrid` |
| ENABLE_COMPRESSION | 是否启用压缩 | `false` |
| MIN_SIZE_TO_COMPRESS | 最小压缩阈值(字节) | `1024` |
| GC_PERCENT | Go GC触发百分比 | `100` |
| GC_PERCENT | Go GC触发百分比 | `50` |
| ASYNC_MAX_BACKGROUND_WORKERS | 最大后台工作者数量 | CPU核心数×5 |
| ASYNC_MAX_BACKGROUND_TASKS | 最大后台任务数量 | 工作者数×5 |
| ASYNC_CACHE_TTL_HOURS | 异步缓存有效期(小时) | `1` |
@@ -153,7 +153,7 @@ CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -extldflags '-sta
```
[program:pansou]
environment=PORT=8888,CHANNELS="tgsearchers3,yunpanxunlei,tianyifc,BaiduCloudDisk,txtyzy,peccxinpd,gotopan,xingqiump4,yunpanqk,PanjClub,kkxlzy,baicaoZY,MCPH01,share_aliyun,bdwpzhpd,ysxb48,jdjdn1111,yggpan,MCPH086,zaihuayun,Q66Share,NewAliPan,ypquark,Oscar_4Kmovies,ucwpzy,alyp_TV,alyp_4K_Movies,shareAliyun,alyp_1,dianyingshare,Quark_Movies,XiangxiuNBB,NewQuark,ydypzyfx,kuakeyun,ucquark,xx123pan,yingshifenxiang123,zyfb123,tyypzhpd,tianyirigeng,cloudtianyi,hdhhd21,Lsp115,oneonefivewpfx,Maidanglaocom,qixingzhenren,taoxgzy,tgsearchers115,Channel_Shares_115,tyysypzypd,vip115hot,wp123zy,yunpan139,yunpan189,yunpanuc,yydf_hzl,alyp_Animation,alyp_JLP,leoziyuan",ENABLED_PLUGINS="labi,zhizhen,shandian,duoduo,muou"
environment=PORT=8888,CHANNELS="tgsearchers3,Aliyun_4K_Movies,bdbdndn11,yunpanx,bsbdbfjfjff,yp123pan,sbsbsnsqq,yunpanxunlei,tianyifc,BaiduCloudDisk,txtyzy,peccxinpd,gotopan,PanjClub,kkxlzy,baicaoZY,MCPH01,bdwpzhpd,ysxb48,jdjdn1111,yggpan,MCPH086,zaihuayun,Q66Share,Oscar_4Kmovies,ucwpzy,shareAliyun,alyp_1,dianyingshare,Quark_Movies,XiangxiuNBB,ydypzyfx,ucquark,xx123pan,yingshifenxiang123,zyfb123,tyypzhpd,tianyirigeng,cloudtianyi,hdhhd21,Lsp115,oneonefivewpfx,qixingzhenren,taoxgzy,Channel_Shares_115,tyysypzypd,vip115hot,wp123zy,yunpan139,yunpan189,yunpanuc,yydf_hzl,leoziyuan,pikpakpan,Q_dongman,yoyokuakeduanju",ENABLED_PLUGINS="labi,zhizhen,shandian,duoduo,muou"
command=/home/work/pansou/pansou
directory=/home/work/pansou
autostart=true

4
config/config.go
View File

@@ -260,11 +260,11 @@ func getMinSizeToCompress() int {
func getGCPercent() int {
percentEnv := os.Getenv("GC_PERCENT")
if percentEnv == "" {
return 100 // 默认100%
return 50 // 默认50% - 优化内存管理，更频繁的GC避免内存暴涨
}
percent, err := strconv.Atoi(percentEnv)
if err != nil || percent <= 0 {
return 100
return 50 // 错误时也使用优化后的默认值
}
return percent
}

2
docker-compose.yml
View File

@@ -9,7 +9,7 @@ services:
- "8888:8888"
environment:
- PORT=8888
- CHANNELS=tgsearchers3,SharePanBaidu,yunpanxunlei,tianyifc,BaiduCloudDisk
- CHANNELS=tgsearchers3,Aliyun_4K_Movies,bdbdndn11,yunpanx,bsbdbfjfjff,yp123pan,sbsbsnsqq,yunpanxunlei,tianyifc,BaiduCloudDisk,txtyzy,peccxinpd,gotopan,PanjClub,kkxlzy,baicaoZY,MCPH01,bdwpzhpd,ysxb48,jdjdn1111,yggpan,MCPH086,zaihuayun,Q66Share,Oscar_4Kmovies,ucwpzy,shareAliyun,alyp_1,dianyingshare,Quark_Movies,XiangxiuNBB,ydypzyfx,ucquark,xx123pan,yingshifenxiang123,zyfb123,tyypzhpd,tianyirigeng,cloudtianyi,hdhhd21,Lsp115,oneonefivewpfx,qixingzhenren,taoxgzy,Channel_Shares_115,tyysypzypd,vip115hot,wp123zy,yunpan139,yunpan189,yunpanuc,yydf_hzl,leoziyuan,pikpakpan,Q_dongman,yoyokuakeduanju
# 必须指定启用的插件，多个插件用逗号分隔
- ENABLED_PLUGINS=labi,zhizhen,shandian,duoduo,muou,wanou
- CACHE_ENABLED=true

8
main.go
View File

@@ -90,7 +90,7 @@ func initApp() {
// 初始化HTTP客户端
util.InitHTTPClient()
// 🔥 初始化缓存写入管理器
// 初始化缓存写入管理器
var err error
globalCacheWriteManager, err = cache.NewDelayedBatchWriteManager()
if err != nil {
@@ -188,20 +188,20 @@ func startServer() {
<-quit
fmt.Println("正在关闭服务器...")
// 🔥 优先保存缓存数据到磁盘（数据安全第一）
// 优先保存缓存数据到磁盘（数据安全第一）
// 增加关闭超时时间，确保数据有足够时间保存
shutdownTimeout := 10 * time.Second
if globalCacheWriteManager != nil {
if err := globalCacheWriteManager.Shutdown(shutdownTimeout); err != nil {
log.Printf("❌ 缓存数据保存失败: %v", err)
log.Printf("缓存数据保存失败: %v", err)
}
}
// 额外确保内存缓存也被保存（双重保障）
if mainCache := service.GetEnhancedTwoLevelCache(); mainCache != nil {
if err := mainCache.FlushMemoryToDisk(); err != nil {
log.Printf("❌ 内存缓存同步失败: %v", err)
log.Printf("内存缓存同步失败: %v", err)
}
}

7
plugin/huban/huban.go
View File

@@ -176,7 +176,7 @@ func (p *HubanAsyncPlugin) searchImpl(client *http.Client, keyword string, ext m
return results, nil
} else if i == 0 {
// 第一个域名失败，记录日志但继续尝试第二个
fmt.Printf("[%s] 域名1失败，尝试域名2: %v\n", p.Name(), err)
// fmt.Printf("[%s] 域名1失败，尝试域名2: %v\n", p.Name(), err)
}
}
@@ -209,10 +209,7 @@ func (p *HubanAsyncPlugin) tryRequest(searchURL string, client *http.Client) ([]
defer resp.Body.Close()
// 解析JSON响应
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("读取响应失败: %w", err)
}
body, _ := io.ReadAll(resp.Body)
var apiResponse HubanAPIResponse
if err := json.Unmarshal(body, &apiResponse); err != nil {

2
service/cache_integration.go
View File

@@ -40,7 +40,7 @@ func NewCacheWriteIntegration(mainCache *cache.EnhancedTwoLevelCache) (*CacheWri
integration.initialized = true
fmt.Printf("✅ [缓存写入集成] 初始化完成\n")
fmt.Printf("[缓存写入集成] 初始化完成\n")
return integration, nil
}

42
service/search_service.go
View File

@@ -236,7 +236,7 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach
return
}
// 🔧 设置全局序列化器，确保异步插件与主程序使用相同的序列化格式
// 设置全局序列化器，确保异步插件与主程序使用相同的序列化格式
serializer := mainCache.GetSerializer()
if serializer != nil {
plugin.SetGlobalCacheSerializer(serializer)
@@ -244,12 +244,12 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach
// 创建缓存更新函数（支持IsFinal参数）- 接收原始数据并与现有缓存合并
cacheUpdater := func(key string, newResults []model.SearchResult, ttl time.Duration, isFinal bool, keyword string, pluginName string) error {
// 🚀 优化：如果新结果为空，跳过缓存更新（避免无效操作）
// 优化：如果新结果为空，跳过缓存更新（避免无效操作）
if len(newResults) == 0 {
return nil
}
// 🔧 获取现有缓存数据进行合并
// 获取现有缓存数据进行合并
var finalResults []model.SearchResult
if existingData, hit, err := mainCache.Get(key); err == nil && hit {
var existingResults []model.SearchResult
@@ -268,9 +268,9 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach
if config.AppConfig != nil && config.AppConfig.AsyncLogEnabled {
displayKey := key[:8] + "..."
if keyword != "" {
fmt.Printf("⚠️ [异步插件 %s] 缓存反序列化失败，使用新结果: %s(关键词:%s) | 结果数: %d\n", pluginName, displayKey, keyword, len(newResults))
fmt.Printf("[异步插件 %s] 缓存反序列化失败，使用新结果: %s(关键词:%s) | 结果数: %d\n", pluginName, displayKey, keyword, len(newResults))
} else {
fmt.Printf("⚠️ [异步插件 %s] 缓存反序列化失败，使用新结果: %s | 结果数: %d\n", pluginName, key, len(newResults))
fmt.Printf("[异步插件 %s] 缓存反序列化失败，使用新结果: %s | 结果数: %d\n", pluginName, key, len(newResults))
}
}
}
@@ -287,18 +287,13 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach
}
}
// 🔧 序列化合并后的结果
// 序列化合并后的结果
data, err := mainCache.GetSerializer().Serialize(finalResults)
if err != nil {
fmt.Printf("❌ [缓存更新] 序列化失败: %s | 错误: %v\n", key, err)
fmt.Printf("[缓存更新] 序列化失败: %s | 错误: %v\n", key, err)
return err
}
// 🔥 使用新的缓存写入管理器
// 注意：获取外部引用需要导入main包
// 为了避免循环依赖，我们暂时通过全局变量访问
// TODO: 优化架构，使用依赖注入方式
// 先更新内存缓存（立即可见）
if err := mainCache.SetMemoryOnly(key, data, ttl); err != nil {
return fmt.Errorf("内存缓存更新失败: %v", err)
@@ -942,7 +937,7 @@ func isEmpty(line string) bool {
// 将搜索结果按网盘类型分组
func mergeResultsByType(results []model.SearchResult, keyword string, cloudTypes []string) model.MergedLinks {
// 创建合并结果的映射
mergedLinks := make(model.MergedLinks, 10) // 预分配容量，假设有10种不同的网盘类型
mergedLinks := make(model.MergedLinks, 12) // 预分配容量，假设有12种不同的网盘类型
// 用于去重的映射，键为URL
uniqueLinks := make(map[string]model.MergedLink)
@@ -1123,17 +1118,6 @@ func mergeResultsByType(results []model.SearchResult, keyword string, cloudTypes
mergedLinks[linkType] = append(mergedLinks[linkType], mergedLink)
}
// 注意：不再重新排序，保持SearchResult阶段的权重排序结果
// 原来的时间排序会覆盖权重排序，现在注释掉
/*
// 对每种类型的链接按时间排序（新的在前面）
for linkType, links := range mergedLinks {
sort.Slice(links, func(i, j int) bool {
return links[i].Datetime.After(links[j].Datetime)
})
mergedLinks[linkType] = links
}
*/
// 如果指定了cloudTypes，则过滤结果
if len(cloudTypes) > 0 {
@@ -1263,7 +1247,7 @@ func (s *SearchService) searchPlugins(keyword string, plugins []string, forceRef
return results, nil
} else {
displayKey := cacheKey[:8] + "..."
fmt.Printf("❌ [主服务] 缓存反序列化失败: %s(关键词:%s) | 错误: %v\n", displayKey, keyword, err)
fmt.Printf("[主服务] 缓存反序列化失败: %s(关键词:%s) | 错误: %v\n", displayKey, keyword, err)
}
}
}
@@ -1354,7 +1338,7 @@ func (s *SearchService) searchPlugins(keyword string, plugins []string, forceRef
}
}
// 🔧 恢复主程序缓存更新：确保最终合并结果被正确缓存
// 恢复主程序缓存更新：确保最终合并结果被正确缓存
if cacheInitialized && config.AppConfig.CacheEnabled {
go func(res []model.SearchResult, kw string, key string) {
ttl := time.Duration(config.AppConfig.CacheTTLMinutes) * time.Minute
@@ -1363,12 +1347,12 @@ func (s *SearchService) searchPlugins(keyword string, plugins []string, forceRef
if enhancedTwoLevelCache != nil {
data, err := enhancedTwoLevelCache.GetSerializer().Serialize(res)
if err != nil {
fmt.Printf("❌ [主程序] 缓存序列化失败: %s | 错误: %v\n", key, err)
fmt.Printf("[主程序] 缓存序列化失败: %s | 错误: %v\n", key, err)
return
}
// 主程序最后更新，覆盖可能有问题的异步插件缓存
// 🔥 修复：使用同步方式确保数据写入磁盘
// 主程序最后更新，覆盖可能有问题的异步插件缓存
// 使用同步方式确保数据写入磁盘
enhancedTwoLevelCache.SetBothLevels(key, data, ttl)
if config.AppConfig != nil && config.AppConfig.AsyncLogEnabled {
fmt.Printf("[主程序] 缓存更新完成: %s | 结果数: %d",

884
util/cache/adaptive_tuning_engine.go vendored
View File

@@ -1,884 +0,0 @@
package cache
import (
"fmt"
"math"
"sync"
"sync/atomic"
"time"
)
// AdaptiveTuningEngine 自适应调优引擎
type AdaptiveTuningEngine struct {
// 核心组件
metricCollector *MetricCollector
performanceAnalyzer *PerformanceAnalyzer
predictiveModel *PredictiveModel
tuningStrategy *TuningStrategy
// 配置参数
config *AdaptiveTuningConfig
// 运行状态
isRunning int32
shutdownChan chan struct{}
// 调优历史
tuningHistory []*TuningRecord
historyMutex sync.RWMutex
maxHistorySize int
// 学习数据
learningData *LearningDataset
// 统计信息
stats *TuningEngineStats
mutex sync.RWMutex
}
// AdaptiveTuningConfig 自适应调优配置
type AdaptiveTuningConfig struct {
// 调优间隔
TuningInterval time.Duration
MetricInterval time.Duration
// 性能阈值
CPUUsageThreshold float64
MemoryThreshold int64
LatencyThreshold time.Duration
// 学习参数
LearningRate float64
AdaptationSpeed float64
StabilityFactor float64
// 调优范围
MinBatchInterval time.Duration
MaxBatchInterval time.Duration
MinBatchSize int
MaxBatchSize int
// 安全参数
MaxAdjustmentRatio float64 // 最大调整幅度
RollbackThreshold float64 // 回滚阈值
// 预测参数
PredictionWindow time.Duration
ConfidenceThreshold float64
}
// MetricCollector 指标收集器
type MetricCollector struct {
// 系统指标
systemMetrics *SystemMetrics
// 应用指标
applicationMetrics *ApplicationMetrics
// 缓存指标
cacheMetrics *CacheMetrics
// 历史数据
metricsHistory []MetricSnapshot
historyMutex sync.RWMutex
maxHistorySize int
// 采集状态
isCollecting int32
collectionChan chan struct{}
}
// SystemMetrics 系统指标
type SystemMetrics struct {
Timestamp time.Time
CPUUsage float64
MemoryUsage int64
MemoryTotal int64
DiskIORate float64
NetworkIORate float64
GoroutineCount int
GCPauseDuration time.Duration
HeapSize int64
AllocRate float64
}
// ApplicationMetrics 应用指标
type ApplicationMetrics struct {
Timestamp time.Time
RequestRate float64
ResponseTime time.Duration
ErrorRate float64
ThroughputMBps float64
ConcurrentUsers int
QueueDepth int
ProcessingRate float64
}
// CacheMetrics 缓存指标
type CacheMetrics struct {
Timestamp time.Time
HitRate float64
WriteRate float64
ReadRate float64
EvictionRate float64
CompressionRatio float64
StorageUsage int64
BufferUtilization float64
BatchEfficiency float64
}
// MetricSnapshot 指标快照
type MetricSnapshot struct {
Timestamp time.Time
System SystemMetrics
Application ApplicationMetrics
Cache CacheMetrics
// 综合指标
OverallPerformance float64
Efficiency float64
Stability float64
}
// PerformanceAnalyzer 性能分析器
type PerformanceAnalyzer struct {
// 分析算法
trendAnalyzer *TrendAnalyzer
anomalyDetector *AnomalyDetector
correlationAnalyzer *CorrelationAnalyzer
// 分析结果
currentTrends map[string]Trend
detectedAnomalies []Anomaly
correlations map[string]float64
mutex sync.RWMutex
}
// Trend 趋势
type Trend struct {
Metric string
Direction string // increasing, decreasing, stable
Slope float64
Confidence float64
Duration time.Duration
Prediction float64
}
// Anomaly 异常
type Anomaly struct {
Metric string
Timestamp time.Time
Severity string // low, medium, high
Value float64
ExpectedRange [2]float64
Description string
Impact float64
}
// PredictiveModel 预测模型
type PredictiveModel struct {
// 模型类型
modelType string // linear_regression, exponential_smoothing, arima
// 模型参数
coefficients []float64
seasonalFactors []float64
trendComponent float64
// 训练数据
trainingData []DataPoint
testData []DataPoint
// 模型性能
accuracy float64
rmse float64
mae float64
// 预测结果
predictions map[string]Prediction
mutex sync.RWMutex
}
// DataPoint 数据点
type DataPoint struct {
Timestamp time.Time
Values map[string]float64
Label string
}
// Prediction 预测
type Prediction struct {
Metric string
FutureValue float64
Confidence float64
TimeHorizon time.Duration
PredictedAt time.Time
ActualValue *float64 // 用于验证预测准确性
}
// TuningStrategy 调优策略
type TuningStrategy struct {
// 策略类型
strategyType string // conservative, aggressive, balanced
// 调优规则
rules []*TuningRule
// 参数调整
parameterAdjustments map[string]ParameterAdjustment
// 执行历史
executionHistory []*StrategyExecution
mutex sync.RWMutex
}
// TuningRule 调优规则
type TuningRule struct {
Name string
Condition func(*MetricSnapshot) bool
Action func(*AdaptiveTuningEngine) (*TuningDecision, error)
Priority int
Enabled bool
LastTriggered time.Time
TriggerCount int64
}
// ParameterAdjustment 参数调整
type ParameterAdjustment struct {
ParameterName string
CurrentValue interface{}
ProposedValue interface{}
AdjustmentRatio float64
Reason string
ExpectedImpact string
Risk string
}
// TuningDecision 调优决策
type TuningDecision struct {
Timestamp time.Time
Trigger string
Adjustments []ParameterAdjustment
Confidence float64
ExpectedImprovement float64
Risk float64
AutoExecute bool
}
// StrategyExecution 策略执行
type StrategyExecution struct {
Timestamp time.Time
Decision *TuningDecision
Executed bool
Result *ExecutionResult
}
// ExecutionResult 执行结果
type ExecutionResult struct {
Success bool
Error error
PerformanceBefore float64
PerformanceAfter float64
Improvement float64
SideEffects []string
}
// TuningRecord 调优记录
type TuningRecord struct {
Timestamp time.Time
Type string // automatic, manual, rollback
Parameters map[string]interface{}
Reason string
Result *TuningResult
}
// TuningResult 调优结果
type TuningResult struct {
Success bool
PerformanceGain float64
ResourceUsageChange float64
StabilityImpact float64
UserExperienceChange float64
Duration time.Duration
}
// LearningDataset 学习数据集
type LearningDataset struct {
Features [][]float64
Labels []float64
Weights []float64
// 数据统计
FeatureStats []FeatureStatistics
LabelStats LabelStatistics
// 数据划分
TrainingSplit float64
ValidationSplit float64
TestSplit float64
mutex sync.RWMutex
}
// FeatureStatistics 特征统计
type FeatureStatistics struct {
Name string
Mean float64
Std float64
Min float64
Max float64
Correlation float64
}
// LabelStatistics 标签统计
type LabelStatistics struct {
Mean float64
Std float64
Min float64
Max float64
Distribution map[string]int
}
// TuningEngineStats 调优引擎统计
type TuningEngineStats struct {
// 基础统计
TotalAdjustments int64
SuccessfulAdjustments int64
FailedAdjustments int64
RollbackCount int64
// 性能统计
AverageImprovement float64
MaxImprovement float64
TotalImprovement float64
// 学习统计
ModelAccuracy float64
PredictionAccuracy float64
LearningIterations int64
// 时间统计
AverageDecisionTime time.Duration
TotalTuningTime time.Duration
LastTuningTime time.Time
// 系统影响
CPUOverhead float64
MemoryOverhead int64
mutex sync.RWMutex
}
// NewAdaptiveTuningEngine 创建自适应调优引擎
func NewAdaptiveTuningEngine() *AdaptiveTuningEngine {
config := &AdaptiveTuningConfig{
TuningInterval: 5 * time.Minute,
MetricInterval: 30 * time.Second,
CPUUsageThreshold: 0.8,
MemoryThreshold: 500 * 1024 * 1024, // 500MB
LatencyThreshold: 10 * time.Second,
LearningRate: 0.01,
AdaptationSpeed: 0.1,
StabilityFactor: 0.9,
MinBatchInterval: 10 * time.Second,
MaxBatchInterval: 10 * time.Minute,
MinBatchSize: 10,
MaxBatchSize: 1000,
MaxAdjustmentRatio: 0.3, // 最大30%调整
RollbackThreshold: 0.1, // 性能下降10%触发回滚
PredictionWindow: 1 * time.Hour,
ConfidenceThreshold: 0.7,
}
engine := &AdaptiveTuningEngine{
config: config,
shutdownChan: make(chan struct{}),
maxHistorySize: 1000,
tuningHistory: make([]*TuningRecord, 0),
stats: &TuningEngineStats{
LastTuningTime: time.Now(),
},
}
// 初始化组件
engine.metricCollector = NewMetricCollector()
engine.performanceAnalyzer = NewPerformanceAnalyzer()
engine.predictiveModel = NewPredictiveModel()
engine.tuningStrategy = NewTuningStrategy()
engine.learningData = NewLearningDataset()
return engine
}
// Start 启动自适应调优引擎
func (a *AdaptiveTuningEngine) Start() error {
if !atomic.CompareAndSwapInt32(&a.isRunning, 0, 1) {
return fmt.Errorf("调优引擎已在运行中")
}
// 启动指标收集
if err := a.metricCollector.Start(a.config.MetricInterval); err != nil {
return fmt.Errorf("启动指标收集失败: %v", err)
}
// 启动主调优循环
go a.tuningLoop()
// 启动性能分析循环
go a.analysisLoop()
// 启动模型训练循环
go a.learningLoop()
fmt.Printf("🧠 [自适应调优引擎] 启动完成，调优间隔: %v\n", a.config.TuningInterval)
return nil
}
// tuningLoop 调优循环
func (a *AdaptiveTuningEngine) tuningLoop() {
ticker := time.NewTicker(a.config.TuningInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
a.performTuning()
case <-a.shutdownChan:
return
}
}
}
// analysisLoop 分析循环
func (a *AdaptiveTuningEngine) analysisLoop() {
ticker := time.NewTicker(a.config.MetricInterval * 2) // 分析频率低于采集频率
defer ticker.Stop()
for {
select {
case <-ticker.C:
a.performAnalysis()
case <-a.shutdownChan:
return
}
}
}
// learningLoop 学习循环
func (a *AdaptiveTuningEngine) learningLoop() {
ticker := time.NewTicker(15 * time.Minute) // 每15分钟学习一次
defer ticker.Stop()
for {
select {
case <-ticker.C:
a.performLearning()
case <-a.shutdownChan:
return
}
}
}
// performTuning 执行调优
func (a *AdaptiveTuningEngine) performTuning() {
startTime := time.Now()
// 获取当前指标
currentMetrics := a.metricCollector.GetLatestMetrics()
if currentMetrics == nil {
return
}
// 分析性能状态
performanceIssues := a.performanceAnalyzer.AnalyzeIssues(currentMetrics)
// 生成调优决策
decision := a.tuningStrategy.GenerateDecision(currentMetrics, performanceIssues)
if decision == nil {
return
}
// 验证决策合理性
if !a.validateDecision(decision) {
fmt.Printf("⚠️ [调优引擎] 决策验证失败，跳过执行\n")
return
}
// 执行调优
result := a.executeDecision(decision)
// 记录调优历史
record := &TuningRecord{
Timestamp: time.Now(),
Type: "automatic",
Parameters: a.extractParameters(decision),
Reason: decision.Trigger,
Result: result,
}
a.addTuningRecord(record)
// 更新统计
a.updateTuningStats(result, time.Since(startTime))
if result.Success {
fmt.Printf("✅ [调优引擎] 自动调优完成，性能提升: %.2f%%\n", result.PerformanceGain*100)
} else {
fmt.Printf("❌ [调优引擎] 调优失败，考虑回滚\n")
a.considerRollback(decision, result)
}
}
// performAnalysis 执行性能分析
func (a *AdaptiveTuningEngine) performAnalysis() {
// 趋势分析
a.performanceAnalyzer.AnalyzeTrends(a.metricCollector.GetMetricsHistory(100))
// 异常检测
a.performanceAnalyzer.DetectAnomalies(a.metricCollector.GetLatestMetrics())
// 相关性分析
a.performanceAnalyzer.AnalyzeCorrelations(a.metricCollector.GetMetricsHistory(50))
}
// performLearning 执行机器学习
func (a *AdaptiveTuningEngine) performLearning() {
// 收集训练数据
a.collectTrainingData()
// 训练预测模型
if err := a.predictiveModel.Train(a.learningData); err != nil {
fmt.Printf("⚠️ [调优引擎] 模型训练失败: %v\n", err)
return
}
// 验证模型性能
accuracy := a.predictiveModel.Validate()
// 更新统计
a.mutex.Lock()
a.stats.ModelAccuracy = accuracy
a.stats.LearningIterations++
a.mutex.Unlock()
fmt.Printf("🎓 [调优引擎] 模型训练完成，准确率: %.2f%%\n", accuracy*100)
}
// validateDecision 验证调优决策
func (a *AdaptiveTuningEngine) validateDecision(decision *TuningDecision) bool {
// 检查置信度
if decision.Confidence < a.config.ConfidenceThreshold {
return false
}
// 检查风险级别
if decision.Risk > 0.7 { // 风险过高
return false
}
// 检查调整幅度
for _, adj := range decision.Adjustments {
if math.Abs(adj.AdjustmentRatio) > a.config.MaxAdjustmentRatio {
return false
}
}
return true
}
// executeDecision 执行调优决策
func (a *AdaptiveTuningEngine) executeDecision(decision *TuningDecision) *TuningResult {
startTime := time.Now()
// 获取执行前性能基线
beforeMetrics := a.metricCollector.GetLatestMetrics()
performanceBefore := a.calculateOverallPerformance(beforeMetrics)
// 执行参数调整
success := true
for _, adjustment := range decision.Adjustments {
if err := a.applyParameterAdjustment(adjustment); err != nil {
success = false
break
}
}
if !success {
return &TuningResult{
Success: false,
PerformanceGain: 0,
ResourceUsageChange: 0,
StabilityImpact: 0,
Duration: time.Since(startTime),
}
}
// 等待一段时间观察效果
time.Sleep(30 * time.Second)
// 获取执行后性能
afterMetrics := a.metricCollector.GetLatestMetrics()
performanceAfter := a.calculateOverallPerformance(afterMetrics)
performanceGain := (performanceAfter - performanceBefore) / performanceBefore
// 计算资源使用变化
resourceBefore := float64(beforeMetrics.System.MemoryUsage + int64(beforeMetrics.System.CPUUsage*1000))
resourceAfter := float64(afterMetrics.System.MemoryUsage + int64(afterMetrics.System.CPUUsage*1000))
resourceChange := (resourceAfter - resourceBefore) / resourceBefore
return &TuningResult{
Success: true,
PerformanceGain: performanceGain,
ResourceUsageChange: resourceChange,
StabilityImpact: a.calculateStabilityImpact(beforeMetrics, afterMetrics),
UserExperienceChange: performanceGain, // 简化假设
Duration: time.Since(startTime),
}
}
// calculateOverallPerformance 计算整体性能分数
func (a *AdaptiveTuningEngine) calculateOverallPerformance(metrics *MetricSnapshot) float64 {
if metrics == nil {
return 0
}
// 性能分数计算（0-100分）
cpuScore := math.Max(0, (1.0-metrics.System.CPUUsage)*40) // CPU使用率越低越好，最高40分
memoryScore := math.Max(0, (1.0-float64(metrics.System.MemoryUsage)/float64(metrics.System.MemoryTotal))*30) // 内存使用率越低越好，最高30分
responseScore := math.Max(0, (1.0-math.Min(1.0, float64(metrics.Application.ResponseTime)/float64(time.Second)))*20) // 响应时间越短越好，最高20分
cacheScore := metrics.Cache.HitRate * 10 // 缓存命中率越高越好，最高10分
return cpuScore + memoryScore + responseScore + cacheScore
}
// calculateStabilityImpact 计算稳定性影响
func (a *AdaptiveTuningEngine) calculateStabilityImpact(before, after *MetricSnapshot) float64 {
if before == nil || after == nil {
return 0
}
// 简化的稳定性计算：比较关键指标的变化
cpuVariation := math.Abs(after.System.CPUUsage - before.System.CPUUsage)
memoryVariation := math.Abs(float64(after.System.MemoryUsage-before.System.MemoryUsage) / float64(before.System.MemoryUsage))
// 变化越小，稳定性越好
stabilityScore := 1.0 - (cpuVariation*0.5 + memoryVariation*0.5)
return math.Max(0, stabilityScore)
}
// applyParameterAdjustment 应用参数调整
func (a *AdaptiveTuningEngine) applyParameterAdjustment(adjustment ParameterAdjustment) error {
// 这里应该调用具体的参数设置函数
// 暂时模拟实现
fmt.Printf("🔧 [调优引擎] 调整参数 %s: %v -> %v (%.1f%%)\n",
adjustment.ParameterName,
adjustment.CurrentValue,
adjustment.ProposedValue,
adjustment.AdjustmentRatio*100)
return nil
}
// collectTrainingData 收集训练数据
func (a *AdaptiveTuningEngine) collectTrainingData() {
history := a.metricCollector.GetMetricsHistory(200)
_ = a.getTuningHistory(50) // 暂时不使用调优历史
// 构建特征和标签
for i, metrics := range history {
if i < len(history)-1 {
// 特征：当前指标
features := []float64{
metrics.System.CPUUsage,
float64(metrics.System.MemoryUsage) / 1024 / 1024, // MB
float64(metrics.Application.ResponseTime) / float64(time.Millisecond),
metrics.Cache.HitRate,
metrics.Cache.CompressionRatio,
}
// 标签：下一时刻的整体性能
nextMetrics := history[i+1]
label := a.calculateOverallPerformance(&nextMetrics)
// 添加到学习数据集
a.learningData.mutex.Lock()
a.learningData.Features = append(a.learningData.Features, features)
a.learningData.Labels = append(a.learningData.Labels, label)
a.learningData.Weights = append(a.learningData.Weights, 1.0)
a.learningData.mutex.Unlock()
}
}
// 限制数据集大小
a.learningData.mutex.Lock()
maxSize := 1000
if len(a.learningData.Features) > maxSize {
excess := len(a.learningData.Features) - maxSize
a.learningData.Features = a.learningData.Features[excess:]
a.learningData.Labels = a.learningData.Labels[excess:]
a.learningData.Weights = a.learningData.Weights[excess:]
}
a.learningData.mutex.Unlock()
}
// considerRollback 考虑回滚
func (a *AdaptiveTuningEngine) considerRollback(decision *TuningDecision, result *TuningResult) {
if result.PerformanceGain < -a.config.RollbackThreshold {
fmt.Printf("🔄 [调优引擎] 触发自动回滚，性能下降: %.2f%%\n", result.PerformanceGain*100)
a.performRollback(decision)
}
}
// performRollback 执行回滚
func (a *AdaptiveTuningEngine) performRollback(originalDecision *TuningDecision) {
// 创建回滚决策
rollbackDecision := &TuningDecision{
Timestamp: time.Now(),
Trigger: "automatic_rollback",
Adjustments: make([]ParameterAdjustment, 0),
Confidence: 1.0,
AutoExecute: true,
}
// 反向调整所有参数
for _, adjustment := range originalDecision.Adjustments {
rollbackAdjustment := ParameterAdjustment{
ParameterName: adjustment.ParameterName,
CurrentValue: adjustment.ProposedValue,
ProposedValue: adjustment.CurrentValue,
AdjustmentRatio: -adjustment.AdjustmentRatio,
Reason: "rollback",
ExpectedImpact: "restore_stability",
Risk: "low",
}
rollbackDecision.Adjustments = append(rollbackDecision.Adjustments, rollbackAdjustment)
}
// 执行回滚
result := a.executeDecision(rollbackDecision)
// 记录回滚
record := &TuningRecord{
Timestamp: time.Now(),
Type: "rollback",
Parameters: a.extractParameters(rollbackDecision),
Reason: "performance_degradation",
Result: result,
}
a.addTuningRecord(record)
// 更新统计
atomic.AddInt64(&a.stats.RollbackCount, 1)
}
// addTuningRecord 添加调优记录
func (a *AdaptiveTuningEngine) addTuningRecord(record *TuningRecord) {
a.historyMutex.Lock()
defer a.historyMutex.Unlock()
a.tuningHistory = append(a.tuningHistory, record)
// 限制历史记录大小
if len(a.tuningHistory) > a.maxHistorySize {
a.tuningHistory = a.tuningHistory[1:]
}
}
// updateTuningStats 更新调优统计
func (a *AdaptiveTuningEngine) updateTuningStats(result *TuningResult, decisionTime time.Duration) {
a.stats.mutex.Lock()
defer a.stats.mutex.Unlock()
a.stats.TotalAdjustments++
if result.Success {
a.stats.SuccessfulAdjustments++
a.stats.TotalImprovement += result.PerformanceGain
a.stats.AverageImprovement = a.stats.TotalImprovement / float64(a.stats.SuccessfulAdjustments)
if result.PerformanceGain > a.stats.MaxImprovement {
a.stats.MaxImprovement = result.PerformanceGain
}
} else {
a.stats.FailedAdjustments++
}
// 更新时间统计
a.stats.TotalTuningTime += decisionTime
a.stats.AverageDecisionTime = time.Duration(int64(a.stats.TotalTuningTime) / a.stats.TotalAdjustments)
a.stats.LastTuningTime = time.Now()
}
// extractParameters 提取决策参数
func (a *AdaptiveTuningEngine) extractParameters(decision *TuningDecision) map[string]interface{} {
params := make(map[string]interface{})
for _, adj := range decision.Adjustments {
params[adj.ParameterName] = adj.ProposedValue
}
return params
}
// getTuningHistory 获取调优历史
func (a *AdaptiveTuningEngine) getTuningHistory(limit int) []*TuningRecord {
a.historyMutex.RLock()
defer a.historyMutex.RUnlock()
if limit <= 0 || limit > len(a.tuningHistory) {
limit = len(a.tuningHistory)
}
history := make([]*TuningRecord, limit)
startIndex := len(a.tuningHistory) - limit
copy(history, a.tuningHistory[startIndex:])
return history
}
// Stop 停止自适应调优引擎
func (a *AdaptiveTuningEngine) Stop() error {
if !atomic.CompareAndSwapInt32(&a.isRunning, 1, 0) {
return nil
}
// 停止指标收集
a.metricCollector.Stop()
// 停止所有循环
close(a.shutdownChan)
fmt.Printf("🧠 [自适应调优引擎] 已停止\n")
return nil
}
// GetStats 获取调优引擎统计
func (a *AdaptiveTuningEngine) GetStats() *TuningEngineStats {
a.stats.mutex.RLock()
defer a.stats.mutex.RUnlock()
statsCopy := *a.stats
return &statsCopy
}
// GetTuningReport 获取调优报告
func (a *AdaptiveTuningEngine) GetTuningReport() map[string]interface{} {
stats := a.GetStats()
recentHistory := a.getTuningHistory(10)
return map[string]interface{}{
"engine_stats": stats,
"recent_history": recentHistory,
"current_trends": a.performanceAnalyzer.GetCurrentTrends(),
"anomalies": a.performanceAnalyzer.GetDetectedAnomalies(),
"predictions": a.predictiveModel.GetPredictions(),
"model_accuracy": a.predictiveModel.GetAccuracy(),
}
}

620
util/cache/advanced_data_merger.go vendored
View File

@@ -1,620 +0,0 @@
package cache
import (
"fmt"
"sync"
"time"
"pansou/model"
)
// AdvancedDataMerger 高级数据合并器
type AdvancedDataMerger struct {
// 合并策略
mergeStrategies map[string]MergeStrategy
// 合并规则
mergeRules []*MergeRule
// 统计信息
totalMerges int64
successfulMerges int64
failedMerges int64
// 缓存去重
deduplicationMap map[string]*CacheOperation
dedupMutex sync.RWMutex
// 性能监控
mergeMetrics *MergeMetrics
mutex sync.RWMutex
}
// MergeStrategy 合并策略接口
type MergeStrategy interface {
CanMerge(existing *CacheOperation, new *CacheOperation) bool
Merge(existing *CacheOperation, new *CacheOperation) (*CacheOperation, error)
GetPriority() int
}
// MergeRule 合并规则
type MergeRule struct {
Name string
Description string
Condition func(*CacheOperation, *CacheOperation) bool
MergeFunc func(*CacheOperation, *CacheOperation) (*CacheOperation, error)
Priority int
Enabled bool
}
// MergeMetrics 合并指标
type MergeMetrics struct {
// 时间统计
AverageMergeTime time.Duration
MaxMergeTime time.Duration
TotalMergeTime time.Duration
// 数据统计
DataSizeBefore int64
DataSizeAfter int64
CompressionRatio float64
// 类型统计
MergesByType map[string]int64
MergesByPlugin map[string]int64
MergesByKeyword map[string]int64
// 效率统计
DuplicatesRemoved int64
ResultsConsolidated int64
StorageSaved int64
}
// NewAdvancedDataMerger 创建高级数据合并器
func NewAdvancedDataMerger() *AdvancedDataMerger {
merger := &AdvancedDataMerger{
mergeStrategies: make(map[string]MergeStrategy),
deduplicationMap: make(map[string]*CacheOperation),
mergeMetrics: &MergeMetrics{
MergesByType: make(map[string]int64),
MergesByPlugin: make(map[string]int64),
MergesByKeyword: make(map[string]int64),
},
}
// 初始化合并策略
merger.initializeMergeStrategies()
// 初始化合并规则
merger.initializeMergeRules()
return merger
}
// initializeMergeStrategies 初始化合并策略
func (m *AdvancedDataMerger) initializeMergeStrategies() {
// 注册同键合并策略
m.mergeStrategies["same_key"] = &SameKeyMergeStrategy{}
// 注册同插件同关键词策略
m.mergeStrategies["same_plugin_keyword"] = &SamePluginKeywordMergeStrategy{}
// 注册结果去重策略
m.mergeStrategies["deduplication"] = &DeduplicationMergeStrategy{}
// 注册内容相似性策略
m.mergeStrategies["content_similarity"] = &ContentSimilarityMergeStrategy{}
}
// initializeMergeRules 初始化合并规则
func (m *AdvancedDataMerger) initializeMergeRules() {
m.mergeRules = []*MergeRule{
{
Name: "完全相同键合并",
Description: "合并具有完全相同缓存键的操作",
Condition: func(existing, new *CacheOperation) bool {
return existing.Key == new.Key
},
MergeFunc: m.mergeSameKey,
Priority: 1,
Enabled: true,
},
{
Name: "同插件同关键词合并",
Description: "合并同一插件对同一关键词的搜索结果",
Condition: func(existing, new *CacheOperation) bool {
return existing.PluginName == new.PluginName &&
existing.Keyword == new.Keyword &&
existing.Key != new.Key
},
MergeFunc: m.mergeSamePluginKeyword,
Priority: 2,
Enabled: true,
},
{
Name: "时间窗口内合并",
Description: "合并时间窗口内的相似操作",
Condition: func(existing, new *CacheOperation) bool {
timeDiff := new.Timestamp.Sub(existing.Timestamp)
return timeDiff >= 0 && timeDiff <= 5*time.Minute &&
existing.PluginName == new.PluginName
},
MergeFunc: m.mergeTimeWindow,
Priority: 3,
Enabled: true,
},
{
Name: "结果去重合并",
Description: "去除重复的搜索结果",
Condition: func(existing, new *CacheOperation) bool {
return m.hasOverlapResults(existing, new)
},
MergeFunc: m.mergeDeduplication,
Priority: 4,
Enabled: true,
},
}
}
// TryMergeOperation 尝试合并操作
func (m *AdvancedDataMerger) TryMergeOperation(buffer *GlobalBuffer, newOp *CacheOperation) bool {
startTime := time.Now()
defer func() {
mergeTime := time.Since(startTime)
m.updateMergeMetrics(mergeTime)
}()
m.totalMerges++
// 🔍 在缓冲区中寻找可合并的操作
merged := false
for i, existingOp := range buffer.Operations {
if m.canMergeOperations(existingOp, newOp) {
// 🚀 执行合并
mergedOp, err := m.performMerge(existingOp, newOp)
if err != nil {
m.failedMerges++
continue
}
// 替换原操作
buffer.Operations[i] = mergedOp
// 更新统计
m.successfulMerges++
m.updateMergeStatistics(existingOp, newOp, mergedOp)
merged = true
break
}
}
return merged
}
// canMergeOperations 检查是否可以合并操作
func (m *AdvancedDataMerger) canMergeOperations(existing, new *CacheOperation) bool {
// 按优先级检查合并规则
for _, rule := range m.mergeRules {
if rule.Enabled && rule.Condition(existing, new) {
return true
}
}
return false
}
// performMerge 执行合并
func (m *AdvancedDataMerger) performMerge(existing, new *CacheOperation) (*CacheOperation, error) {
// 找到最高优先级的适用规则
var bestRule *MergeRule
for _, rule := range m.mergeRules {
if rule.Enabled && rule.Condition(existing, new) {
if bestRule == nil || rule.Priority < bestRule.Priority {
bestRule = rule
}
}
}
if bestRule == nil {
return nil, fmt.Errorf("未找到适用的合并规则")
}
// 执行合并
return bestRule.MergeFunc(existing, new)
}
// mergeSameKey 合并相同键的操作
func (m *AdvancedDataMerger) mergeSameKey(existing, new *CacheOperation) (*CacheOperation, error) {
// 合并搜索结果
mergedResults := m.mergeSearchResults(existing.Data, new.Data)
merged := &CacheOperation{
Key: existing.Key,
Data: mergedResults,
TTL: m.chooseLongerTTL(existing.TTL, new.TTL),
PluginName: existing.PluginName, // 保持原插件名
Keyword: existing.Keyword, // 保持原关键词
Timestamp: new.Timestamp, // 使用最新时间戳
Priority: m.chooseBetterPriority(existing.Priority, new.Priority),
DataSize: existing.DataSize + new.DataSize, // 累计数据大小
IsFinal: existing.IsFinal || new.IsFinal, // 任一为最终结果则为最终结果
}
return merged, nil
}
// mergeSamePluginKeyword 合并同插件同关键词操作
func (m *AdvancedDataMerger) mergeSamePluginKeyword(existing, new *CacheOperation) (*CacheOperation, error) {
// 生成新的合并键
mergedKey := fmt.Sprintf("merged_%s_%s_%d",
existing.PluginName, existing.Keyword, time.Now().Unix())
// 合并搜索结果
mergedResults := m.mergeSearchResults(existing.Data, new.Data)
merged := &CacheOperation{
Key: mergedKey,
Data: mergedResults,
TTL: m.chooseLongerTTL(existing.TTL, new.TTL),
PluginName: existing.PluginName,
Keyword: existing.Keyword,
Timestamp: new.Timestamp,
Priority: m.chooseBetterPriority(existing.Priority, new.Priority),
DataSize: len(mergedResults) * 500, // 重新估算数据大小
IsFinal: existing.IsFinal || new.IsFinal,
}
return merged, nil
}
// mergeTimeWindow 合并时间窗口内的操作
func (m *AdvancedDataMerger) mergeTimeWindow(existing, new *CacheOperation) (*CacheOperation, error) {
// 时间窗口合并策略：保留最新的元信息，合并数据
mergedResults := m.mergeSearchResults(existing.Data, new.Data)
merged := &CacheOperation{
Key: new.Key, // 使用新的键
Data: mergedResults,
TTL: new.TTL, // 使用新的TTL
PluginName: new.PluginName,
Keyword: new.Keyword,
Timestamp: new.Timestamp,
Priority: new.Priority,
DataSize: len(mergedResults) * 500,
IsFinal: new.IsFinal,
}
return merged, nil
}
// mergeDeduplication 去重合并
func (m *AdvancedDataMerger) mergeDeduplication(existing, new *CacheOperation) (*CacheOperation, error) {
// 执行深度去重
deduplicatedResults := m.deduplicateSearchResults(existing.Data, new.Data)
merged := &CacheOperation{
Key: existing.Key,
Data: deduplicatedResults,
TTL: m.chooseLongerTTL(existing.TTL, new.TTL),
PluginName: existing.PluginName,
Keyword: existing.Keyword,
Timestamp: new.Timestamp,
Priority: m.chooseBetterPriority(existing.Priority, new.Priority),
DataSize: len(deduplicatedResults) * 500,
IsFinal: existing.IsFinal || new.IsFinal,
}
return merged, nil
}
// mergeSearchResults 合并搜索结果
func (m *AdvancedDataMerger) mergeSearchResults(existing, new []model.SearchResult) []model.SearchResult {
// 使用map去重
resultMap := make(map[string]model.SearchResult)
// 添加现有结果
for _, result := range existing {
key := m.generateResultKey(result)
resultMap[key] = result
}
// 添加新结果，自动去重
for _, result := range new {
key := m.generateResultKey(result)
if existingResult, exists := resultMap[key]; exists {
// 合并相同结果的信息
mergedResult := m.mergeIndividualResults(existingResult, result)
resultMap[key] = mergedResult
} else {
resultMap[key] = result
}
}
// 转换回切片
merged := make([]model.SearchResult, 0, len(resultMap))
for _, result := range resultMap {
merged = append(merged, result)
}
return merged
}
// deduplicateSearchResults 深度去重搜索结果
func (m *AdvancedDataMerger) deduplicateSearchResults(existing, new []model.SearchResult) []model.SearchResult {
// 更严格的去重逻辑
resultMap := make(map[string]model.SearchResult)
duplicateCount := 0
// 处理现有结果
for _, result := range existing {
key := m.generateResultKey(result)
resultMap[key] = result
}
// 处理新结果
for _, result := range new {
key := m.generateResultKey(result)
if _, exists := resultMap[key]; !exists {
resultMap[key] = result
} else {
duplicateCount++
}
}
// 更新去重统计
m.mergeMetrics.DuplicatesRemoved += int64(duplicateCount)
// 转换回切片
deduplicated := make([]model.SearchResult, 0, len(resultMap))
for _, result := range resultMap {
deduplicated = append(deduplicated, result)
}
return deduplicated
}
// generateResultKey 生成结果键用于去重
func (m *AdvancedDataMerger) generateResultKey(result model.SearchResult) string {
// 使用标题和主要链接生成唯一键
key := result.Title
if len(result.Links) > 0 {
key += "_" + result.Links[0].URL
}
return key
}
// mergeIndividualResults 合并单个结果
func (m *AdvancedDataMerger) mergeIndividualResults(existing, new model.SearchResult) model.SearchResult {
merged := existing
// 选择更完整的内容
if len(new.Content) > len(existing.Content) {
merged.Content = new.Content
}
// 合并链接
linkMap := make(map[string]model.Link)
for _, link := range existing.Links {
linkMap[link.URL] = link
}
for _, link := range new.Links {
linkMap[link.URL] = link
}
links := make([]model.Link, 0, len(linkMap))
for _, link := range linkMap {
links = append(links, link)
}
merged.Links = links
// 合并标签
tagMap := make(map[string]bool)
for _, tag := range existing.Tags {
tagMap[tag] = true
}
for _, tag := range new.Tags {
tagMap[tag] = true
}
tags := make([]string, 0, len(tagMap))
for tag := range tagMap {
tags = append(tags, tag)
}
merged.Tags = tags
// 使用更新的时间
if new.Datetime.After(existing.Datetime) {
merged.Datetime = new.Datetime
}
return merged
}
// hasOverlapResults 检查是否有重叠结果
func (m *AdvancedDataMerger) hasOverlapResults(existing, new *CacheOperation) bool {
if len(existing.Data) == 0 || len(new.Data) == 0 {
return false
}
// 简单重叠检测：检查前几个结果的标题
checkCount := 3
if len(existing.Data) < checkCount {
checkCount = len(existing.Data)
}
if len(new.Data) < checkCount {
checkCount = len(new.Data)
}
for i := 0; i < checkCount; i++ {
for j := 0; j < checkCount; j++ {
if existing.Data[i].Title == new.Data[j].Title {
return true
}
}
}
return false
}
// chooseLongerTTL 选择更长的TTL
func (m *AdvancedDataMerger) chooseLongerTTL(ttl1, ttl2 time.Duration) time.Duration {
if ttl1 > ttl2 {
return ttl1
}
return ttl2
}
// chooseBetterPriority 选择更好的优先级
func (m *AdvancedDataMerger) chooseBetterPriority(priority1, priority2 int) int {
if priority1 < priority2 { // 数字越小优先级越高
return priority1
}
return priority2
}
// updateMergeMetrics 更新合并指标
func (m *AdvancedDataMerger) updateMergeMetrics(mergeTime time.Duration) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.mergeMetrics.TotalMergeTime += mergeTime
// 更新平均时间
if m.successfulMerges > 0 {
m.mergeMetrics.AverageMergeTime = time.Duration(
int64(m.mergeMetrics.TotalMergeTime) / m.successfulMerges)
}
// 更新最大时间
if mergeTime > m.mergeMetrics.MaxMergeTime {
m.mergeMetrics.MaxMergeTime = mergeTime
}
}
// updateMergeStatistics 更新合并统计
func (m *AdvancedDataMerger) updateMergeStatistics(existing, new, merged *CacheOperation) {
m.mutex.Lock()
defer m.mutex.Unlock()
// 数据大小统计
beforeSize := int64(existing.DataSize + new.DataSize)
afterSize := int64(merged.DataSize)
m.mergeMetrics.DataSizeBefore += beforeSize
m.mergeMetrics.DataSizeAfter += afterSize
// 计算压缩比例
if m.mergeMetrics.DataSizeBefore > 0 {
m.mergeMetrics.CompressionRatio = float64(m.mergeMetrics.DataSizeAfter) /
float64(m.mergeMetrics.DataSizeBefore)
}
// 按类型统计
m.mergeMetrics.MergesByPlugin[merged.PluginName]++
m.mergeMetrics.MergesByKeyword[merged.Keyword]++
// 结果整合统计
originalCount := int64(len(existing.Data) + len(new.Data))
mergedCount := int64(len(merged.Data))
consolidated := originalCount - mergedCount
if consolidated > 0 {
m.mergeMetrics.ResultsConsolidated += consolidated
m.mergeMetrics.StorageSaved += beforeSize - afterSize
}
}
// GetMergeStats 获取合并统计
func (m *AdvancedDataMerger) GetMergeStats() map[string]interface{} {
m.mutex.RLock()
defer m.mutex.RUnlock()
successRate := float64(0)
if m.totalMerges > 0 {
successRate = float64(m.successfulMerges) / float64(m.totalMerges)
}
return map[string]interface{}{
"total_merges": m.totalMerges,
"successful_merges": m.successfulMerges,
"failed_merges": m.failedMerges,
"success_rate": successRate,
"merge_metrics": m.mergeMetrics,
"average_merge_time": m.mergeMetrics.AverageMergeTime,
"max_merge_time": m.mergeMetrics.MaxMergeTime,
"compression_ratio": m.mergeMetrics.CompressionRatio,
"duplicates_removed": m.mergeMetrics.DuplicatesRemoved,
"results_consolidated": m.mergeMetrics.ResultsConsolidated,
"storage_saved": m.mergeMetrics.StorageSaved,
}
}
// 实现各种合并策略
// SameKeyMergeStrategy 相同键合并策略
type SameKeyMergeStrategy struct{}
func (s *SameKeyMergeStrategy) CanMerge(existing, new *CacheOperation) bool {
return existing.Key == new.Key
}
func (s *SameKeyMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) {
// 委托给合并器的方法
return nil, fmt.Errorf("应该使用合并器的方法")
}
func (s *SameKeyMergeStrategy) GetPriority() int {
return 1
}
// SamePluginKeywordMergeStrategy 同插件同关键词合并策略
type SamePluginKeywordMergeStrategy struct{}
func (s *SamePluginKeywordMergeStrategy) CanMerge(existing, new *CacheOperation) bool {
return existing.PluginName == new.PluginName && existing.Keyword == new.Keyword
}
func (s *SamePluginKeywordMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) {
return nil, fmt.Errorf("应该使用合并器的方法")
}
func (s *SamePluginKeywordMergeStrategy) GetPriority() int {
return 2
}
// DeduplicationMergeStrategy 去重合并策略
type DeduplicationMergeStrategy struct{}
func (s *DeduplicationMergeStrategy) CanMerge(existing, new *CacheOperation) bool {
// 检查是否有重复结果
return len(existing.Data) > 0 && len(new.Data) > 0
}
func (s *DeduplicationMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) {
return nil, fmt.Errorf("应该使用合并器的方法")
}
func (s *DeduplicationMergeStrategy) GetPriority() int {
return 4
}
// ContentSimilarityMergeStrategy 内容相似性合并策略
type ContentSimilarityMergeStrategy struct{}
func (s *ContentSimilarityMergeStrategy) CanMerge(existing, new *CacheOperation) bool {
// 简单的相似性检测：关键词相似度
return existing.Keyword == new.Keyword ||
(len(existing.Keyword) > 3 && len(new.Keyword) > 3 &&
existing.Keyword[:3] == new.Keyword[:3])
}
func (s *ContentSimilarityMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) {
return nil, fmt.Errorf("应该使用合并器的方法")
}
func (s *ContentSimilarityMergeStrategy) GetPriority() int {
return 5
}

905
util/cache/buffer_status_monitor.go vendored
View File

@@ -1,905 +0,0 @@
package cache
import (
"fmt"
"runtime"
"sync"
"sync/atomic"
"time"
"pansou/util/json"
)
// BufferStatusMonitor 缓冲区状态监控器
type BufferStatusMonitor struct {
// 监控配置
monitorInterval time.Duration
alertThresholds *AlertThresholds
// 监控状态
isMonitoring int32
shutdownChan chan struct{}
// 健康检查
healthChecker *HealthChecker
// 报警系统
alertManager *AlertManager
// 性能指标
performanceMetrics *PerformanceMetrics
// 监控数据
monitoringData *MonitoringData
dataMutex sync.RWMutex
// 历史记录
historyBuffer []MonitorSnapshot
historyMutex sync.Mutex
maxHistorySize int
}
// AlertThresholds 报警阈值
type AlertThresholds struct {
// 内存阈值
MemoryUsageWarning int64 // 内存使用警告阈值（字节）
MemoryUsageCritical int64 // 内存使用严重阈值（字节）
// 缓冲区阈值
BufferCountWarning int // 缓冲区数量警告阈值
BufferCountCritical int // 缓冲区数量严重阈值
// 操作阈值
OperationQueueWarning int // 操作队列警告阈值
OperationQueueCritical int // 操作队列严重阈值
// 时间阈值
ProcessTimeWarning time.Duration // 处理时间警告阈值
ProcessTimeCritical time.Duration // 处理时间严重阈值
// 成功率阈值
SuccessRateWarning float64 // 成功率警告阈值
SuccessRateCritical float64 // 成功率严重阈值
}
// HealthChecker 健康检查器
type HealthChecker struct {
lastHealthCheck time.Time
healthCheckInterval time.Duration
healthStatus HealthStatus
healthHistory []HealthCheckResult
mutex sync.RWMutex
}
// HealthStatus 健康状态
type HealthStatus struct {
Overall string `json:"overall"` // healthy, warning, critical
LastCheck time.Time `json:"last_check"`
Components map[string]ComponentHealth `json:"components"`
Issues []HealthIssue `json:"issues,omitempty"`
}
// ComponentHealth 组件健康状态
type ComponentHealth struct {
Status string `json:"status"`
LastCheck time.Time `json:"last_check"`
Metrics map[string]interface{} `json:"metrics"`
Message string `json:"message,omitempty"`
}
// HealthIssue 健康问题
type HealthIssue struct {
Component string `json:"component"`
Severity string `json:"severity"` // warning, critical
Message string `json:"message"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
Count int `json:"count"`
}
// HealthCheckResult 健康检查结果
type HealthCheckResult struct {
Timestamp time.Time `json:"timestamp"`
Status string `json:"status"`
CheckTime time.Duration `json:"check_time"`
Issues []HealthIssue `json:"issues"`
}
// AlertManager 报警管理器
type AlertManager struct {
alerts []Alert
alertHistory []Alert
mutex sync.RWMutex
maxAlertHistory int
// 报警配置
alertCooldown map[string]time.Time // 报警冷却时间
cooldownPeriod time.Duration // 冷却期间
}
// Alert 报警
type Alert struct {
ID string `json:"id"`
Level string `json:"level"` // info, warning, critical
Component string `json:"component"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
Resolved bool `json:"resolved"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
}
// PerformanceMetrics 性能指标
type PerformanceMetrics struct {
// CPU指标
CPUUsage float64 `json:"cpu_usage"`
CPUHistory []float64 `json:"cpu_history"`
// 内存指标
MemoryUsage int64 `json:"memory_usage"`
MemoryHistory []int64 `json:"memory_history"`
GCStats runtime.MemStats `json:"gc_stats"`
// 吞吐量指标
OperationsPerSecond float64 `json:"operations_per_second"`
ThroughputHistory []float64 `json:"throughput_history"`
// 延迟指标
AverageLatency time.Duration `json:"average_latency"`
P95Latency time.Duration `json:"p95_latency"`
P99Latency time.Duration `json:"p99_latency"`
LatencyHistory []time.Duration `json:"latency_history"`
// 错误率指标
ErrorRate float64 `json:"error_rate"`
ErrorHistory []float64 `json:"error_history"`
// 资源利用率
DiskIORate float64 `json:"disk_io_rate"`
NetworkIORate float64 `json:"network_io_rate"`
// 更新时间
LastUpdated time.Time `json:"last_updated"`
}
// MonitoringData 监控数据
type MonitoringData struct {
// 系统状态
SystemHealth HealthStatus `json:"system_health"`
PerformanceMetrics PerformanceMetrics `json:"performance_metrics"`
// 缓冲区状态
BufferStates map[string]BufferState `json:"buffer_states"`
GlobalBufferStats *GlobalBufferStats `json:"global_buffer_stats"`
// 实时统计
RealTimeStats RealTimeStats `json:"real_time_stats"`
// 趋势分析
TrendAnalysis TrendAnalysis `json:"trend_analysis"`
// 预测数据
Predictions PredictionData `json:"predictions"`
}
// BufferState 缓冲区状态
type BufferState struct {
ID string `json:"id"`
Size int `json:"size"`
Capacity int `json:"capacity"`
UtilizationRate float64 `json:"utilization_rate"`
LastActivity time.Time `json:"last_activity"`
OperationsPerMin float64 `json:"operations_per_min"`
AverageDataSize int64 `json:"average_data_size"`
CompressionRatio float64 `json:"compression_ratio"`
Health string `json:"health"`
}
// RealTimeStats 实时统计
type RealTimeStats struct {
ActiveOperations int `json:"active_operations"`
QueuedOperations int `json:"queued_operations"`
ProcessingRate float64 `json:"processing_rate"`
ThroughputMBps float64 `json:"throughput_mbps"`
CacheHitRate float64 `json:"cache_hit_rate"`
CompressionRatio float64 `json:"compression_ratio"`
ErrorRate float64 `json:"error_rate"`
LastUpdated time.Time `json:"last_updated"`
}
// TrendAnalysis 趋势分析
type TrendAnalysis struct {
MemoryTrend string `json:"memory_trend"` // increasing, decreasing, stable
ThroughputTrend string `json:"throughput_trend"`
ErrorRateTrend string `json:"error_rate_trend"`
BufferUsageTrend string `json:"buffer_usage_trend"`
AnalysisTime time.Time `json:"analysis_time"`
Confidence float64 `json:"confidence"`
}
// PredictionData 预测数据
type PredictionData struct {
MemoryUsageIn1Hour int64 `json:"memory_usage_in_1hour"`
MemoryUsageIn24Hours int64 `json:"memory_usage_in_24hours"`
BufferOverflowRisk float64 `json:"buffer_overflow_risk"`
SystemLoadPrediction float64 `json:"system_load_prediction"`
RecommendedActions []string `json:"recommended_actions"`
ConfidenceLevel float64 `json:"confidence_level"`
PredictionTime time.Time `json:"prediction_time"`
}
// MonitorSnapshot 监控快照
type MonitorSnapshot struct {
Timestamp time.Time `json:"timestamp"`
SystemHealth HealthStatus `json:"system_health"`
BufferCount int `json:"buffer_count"`
TotalMemoryUsage int64 `json:"total_memory_usage"`
OperationsPerSecond float64 `json:"operations_per_second"`
ErrorRate float64 `json:"error_rate"`
CacheHitRate float64 `json:"cache_hit_rate"`
}
// NewBufferStatusMonitor 创建缓冲区状态监控器
func NewBufferStatusMonitor() *BufferStatusMonitor {
monitor := &BufferStatusMonitor{
monitorInterval: 30 * time.Second, // 30秒监控间隔
shutdownChan: make(chan struct{}),
maxHistorySize: 288, // 保存24小时历史（每30秒一个，24*60*2=2880，简化为288）
alertThresholds: &AlertThresholds{
MemoryUsageWarning: 50 * 1024 * 1024, // 50MB
MemoryUsageCritical: 100 * 1024 * 1024, // 100MB
BufferCountWarning: 40,
BufferCountCritical: 50,
OperationQueueWarning: 500,
OperationQueueCritical: 1000,
ProcessTimeWarning: 5 * time.Second,
ProcessTimeCritical: 15 * time.Second,
SuccessRateWarning: 0.95, // 95%
SuccessRateCritical: 0.90, // 90%
},
monitoringData: &MonitoringData{
BufferStates: make(map[string]BufferState),
RealTimeStats: RealTimeStats{},
TrendAnalysis: TrendAnalysis{},
Predictions: PredictionData{},
},
}
// 初始化组件
monitor.healthChecker = &HealthChecker{
healthCheckInterval: 1 * time.Minute,
healthStatus: HealthStatus{
Overall: "healthy",
Components: make(map[string]ComponentHealth),
Issues: make([]HealthIssue, 0),
},
healthHistory: make([]HealthCheckResult, 0),
}
monitor.alertManager = &AlertManager{
alerts: make([]Alert, 0),
alertHistory: make([]Alert, 0),
maxAlertHistory: 1000,
alertCooldown: make(map[string]time.Time),
cooldownPeriod: 5 * time.Minute, // 5分钟冷却期
}
monitor.performanceMetrics = &PerformanceMetrics{
CPUHistory: make([]float64, 0),
MemoryHistory: make([]int64, 0),
ThroughputHistory: make([]float64, 0),
LatencyHistory: make([]time.Duration, 0),
ErrorHistory: make([]float64, 0),
}
return monitor
}
// Start 启动监控器
func (b *BufferStatusMonitor) Start(globalManager *GlobalBufferManager) {
if !atomic.CompareAndSwapInt32(&b.isMonitoring, 0, 1) {
return // 已经在监控中
}
// 状态监控器启动（静默）
go b.monitoringLoop(globalManager)
go b.healthCheckLoop()
go b.alertProcessingLoop()
}
// monitoringLoop 监控循环
func (b *BufferStatusMonitor) monitoringLoop(globalManager *GlobalBufferManager) {
ticker := time.NewTicker(b.monitorInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
b.collectMetrics(globalManager)
b.analyzeData()
b.checkAlerts()
b.updatePredictions()
b.saveSnapshot()
case <-b.shutdownChan:
return
}
}
}
// healthCheckLoop 健康检查循环
func (b *BufferStatusMonitor) healthCheckLoop() {
ticker := time.NewTicker(b.healthChecker.healthCheckInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
b.performHealthCheck()
case <-b.shutdownChan:
return
}
}
}
// alertProcessingLoop 报警处理循环
func (b *BufferStatusMonitor) alertProcessingLoop() {
ticker := time.NewTicker(1 * time.Minute) // 每分钟检查一次报警
defer ticker.Stop()
for {
select {
case <-ticker.C:
b.processAlerts()
case <-b.shutdownChan:
return
}
}
}
// collectMetrics 收集指标
func (b *BufferStatusMonitor) collectMetrics(globalManager *GlobalBufferManager) {
b.dataMutex.Lock()
defer b.dataMutex.Unlock()
// 收集全局缓冲区统计
b.monitoringData.GlobalBufferStats = globalManager.GetStats()
// 收集缓冲区状态
bufferInfo := globalManager.GetBufferInfo()
for id, info := range bufferInfo {
if infoMap, ok := info.(map[string]interface{}); ok {
bufferState := BufferState{
ID: id,
LastActivity: time.Now(),
Health: "healthy",
}
// 提取缓冲区信息
if size, ok := infoMap["total_operations"].(int64); ok {
bufferState.Size = int(size)
}
if dataSize, ok := infoMap["total_data_size"].(int64); ok {
bufferState.AverageDataSize = dataSize
}
if ratio, ok := infoMap["compress_ratio"].(float64); ok {
bufferState.CompressionRatio = ratio
}
b.monitoringData.BufferStates[id] = bufferState
}
}
// 收集性能指标
b.collectPerformanceMetrics()
// 更新实时统计
b.updateRealTimeStats()
}
// collectPerformanceMetrics 收集性能指标
func (b *BufferStatusMonitor) collectPerformanceMetrics() {
// 收集内存统计
runtime.ReadMemStats(&b.performanceMetrics.GCStats)
currentMemory := int64(b.performanceMetrics.GCStats.Alloc)
b.performanceMetrics.MemoryUsage = currentMemory
// 更新内存历史
b.performanceMetrics.MemoryHistory = append(b.performanceMetrics.MemoryHistory, currentMemory)
if len(b.performanceMetrics.MemoryHistory) > 100 { // 保留最近100个数据点
b.performanceMetrics.MemoryHistory = b.performanceMetrics.MemoryHistory[1:]
}
// 简化的CPU使用率估算（基于GC统计）
gcCPUPercent := float64(b.performanceMetrics.GCStats.GCCPUFraction) * 100
b.performanceMetrics.CPUUsage = gcCPUPercent
// 更新CPU历史
b.performanceMetrics.CPUHistory = append(b.performanceMetrics.CPUHistory, gcCPUPercent)
if len(b.performanceMetrics.CPUHistory) > 100 {
b.performanceMetrics.CPUHistory = b.performanceMetrics.CPUHistory[1:]
}
b.performanceMetrics.LastUpdated = time.Now()
}
// updateRealTimeStats 更新实时统计
func (b *BufferStatusMonitor) updateRealTimeStats() {
stats := &b.monitoringData.RealTimeStats
if b.monitoringData.GlobalBufferStats != nil {
globalStats := b.monitoringData.GlobalBufferStats
// 活跃操作数
stats.ActiveOperations = int(globalStats.ActiveBuffers)
// 处理速率（操作/秒）
if globalStats.TotalOperationsBuffered > 0 {
stats.ProcessingRate = float64(globalStats.TotalOperationsBuffered) /
time.Since(globalStats.LastCleanupTime).Seconds()
}
// 压缩比例
stats.CompressionRatio = globalStats.AverageCompressionRatio
// 缓存命中率
stats.CacheHitRate = globalStats.HitRate
}
// 内存使用（MB/s）
if b.performanceMetrics.MemoryUsage > 0 {
stats.ThroughputMBps = float64(b.performanceMetrics.MemoryUsage) / 1024 / 1024
}
stats.LastUpdated = time.Now()
}
// analyzeData 分析数据
func (b *BufferStatusMonitor) analyzeData() {
b.analyzeTrends()
b.detectAnomalies()
}
// analyzeTrends 分析趋势
func (b *BufferStatusMonitor) analyzeTrends() {
trends := &b.monitoringData.TrendAnalysis
// 内存趋势分析
if len(b.performanceMetrics.MemoryHistory) >= 3 {
recent := b.performanceMetrics.MemoryHistory[len(b.performanceMetrics.MemoryHistory)-3:]
if recent[2] > recent[1] && recent[1] > recent[0] {
trends.MemoryTrend = "increasing"
} else if recent[2] < recent[1] && recent[1] < recent[0] {
trends.MemoryTrend = "decreasing"
} else {
trends.MemoryTrend = "stable"
}
}
// 缓冲区使用趋势
bufferCount := len(b.monitoringData.BufferStates)
if bufferCount > b.alertThresholds.BufferCountWarning {
trends.BufferUsageTrend = "increasing"
} else {
trends.BufferUsageTrend = "stable"
}
trends.AnalysisTime = time.Now()
trends.Confidence = 0.8 // 简化的置信度
}
// detectAnomalies 检测异常
func (b *BufferStatusMonitor) detectAnomalies() {
// 内存异常检测
if b.performanceMetrics.MemoryUsage > b.alertThresholds.MemoryUsageCritical {
b.triggerAlert("memory", "critical",
fmt.Sprintf("内存使用过高: %d bytes", b.performanceMetrics.MemoryUsage))
} else if b.performanceMetrics.MemoryUsage > b.alertThresholds.MemoryUsageWarning {
b.triggerAlert("memory", "warning",
fmt.Sprintf("内存使用警告: %d bytes", b.performanceMetrics.MemoryUsage))
}
// 缓冲区数量异常检测
bufferCount := len(b.monitoringData.BufferStates)
if bufferCount > b.alertThresholds.BufferCountCritical {
b.triggerAlert("buffer_count", "critical",
fmt.Sprintf("缓冲区数量过多: %d", bufferCount))
} else if bufferCount > b.alertThresholds.BufferCountWarning {
b.triggerAlert("buffer_count", "warning",
fmt.Sprintf("缓冲区数量警告: %d", bufferCount))
}
}
// checkAlerts 检查报警
func (b *BufferStatusMonitor) checkAlerts() {
// 检查系统健康状态
if b.healthChecker.healthStatus.Overall == "critical" {
b.triggerAlert("system_health", "critical", "系统健康状态严重")
} else if b.healthChecker.healthStatus.Overall == "warning" {
b.triggerAlert("system_health", "warning", "系统健康状态警告")
}
}
// triggerAlert 触发报警
func (b *BufferStatusMonitor) triggerAlert(component, level, message string) {
alertKey := fmt.Sprintf("%s_%s", component, level)
// 检查冷却期
b.alertManager.mutex.Lock()
if lastAlert, exists := b.alertManager.alertCooldown[alertKey]; exists {
if time.Since(lastAlert) < b.alertManager.cooldownPeriod {
b.alertManager.mutex.Unlock()
return // 还在冷却期内
}
}
// 创建新报警
alert := Alert{
ID: fmt.Sprintf("%s_%d", alertKey, time.Now().Unix()),
Level: level,
Component: component,
Message: message,
Timestamp: time.Now(),
Metadata: make(map[string]interface{}),
Resolved: false,
}
// 添加相关指标作为元数据
alert.Metadata["memory_usage"] = b.performanceMetrics.MemoryUsage
alert.Metadata["buffer_count"] = len(b.monitoringData.BufferStates)
alert.Metadata["cpu_usage"] = b.performanceMetrics.CPUUsage
b.alertManager.alerts = append(b.alertManager.alerts, alert)
b.alertManager.alertCooldown[alertKey] = time.Now()
b.alertManager.mutex.Unlock()
// 输出报警日志
// fmt.Printf("🚨 [报警] %s - %s: %s\n", level, component, message)
}
// updatePredictions 更新预测
func (b *BufferStatusMonitor) updatePredictions() {
predictions := &b.monitoringData.Predictions
// 简化的内存使用预测
if len(b.performanceMetrics.MemoryHistory) >= 5 {
history := b.performanceMetrics.MemoryHistory
recent := history[len(history)-5:]
// 简单线性预测
growth := float64(recent[4]-recent[0]) / 4
predictions.MemoryUsageIn1Hour = recent[4] + int64(growth*120) // 2小时数据点预测1小时
predictions.MemoryUsageIn24Hours = recent[4] + int64(growth*2880) // 预测24小时
}
// 缓冲区溢出风险评估
bufferCount := len(b.monitoringData.BufferStates)
if bufferCount > b.alertThresholds.BufferCountWarning {
predictions.BufferOverflowRisk = float64(bufferCount) / float64(b.alertThresholds.BufferCountCritical)
} else {
predictions.BufferOverflowRisk = 0.1
}
// 推荐行动
predictions.RecommendedActions = b.generateRecommendations()
predictions.ConfidenceLevel = 0.7
predictions.PredictionTime = time.Now()
}
// generateRecommendations 生成推荐
func (b *BufferStatusMonitor) generateRecommendations() []string {
recommendations := make([]string, 0)
// 基于内存使用推荐
if b.performanceMetrics.MemoryUsage > b.alertThresholds.MemoryUsageWarning {
recommendations = append(recommendations, "考虑增加内存或减少缓冲区大小")
}
// 基于缓冲区数量推荐
bufferCount := len(b.monitoringData.BufferStates)
if bufferCount > b.alertThresholds.BufferCountWarning {
recommendations = append(recommendations, "考虑调整缓冲区清理频率")
}
// 基于趋势推荐
if b.monitoringData.TrendAnalysis.MemoryTrend == "increasing" {
recommendations = append(recommendations, "内存使用呈增长趋势，建议监控和优化")
}
if len(recommendations) == 0 {
recommendations = append(recommendations, "系统运行正常，继续监控")
}
return recommendations
}
// performHealthCheck 执行健康检查
func (b *BufferStatusMonitor) performHealthCheck() {
startTime := time.Now()
b.healthChecker.mutex.Lock()
defer b.healthChecker.mutex.Unlock()
health := &b.healthChecker.healthStatus
health.LastCheck = time.Now()
health.Issues = make([]HealthIssue, 0)
// 检查内存健康
memoryHealth := b.checkMemoryHealth()
health.Components["memory"] = memoryHealth
// 检查缓冲区健康
bufferHealth := b.checkBufferHealth()
health.Components["buffers"] = bufferHealth
// 检查性能健康
performanceHealth := b.checkPerformanceHealth()
health.Components["performance"] = performanceHealth
// 确定整体健康状态
health.Overall = b.determineOverallHealth()
// 记录健康检查结果
checkResult := HealthCheckResult{
Timestamp: time.Now(),
Status: health.Overall,
CheckTime: time.Since(startTime),
Issues: health.Issues,
}
b.healthChecker.healthHistory = append(b.healthChecker.healthHistory, checkResult)
if len(b.healthChecker.healthHistory) > 100 { // 保留最近100次检查
b.healthChecker.healthHistory = b.healthChecker.healthHistory[1:]
}
b.healthChecker.lastHealthCheck = time.Now()
}
// checkMemoryHealth 检查内存健康
func (b *BufferStatusMonitor) checkMemoryHealth() ComponentHealth {
health := ComponentHealth{
Status: "healthy",
LastCheck: time.Now(),
Metrics: make(map[string]interface{}),
}
memUsage := b.performanceMetrics.MemoryUsage
health.Metrics["usage_bytes"] = memUsage
health.Metrics["usage_mb"] = memUsage / 1024 / 1024
if memUsage > b.alertThresholds.MemoryUsageCritical {
health.Status = "critical"
health.Message = "内存使用严重过高"
} else if memUsage > b.alertThresholds.MemoryUsageWarning {
health.Status = "warning"
health.Message = "内存使用偏高"
} else {
health.Message = "内存使用正常"
}
return health
}
// checkBufferHealth 检查缓冲区健康
func (b *BufferStatusMonitor) checkBufferHealth() ComponentHealth {
health := ComponentHealth{
Status: "healthy",
LastCheck: time.Now(),
Metrics: make(map[string]interface{}),
}
bufferCount := len(b.monitoringData.BufferStates)
health.Metrics["buffer_count"] = bufferCount
health.Metrics["max_buffers"] = b.alertThresholds.BufferCountCritical
if bufferCount > b.alertThresholds.BufferCountCritical {
health.Status = "critical"
health.Message = "缓冲区数量过多"
} else if bufferCount > b.alertThresholds.BufferCountWarning {
health.Status = "warning"
health.Message = "缓冲区数量偏高"
} else {
health.Message = "缓冲区状态正常"
}
return health
}
// checkPerformanceHealth 检查性能健康
func (b *BufferStatusMonitor) checkPerformanceHealth() ComponentHealth {
health := ComponentHealth{
Status: "healthy",
LastCheck: time.Now(),
Metrics: make(map[string]interface{}),
}
cpuUsage := b.performanceMetrics.CPUUsage
health.Metrics["cpu_usage"] = cpuUsage
health.Metrics["gc_cpu_fraction"] = b.performanceMetrics.GCStats.GCCPUFraction
if cpuUsage > 80 {
health.Status = "warning"
health.Message = "CPU使用率偏高"
} else {
health.Message = "性能状态正常"
}
return health
}
// determineOverallHealth 确定整体健康状态
func (b *BufferStatusMonitor) determineOverallHealth() string {
hasCritical := false
hasWarning := false
for _, component := range b.healthChecker.healthStatus.Components {
switch component.Status {
case "critical":
hasCritical = true
case "warning":
hasWarning = true
}
}
if hasCritical {
return "critical"
} else if hasWarning {
return "warning"
}
return "healthy"
}
// processAlerts 处理报警
func (b *BufferStatusMonitor) processAlerts() {
b.alertManager.mutex.Lock()
defer b.alertManager.mutex.Unlock()
// 检查是否有报警需要自动解决
for i := range b.alertManager.alerts {
alert := &b.alertManager.alerts[i]
if !alert.Resolved {
if b.shouldResolveAlert(alert) {
now := time.Now()
alert.Resolved = true
alert.ResolvedAt = &now
fmt.Printf("✅ [报警解决] %s - %s: %s\n",
alert.Level, alert.Component, alert.Message)
}
}
}
// 移动已解决的报警到历史记录
activeAlerts := make([]Alert, 0)
for _, alert := range b.alertManager.alerts {
if !alert.Resolved {
activeAlerts = append(activeAlerts, alert)
} else {
b.alertManager.alertHistory = append(b.alertManager.alertHistory, alert)
}
}
b.alertManager.alerts = activeAlerts
// 限制历史记录大小
if len(b.alertManager.alertHistory) > b.alertManager.maxAlertHistory {
excess := len(b.alertManager.alertHistory) - b.alertManager.maxAlertHistory
b.alertManager.alertHistory = b.alertManager.alertHistory[excess:]
}
}
// shouldResolveAlert 检查是否应该解决报警
func (b *BufferStatusMonitor) shouldResolveAlert(alert *Alert) bool {
switch alert.Component {
case "memory":
return b.performanceMetrics.MemoryUsage < b.alertThresholds.MemoryUsageWarning
case "buffer_count":
return len(b.monitoringData.BufferStates) < b.alertThresholds.BufferCountWarning
case "system_health":
return b.healthChecker.healthStatus.Overall == "healthy"
}
return false
}
// saveSnapshot 保存监控快照
func (b *BufferStatusMonitor) saveSnapshot() {
b.historyMutex.Lock()
defer b.historyMutex.Unlock()
snapshot := MonitorSnapshot{
Timestamp: time.Now(),
SystemHealth: b.healthChecker.healthStatus,
BufferCount: len(b.monitoringData.BufferStates),
TotalMemoryUsage: b.performanceMetrics.MemoryUsage,
OperationsPerSecond: b.monitoringData.RealTimeStats.ProcessingRate,
ErrorRate: b.monitoringData.RealTimeStats.ErrorRate,
CacheHitRate: b.monitoringData.RealTimeStats.CacheHitRate,
}
b.historyBuffer = append(b.historyBuffer, snapshot)
// 限制历史记录大小
if len(b.historyBuffer) > b.maxHistorySize {
b.historyBuffer = b.historyBuffer[1:]
}
}
// Stop 停止监控器
func (b *BufferStatusMonitor) Stop() {
if !atomic.CompareAndSwapInt32(&b.isMonitoring, 1, 0) {
return
}
close(b.shutdownChan)
fmt.Printf("🔍 [缓冲区状态监控器] 已停止监控\n")
}
// GetMonitoringData 获取监控数据
func (b *BufferStatusMonitor) GetMonitoringData() *MonitoringData {
b.dataMutex.RLock()
defer b.dataMutex.RUnlock()
// 深拷贝监控数据
dataCopy := *b.monitoringData
return &dataCopy
}
// GetHealthStatus 获取健康状态
func (b *BufferStatusMonitor) GetHealthStatus() HealthStatus {
b.healthChecker.mutex.RLock()
defer b.healthChecker.mutex.RUnlock()
return b.healthChecker.healthStatus
}
// GetActiveAlerts 获取活跃报警
func (b *BufferStatusMonitor) GetActiveAlerts() []Alert {
b.alertManager.mutex.RLock()
defer b.alertManager.mutex.RUnlock()
alerts := make([]Alert, len(b.alertManager.alerts))
copy(alerts, b.alertManager.alerts)
return alerts
}
// GetMonitorHistory 获取监控历史
func (b *BufferStatusMonitor) GetMonitorHistory(limit int) []MonitorSnapshot {
b.historyMutex.Lock()
defer b.historyMutex.Unlock()
if limit <= 0 || limit > len(b.historyBuffer) {
limit = len(b.historyBuffer)
}
history := make([]MonitorSnapshot, limit)
startIndex := len(b.historyBuffer) - limit
copy(history, b.historyBuffer[startIndex:])
return history
}
// ExportMonitoringReport 导出监控报告
func (b *BufferStatusMonitor) ExportMonitoringReport() (string, error) {
report := map[string]interface{}{
"timestamp": time.Now(),
"monitoring_data": b.GetMonitoringData(),
"health_status": b.GetHealthStatus(),
"active_alerts": b.GetActiveAlerts(),
"performance_metrics": b.performanceMetrics,
"recent_history": b.GetMonitorHistory(50), // 最近50个快照
}
jsonData, err := json.MarshalIndent(report, "", " ")
if err != nil {
return "", fmt.Errorf("导出监控报告失败: %v", err)
}
return string(jsonData), nil
}

109
util/cache/delayed_batch_write_manager.go vendored
View File

@@ -41,25 +41,25 @@ type CacheOperation struct {
// CacheWriteConfig 缓存写入配置
type CacheWriteConfig struct {
// 🎯 核心策略
// 核心策略
Strategy CacheWriteStrategy `env:"CACHE_WRITE_STRATEGY" default:"hybrid"`
// ⏱️ 批量写入参数（自动计算，但可手动覆盖）
// 批量写入参数（自动计算，但可手动覆盖）
MaxBatchInterval time.Duration `env:"BATCH_MAX_INTERVAL"` // 0表示自动计算
MaxBatchSize int `env:"BATCH_MAX_SIZE"` // 0表示自动计算
MaxBatchDataSize int `env:"BATCH_MAX_DATA_SIZE"` // 0表示自动计算
// 🎛️ 行为参数
// 行为参数
HighPriorityRatio float64 `env:"HIGH_PRIORITY_RATIO" default:"0.3"`
EnableCompression bool // 默认启用操作合并
// 📊 内部计算参数（运行时动态调整）
// 内部计算参数（运行时动态调整）
idleThresholdCPU float64 // CPU空闲阈值
idleThresholdDisk float64 // 磁盘空闲阈值
forceFlushInterval time.Duration // 强制刷新间隔
autoTuneInterval time.Duration // 调优检查间隔
// 🔧 约束边界（硬编码）
// 约束边界（硬编码）
minBatchInterval time.Duration // 最小30秒
maxBatchInterval time.Duration // 最大10分钟
minBatchSize int // 最小10个
@@ -68,16 +68,16 @@ type CacheWriteConfig struct {
// Initialize 初始化配置
func (c *CacheWriteConfig) Initialize() error {
// 🔧 设置硬编码约束边界
// 设置硬编码约束边界
c.minBatchInterval = 30 * time.Second
c.maxBatchInterval = 600 * time.Second // 10分钟
c.minBatchSize = 10
c.maxBatchSize = 1000
// 🎯 加载环境变量
// 加载环境变量
c.loadFromEnvironment()
// 🤖 自动计算最优参数（除非手动设置）
// 自动计算最优参数（除非手动设置）
if c.MaxBatchInterval == 0 {
c.MaxBatchInterval = c.calculateOptimalBatchInterval()
}
@@ -88,13 +88,13 @@ func (c *CacheWriteConfig) Initialize() error {
c.MaxBatchDataSize = c.calculateOptimalDataSize()
}
// 🔧 内部参数自动设置
// 内部参数自动设置
c.forceFlushInterval = c.MaxBatchInterval * 5 // 5倍批量间隔
c.autoTuneInterval = 300 * time.Second // 5分钟调优间隔
c.idleThresholdCPU = 0.3 // CPU空闲阈值
c.idleThresholdDisk = 0.5 // 磁盘空闲阈值
// ✅ 参数验证和约束
// 参数验证和约束
return c.validateAndConstraint()
}
@@ -134,7 +134,7 @@ func (c *CacheWriteConfig) loadFromEnvironment() {
// calculateOptimalBatchInterval 计算最优批量间隔
func (c *CacheWriteConfig) calculateOptimalBatchInterval() time.Duration {
// 🎯 基于系统性能动态计算
// 基于系统性能动态计算
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
@@ -164,7 +164,7 @@ func (c *CacheWriteConfig) calculateOptimalBatchInterval() time.Duration {
// calculateOptimalBatchSize 计算最优批量大小
func (c *CacheWriteConfig) calculateOptimalBatchSize() int {
// 🎯 基于CPU核心数和内存动态计算
// 基于CPU核心数和内存动态计算
numCPU := runtime.NumCPU()
var memStats runtime.MemStats
@@ -194,7 +194,7 @@ func (c *CacheWriteConfig) calculateOptimalBatchSize() int {
// calculateOptimalDataSize 计算最优数据大小
func (c *CacheWriteConfig) calculateOptimalDataSize() int {
// 🎯 基于可用内存计算
// 基于可用内存计算
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
availableMemoryGB := float64(memStats.Sys) / 1024 / 1024 / 1024
@@ -214,7 +214,7 @@ func (c *CacheWriteConfig) calculateOptimalDataSize() int {
// validateAndConstraint 验证和约束配置
func (c *CacheWriteConfig) validateAndConstraint() error {
// 🔧 验证配置合理性
// 验证配置合理性
if c.MaxBatchInterval < c.minBatchInterval {
return fmt.Errorf("批量间隔配置错误: MaxBatchInterval(%v) < MinBatchInterval(%v)",
c.MaxBatchInterval, c.minBatchInterval)
@@ -230,7 +230,7 @@ func (c *CacheWriteConfig) validateAndConstraint() error {
c.HighPriorityRatio)
}
// 🎯 应用最终约束
// 应用最终约束
if c.MaxBatchInterval > c.maxBatchInterval {
c.MaxBatchInterval = c.maxBatchInterval
}
@@ -256,7 +256,7 @@ type DelayedBatchWriteManager struct {
queueBuffer []*CacheOperation
queueMutex sync.Mutex
// 🚀 全局缓冲区管理器
// 全局缓冲区管理器
globalBufferManager *GlobalBufferManager
// 统计信息
@@ -320,7 +320,7 @@ func NewDelayedBatchWriteManager() (*DelayedBatchWriteManager, error) {
return nil, fmt.Errorf("配置初始化失败: %v", err)
}
// 🚀 创建全局缓冲区管理器
// 创建全局缓冲区管理器
globalBufferManager := NewGlobalBufferManager(BufferHybrid)
manager := &DelayedBatchWriteManager{
@@ -349,7 +349,7 @@ func (m *DelayedBatchWriteManager) Initialize() error {
m.initMutex.Lock()
defer m.initMutex.Unlock()
// 🚀 初始化全局缓冲区管理器
// 初始化全局缓冲区管理器
if err := m.globalBufferManager.Initialize(); err != nil {
return fmt.Errorf("全局缓冲区管理器初始化失败: %v", err)
}
@@ -364,7 +364,7 @@ func (m *DelayedBatchWriteManager) Initialize() error {
// 启动自动调优goroutine
go m.autoTuningProcessor()
// 🔍 启动全局缓冲区监控
// 启动全局缓冲区监控
go m.globalBufferMonitor()
fmt.Printf("缓存写入策略: %s\n", m.strategy)
@@ -383,7 +383,7 @@ func (m *DelayedBatchWriteManager) HandleCacheOperation(op *CacheOperation) erro
return err
}
// 🔥 关键：无论什么策略，都立即更新内存缓存
// 关键：无论什么策略，都立即更新内存缓存
if err := m.updateMemoryCache(op); err != nil {
return fmt.Errorf("内存缓存更新失败: %v", err)
}
@@ -393,20 +393,20 @@ func (m *DelayedBatchWriteManager) HandleCacheOperation(op *CacheOperation) erro
return m.immediateWriteToDisk(op)
}
// 🚀 使用全局缓冲区管理器进行智能缓冲
// 使用全局缓冲区管理器进行智能缓冲
return m.handleWithGlobalBuffer(op)
}
// handleWithGlobalBuffer 使用全局缓冲区处理操作
func (m *DelayedBatchWriteManager) handleWithGlobalBuffer(op *CacheOperation) error {
// 🎯 尝试添加到全局缓冲区
// 尝试添加到全局缓冲区
buffer, shouldFlush, err := m.globalBufferManager.AddOperation(op)
if err != nil {
// 全局缓冲区失败，降级到本地队列
return m.enqueueForBatchWrite(op)
}
// 🚀 如果需要刷新缓冲区
// 如果需要刷新缓冲区
if shouldFlush {
return m.flushGlobalBuffer(buffer.ID)
}
@@ -425,7 +425,7 @@ func (m *DelayedBatchWriteManager) flushGlobalBuffer(bufferID string) error {
return nil
}
// 🎯 按优先级排序操作
// 按优先级排序操作
sort.Slice(operations, func(i, j int) bool {
if operations[i].Priority != operations[j].Priority {
return operations[i].Priority < operations[j].Priority
@@ -433,14 +433,14 @@ func (m *DelayedBatchWriteManager) flushGlobalBuffer(bufferID string) error {
return operations[i].Timestamp.Before(operations[j].Timestamp)
})
// 📊 统计信息更新
// 统计信息更新
atomic.AddInt64(&m.stats.BatchWrites, 1)
atomic.AddInt64(&m.stats.TotalWrites, 1)
m.stats.LastFlushTime = time.Now()
m.stats.LastFlushTrigger = "全局缓冲区触发"
m.stats.LastBatchSize = len(operations)
// 🚀 批量写入磁盘
// 批量写入磁盘
err = m.batchWriteToDisk(operations)
if err != nil {
atomic.AddInt64(&m.stats.FailedWrites, 1)
@@ -462,7 +462,7 @@ func (m *DelayedBatchWriteManager) globalBufferMonitor() {
for {
select {
case <-ticker.C:
// 🔍 检查是否有过期的缓冲区需要刷新
// 检查是否有过期的缓冲区需要刷新
m.checkAndFlushExpiredBuffers()
case <-m.shutdownChan:
@@ -473,26 +473,26 @@ func (m *DelayedBatchWriteManager) globalBufferMonitor() {
// checkAndFlushExpiredBuffers 检查并刷新过期缓冲区
func (m *DelayedBatchWriteManager) checkAndFlushExpiredBuffers() {
// 🔧 修复：使用原子操作获取需要刷新的缓冲区列表
// 使用原子操作获取需要刷新的缓冲区列表
expiredBuffers := m.globalBufferManager.GetExpiredBuffersForFlush()
flushedCount := 0
for _, bufferID := range expiredBuffers {
if err := m.flushGlobalBuffer(bufferID); err != nil {
// 🎯 改进：区分错误类型，缓冲区不存在是正常情况
// 区分错误类型，缓冲区不存在是正常情况
if isBufferNotExistError(err) {
// 静默处理：缓冲区已被其他线程清理，这是正常的
continue
}
// 只有真正的错误才打印警告
fmt.Printf("⚠️ [全局缓冲区] 刷新缓冲区失败 %s: %v\n", bufferID, err)
fmt.Printf("[全局缓冲区] 刷新缓冲区失败 %s: %v\n", bufferID, err)
} else {
flushedCount++
}
}
if flushedCount > 0 {
fmt.Printf("🔄 [全局缓冲区] 刷新完成，处理 %d 个过期缓冲区\n", flushedCount)
fmt.Printf("[全局缓冲区] 刷新完成，处理 %d 个过期缓冲区\n", flushedCount)
}
}
@@ -505,18 +505,16 @@ func isBufferNotExistError(err error) bool {
// updateMemoryCache 更新内存缓存（立即执行）
func (m *DelayedBatchWriteManager) updateMemoryCache(op *CacheOperation) error {
// 🔥 关键修复：如果有主缓存更新函数，立即更新内存层
// 如果有主缓存更新函数，立即更新内存层
if m.mainCacheUpdater != nil {
// 序列化数据
data, err := m.serializer.Serialize(op.Data)
_, err := m.serializer.Serialize(op.Data)
if err != nil {
return fmt.Errorf("内存缓存数据序列化失败: %v", err)
}
// 这里只更新内存，不写磁盘（磁盘由批量写入处理）
// 注意：mainCacheUpdater实际上是SetBothLevels，会同时更新内存和磁盘
// 为了避免重复写磁盘，我们暂时保持原逻辑
_ = data // 暂不使用，避免编译警告
}
return nil
}
@@ -644,21 +642,21 @@ func (m *DelayedBatchWriteManager) Shutdown(timeout time.Duration) error {
go func() {
var lastErr error
// 🚀 第一步：强制刷新全局缓冲区（优先级最高）
// 第一步：强制刷新全局缓冲区（优先级最高）
if err := m.flushAllGlobalBuffers(); err != nil {
fmt.Printf("❌ [数据保护] 全局缓冲区刷新失败: %v\n", err)
fmt.Printf("[数据保护] 全局缓冲区刷新失败: %v\n", err)
lastErr = err
}
// 🔧 第二步：刷新本地队列
// 第二步：刷新本地队列
if err := m.flushAllPendingData(); err != nil {
fmt.Printf("❌ [数据保护] 本地队列刷新失败: %v\n", err)
fmt.Printf("[数据保护] 本地队列刷新失败: %v\n", err)
lastErr = err
}
// 🔄 第三步：关闭全局缓冲区管理器
// 第三步：关闭全局缓冲区管理器
if err := m.globalBufferManager.Shutdown(); err != nil {
fmt.Printf("❌ [数据保护] 全局缓冲区管理器关闭失败: %v\n", err)
fmt.Printf("[数据保护] 全局缓冲区管理器关闭失败: %v\n", err)
lastErr = err
}
@@ -686,7 +684,7 @@ func (m *DelayedBatchWriteManager) flushAllGlobalBuffers() error {
for bufferID, operations := range allBuffers {
if len(operations) > 0 {
if err := m.batchWriteToDisk(operations); err != nil {
fmt.Printf("❌ [全局缓冲区] 缓冲区 %s 刷新失败: %v\n", bufferID, err)
fmt.Printf("[全局缓冲区] 缓冲区 %s 刷新失败: %v\n", bufferID, err)
lastErr = fmt.Errorf("刷新全局缓冲区 %s 失败: %v", bufferID, err)
continue
}
@@ -799,7 +797,7 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error {
return nil
}
// 🔧 操作合并：如果启用压缩，使用合并后的操作
// 操作合并：如果启用压缩，使用合并后的操作
var operations []*CacheOperation
if m.config.EnableCompression {
operations = m.getCompressedOperations()
@@ -812,7 +810,7 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error {
return nil
}
// 🎯 按优先级排序：确保重要数据优先写入
// 按优先级排序：确保重要数据优先写入
sort.Slice(operations, func(i, j int) bool {
if operations[i].Priority != operations[j].Priority {
return operations[i].Priority < operations[j].Priority // 数字越小优先级越高
@@ -820,13 +818,13 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error {
return operations[i].Timestamp.Before(operations[j].Timestamp)
})
// 📊 统计信息更新
// 统计信息更新
atomic.AddInt64(&m.stats.BatchWrites, 1)
m.stats.LastFlushTime = time.Now()
m.stats.LastFlushTrigger = trigger
m.stats.LastBatchSize = len(operations)
// 🚀 批量写入磁盘
// 批量写入磁盘
err := m.batchWriteToDisk(operations)
if err != nil {
atomic.AddInt64(&m.stats.FailedWrites, 1)
@@ -841,7 +839,7 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error {
m.mapMutex.Unlock()
}
// 📈 成功统计
// 成功统计
atomic.AddInt64(&m.stats.SuccessfulWrites, 1)
atomic.AddInt64(&m.stats.TotalWrites, 1)
m.stats.TotalOperationsWritten += len(operations)
@@ -895,10 +893,10 @@ func (m *DelayedBatchWriteManager) emergencyFlush() error {
// autoTuneParameters 自适应参数调优
func (m *DelayedBatchWriteManager) autoTuneParameters() {
// 🤖 完全自动调优，无需配置开关
// 完全自动调优，无需配置开关
stats := m.collectRecentStats()
// 🎯 调优批量间隔：基于系统负载动态调整
// 调优批量间隔：基于系统负载动态调整
avgSystemLoad := stats.SystemLoadAverage
switch {
case avgSystemLoad > 0.8: // 高负载：延长间隔，减少干扰
@@ -907,7 +905,7 @@ func (m *DelayedBatchWriteManager) autoTuneParameters() {
m.config.MaxBatchInterval = m.maxDuration(m.config.MaxBatchInterval*8/10, m.config.minBatchInterval)
}
// 🎯 调优批量大小：基于写入频率动态调整
// 调优批量大小：基于写入频率动态调整
queueSize := int(atomic.LoadInt32(&m.stats.CurrentQueueSize))
switch {
case queueSize > 200: // 高频：增大批量，提高效率
@@ -954,20 +952,13 @@ func (m *DelayedBatchWriteManager) GetStats() map[string]interface{} {
stats.SystemLoadAverage = float64(stats.TotalWrites) / float64(stats.TotalOperations)
}
// 🚀 获取全局缓冲区统计
// 获取全局缓冲区统计
globalBufferStats := m.globalBufferManager.GetStats()
// 🔍 获取监控数据（如果监控器存在）
var monitoringData *MonitoringData
if m.globalBufferManager.statusMonitor != nil {
monitoringData = m.globalBufferManager.statusMonitor.GetMonitoringData()
}
// 🎯 合并所有统计信息
// 合并所有统计信息
combinedStats := map[string]interface{}{
"write_manager": &stats,
"global_buffer": globalBufferStats,
"monitoring": monitoringData,
"buffer_info": m.globalBufferManager.GetBufferInfo(),
}

2
util/cache/disk_cache.go vendored
View File

@@ -135,7 +135,7 @@ func (c *DiskCache) Set(key string, data []byte, ttl time.Duration) error {
filename := c.getFilename(key)
filePath := filepath.Join(c.path, filename)
// 🔥 确保目录存在（防止外部删除缓存目录）
// 确保目录存在（防止外部删除缓存目录）
if err := os.MkdirAll(c.path, 0755); err != nil {
return fmt.Errorf("创建缓存目录失败: %v", err)
}

8
util/cache/enhanced_two_level_cache.go vendored
View File

@@ -34,7 +34,7 @@ func NewEnhancedTwoLevelCache() (*EnhancedTwoLevelCache, error) {
// 创建序列化器
serializer := NewGobSerializer()
// 🔥 设置内存缓存的磁盘缓存引用，用于LRU淘汰时的备份
// 设置内存缓存的磁盘缓存引用，用于LRU淘汰时的备份
memCache.SetDiskCacheReference(diskCache)
return &EnhancedTwoLevelCache{
@@ -65,7 +65,7 @@ func (c *EnhancedTwoLevelCache) Set(key string, data []byte, ttl time.Duration)
func (c *EnhancedTwoLevelCache) SetMemoryOnly(key string, data []byte, ttl time.Duration) error {
now := time.Now()
// 🔥 只更新内存缓存，不触发磁盘写入
// 只更新内存缓存，不触发磁盘写入
c.memory.SetWithTimestamp(key, data, ttl, now)
return nil
@@ -78,7 +78,7 @@ func (c *EnhancedTwoLevelCache) SetBothLevels(key string, data []byte, ttl time.
// 同步更新内存缓存
c.memory.SetWithTimestamp(key, data, ttl, now)
// 🔥 修复：同步更新磁盘缓存，确保数据立即写入
// 同步更新磁盘缓存，确保数据立即写入
return c.disk.Set(key, data, ttl)
}
@@ -155,7 +155,7 @@ func (c *EnhancedTwoLevelCache) FlushMemoryToDisk() error {
for key, item := range allItems {
// 同步写入到磁盘缓存
if err := c.disk.Set(key, item.Data, item.TTL); err != nil {
fmt.Printf("❌ [内存同步] 同步失败: %s -> %v\n", key, err)
fmt.Printf("[内存同步] 同步失败: %s -> %v\n", key, err)
lastErr = err
continue
}

72
util/cache/global_buffer_manager.go vendored
View File

@@ -71,14 +71,7 @@ type GlobalBufferManager struct {
buffers map[string]*GlobalBuffer // 缓冲区映射
buffersMutex sync.RWMutex // 缓冲区锁
// 搜索模式分析
patternAnalyzer *SearchPatternAnalyzer
// 数据合并器
dataMerger *AdvancedDataMerger
// 状态监控
statusMonitor *BufferStatusMonitor
// 已移除：搜索模式分析、数据合并器、状态监控
// 统计信息
stats *GlobalBufferStats
@@ -132,10 +125,7 @@ func NewGlobalBufferManager(strategy GlobalBufferStrategy) *GlobalBufferManager
},
}
// 初始化组件
manager.patternAnalyzer = NewSearchPatternAnalyzer()
manager.dataMerger = NewAdvancedDataMerger()
manager.statusMonitor = NewBufferStatusMonitor()
// 初始化组件（移除未使用的监控与合并器）
return manager
}
@@ -150,8 +140,7 @@ func (g *GlobalBufferManager) Initialize() error {
g.cleanupTicker = time.NewTicker(5 * time.Minute) // 每5分钟清理一次
go g.cleanupRoutine()
// 启动状态监控
go g.statusMonitor.Start(g)
// 移除状态监控启动（监控已删除）
// 初始化完成（静默）
return nil
@@ -163,13 +152,13 @@ func (g *GlobalBufferManager) AddOperation(op *CacheOperation) (*GlobalBuffer, b
return nil, false, err
}
// 🎯 根据策略确定缓冲区ID
// 根据策略确定缓冲区ID
bufferID := g.determineBufferID(op)
g.buffersMutex.Lock()
defer g.buffersMutex.Unlock()
// 🔧 获取或创建缓冲区
// 获取或创建缓冲区
buffer, exists := g.buffers[bufferID]
if !exists {
buffer = g.createNewBuffer(bufferID, op)
@@ -178,10 +167,10 @@ func (g *GlobalBufferManager) AddOperation(op *CacheOperation) (*GlobalBuffer, b
atomic.AddInt64(&g.stats.ActiveBuffers, 1)
}
// 🚀 添加操作到缓冲区
// 添加操作到缓冲区
shouldFlush := g.addOperationToBuffer(buffer, op)
// 📊 更新统计
// 更新统计
atomic.AddInt64(&g.stats.TotalOperationsBuffered, 1)
return buffer, shouldFlush, nil
@@ -197,8 +186,8 @@ func (g *GlobalBufferManager) determineBufferID(op *CacheOperation) string {
return fmt.Sprintf("plugin_%s", op.PluginName)
case BufferByPattern:
pattern := g.patternAnalyzer.AnalyzePattern(op)
return fmt.Sprintf("pattern_%s", pattern.KeywordPattern)
// 已移除模式分析器，退化为按关键词分组
return fmt.Sprintf("keyword_%s", op.Keyword)
case BufferHybrid:
// 混合策略优化：插件+时间窗口（去掉关键词避免高并发爆炸）
@@ -236,33 +225,26 @@ func (g *GlobalBufferManager) addOperationToBuffer(buffer *GlobalBuffer, op *Cac
buffer.mutex.Lock()
defer buffer.mutex.Unlock()
// 🔧 数据合并优化
merged := g.dataMerger.TryMergeOperation(buffer, op)
if merged {
atomic.AddInt64(&g.stats.TotalOperationsMerged, 1)
atomic.AddInt64(&g.stats.TotalDataMerged, int64(op.DataSize))
} else {
// 添加新操作
buffer.Operations = append(buffer.Operations, op)
buffer.TotalOperations++
buffer.TotalDataSize += int64(op.DataSize)
// 按关键词分组
if buffer.KeywordGroups[op.Keyword] == nil {
buffer.KeywordGroups[op.Keyword] = make([]*CacheOperation, 0)
}
buffer.KeywordGroups[op.Keyword] = append(buffer.KeywordGroups[op.Keyword], op)
// 按插件分组
if buffer.PluginGroups[op.PluginName] == nil {
buffer.PluginGroups[op.PluginName] = make([]*CacheOperation, 0)
}
buffer.PluginGroups[op.PluginName] = append(buffer.PluginGroups[op.PluginName], op)
// 直接追加（已移除数据合并器）
buffer.Operations = append(buffer.Operations, op)
buffer.TotalOperations++
buffer.TotalDataSize += int64(op.DataSize)
// 按关键词分组
if buffer.KeywordGroups[op.Keyword] == nil {
buffer.KeywordGroups[op.Keyword] = make([]*CacheOperation, 0)
}
buffer.KeywordGroups[op.Keyword] = append(buffer.KeywordGroups[op.Keyword], op)
// 按插件分组
if buffer.PluginGroups[op.PluginName] == nil {
buffer.PluginGroups[op.PluginName] = make([]*CacheOperation, 0)
}
buffer.PluginGroups[op.PluginName] = append(buffer.PluginGroups[op.PluginName], op)
buffer.LastUpdatedAt = time.Now()
// 🎯 检查是否应该刷新
// 检查是否应该刷新
return g.shouldFlushBuffer(buffer)
}
@@ -503,10 +485,10 @@ func (g *GlobalBufferManager) GetExpiredBuffersForFlush() []string {
defer g.buffersMutex.RUnlock()
now := time.Now()
expiredBuffers := make([]string, 0, 10) // 🚀 预分配容量，减少内存重分配
expiredBuffers := make([]string, 0, 10) // 预分配容量，减少内存重分配
for id, buffer := range g.buffers {
// 🎯 快速预检查：先检查时间，减少锁竞争
// 快速预检查：先检查时间，减少锁竞争
if now.Sub(buffer.LastUpdatedAt) <= 4*time.Minute {
continue // 跳过未过期的缓冲区
}

208
util/cache/metric_collector.go vendored
View File

@@ -1,208 +0,0 @@
package cache
import (
"runtime"
"sync/atomic"
"time"
)
// NewMetricCollector 创建指标收集器
func NewMetricCollector() *MetricCollector {
return &MetricCollector{
systemMetrics: &SystemMetrics{},
applicationMetrics: &ApplicationMetrics{},
cacheMetrics: &CacheMetrics{},
metricsHistory: make([]MetricSnapshot, 0),
maxHistorySize: 1000, // 保留1000个历史快照
collectionChan: make(chan struct{}),
}
}
// Start 启动指标收集
func (m *MetricCollector) Start(interval time.Duration) error {
if !atomic.CompareAndSwapInt32(&m.isCollecting, 0, 1) {
return nil // 已经在收集中
}
go m.collectionLoop(interval)
return nil
}
// Stop 停止指标收集
func (m *MetricCollector) Stop() {
if atomic.CompareAndSwapInt32(&m.isCollecting, 1, 0) {
close(m.collectionChan)
}
}
// collectionLoop 收集循环
func (m *MetricCollector) collectionLoop(interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
m.collectMetrics()
case <-m.collectionChan:
return
}
}
}
// collectMetrics 收集指标
func (m *MetricCollector) collectMetrics() {
now := time.Now()
// 收集系统指标
systemMetrics := m.collectSystemMetrics(now)
// 收集应用指标
applicationMetrics := m.collectApplicationMetrics(now)
// 收集缓存指标
cacheMetrics := m.collectCacheMetrics(now)
// 创建快照
snapshot := MetricSnapshot{
Timestamp: now,
System: *systemMetrics,
Application: *applicationMetrics,
Cache: *cacheMetrics,
}
// 计算综合指标
snapshot.OverallPerformance = m.calculateOverallPerformance(&snapshot)
snapshot.Efficiency = m.calculateEfficiency(&snapshot)
snapshot.Stability = m.calculateStability(&snapshot)
// 保存快照
m.saveSnapshot(snapshot)
}
// collectSystemMetrics 收集系统指标
func (m *MetricCollector) collectSystemMetrics(timestamp time.Time) *SystemMetrics {
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
return &SystemMetrics{
Timestamp: timestamp,
CPUUsage: float64(memStats.GCCPUFraction),
MemoryUsage: int64(memStats.Alloc),
MemoryTotal: int64(memStats.Sys),
DiskIORate: 0, // 简化实现
NetworkIORate: 0, // 简化实现
GoroutineCount: runtime.NumGoroutine(),
GCPauseDuration: time.Duration(memStats.PauseTotalNs),
HeapSize: int64(memStats.HeapSys),
AllocRate: float64(memStats.Mallocs - memStats.Frees),
}
}
// collectApplicationMetrics 收集应用指标
func (m *MetricCollector) collectApplicationMetrics(timestamp time.Time) *ApplicationMetrics {
// 简化实现，实际应用中应该从应用监控系统获取
return &ApplicationMetrics{
Timestamp: timestamp,
RequestRate: 100.0, // 模拟值
ResponseTime: 50 * time.Millisecond,
ErrorRate: 0.01,
ThroughputMBps: 10.5,
ConcurrentUsers: 50,
QueueDepth: 5,
ProcessingRate: 95.5,
}
}
// collectCacheMetrics 收集缓存指标
func (m *MetricCollector) collectCacheMetrics(timestamp time.Time) *CacheMetrics {
// 简化实现，实际应用中应该从缓存系统获取
return &CacheMetrics{
Timestamp: timestamp,
HitRate: 0.85,
WriteRate: 20.0,
ReadRate: 80.0,
EvictionRate: 2.0,
CompressionRatio: 0.6,
StorageUsage: 1024 * 1024 * 100, // 100MB
BufferUtilization: 0.75,
BatchEfficiency: 0.9,
}
}
// calculateOverallPerformance 计算整体性能
func (m *MetricCollector) calculateOverallPerformance(snapshot *MetricSnapshot) float64 {
// 性能评分算法
cpuScore := (1.0 - snapshot.System.CPUUsage) * 30
memoryScore := (1.0 - float64(snapshot.System.MemoryUsage)/float64(snapshot.System.MemoryTotal)) * 25
responseScore := (1.0 - float64(snapshot.Application.ResponseTime)/float64(time.Second)) * 25
cacheScore := snapshot.Cache.HitRate * 20
return cpuScore + memoryScore + responseScore + cacheScore
}
// calculateEfficiency 计算效率
func (m *MetricCollector) calculateEfficiency(snapshot *MetricSnapshot) float64 {
// 效率评分算法
cacheEfficiency := snapshot.Cache.HitRate * 0.4
batchEfficiency := snapshot.Cache.BatchEfficiency * 0.3
compressionEfficiency := snapshot.Cache.CompressionRatio * 0.3
return cacheEfficiency + batchEfficiency + compressionEfficiency
}
// calculateStability 计算稳定性
func (m *MetricCollector) calculateStability(snapshot *MetricSnapshot) float64 {
// 稳定性评分算法（基于变化率）
errorRateStability := (1.0 - snapshot.Application.ErrorRate) * 0.5
responseTimeStability := 1.0 - (float64(snapshot.Application.ResponseTime) / float64(time.Second))
if responseTimeStability < 0 {
responseTimeStability = 0
}
responseTimeStability *= 0.5
return errorRateStability + responseTimeStability
}
// saveSnapshot 保存快照
func (m *MetricCollector) saveSnapshot(snapshot MetricSnapshot) {
m.historyMutex.Lock()
defer m.historyMutex.Unlock()
m.metricsHistory = append(m.metricsHistory, snapshot)
// 限制历史记录大小
if len(m.metricsHistory) > m.maxHistorySize {
m.metricsHistory = m.metricsHistory[1:]
}
}
// GetLatestMetrics 获取最新指标
func (m *MetricCollector) GetLatestMetrics() *MetricSnapshot {
m.historyMutex.RLock()
defer m.historyMutex.RUnlock()
if len(m.metricsHistory) == 0 {
return nil
}
latest := m.metricsHistory[len(m.metricsHistory)-1]
return &latest
}
// GetMetricsHistory 获取指标历史
func (m *MetricCollector) GetMetricsHistory(limit int) []MetricSnapshot {
m.historyMutex.RLock()
defer m.historyMutex.RUnlock()
if limit <= 0 || limit > len(m.metricsHistory) {
limit = len(m.metricsHistory)
}
history := make([]MetricSnapshot, limit)
startIndex := len(m.metricsHistory) - limit
copy(history, m.metricsHistory[startIndex:])
return history
}

479
util/cache/performance_analyzer.go vendored
View File

@@ -1,479 +0,0 @@
package cache
import (
"fmt"
"math"
"sort"
"sync"
"time"
)
// TrendAnalyzer 趋势分析器
type TrendAnalyzer struct {
trends map[string]*TrendData
mutex sync.RWMutex
}
// TrendData 趋势数据
type TrendData struct {
Values []float64
Timestamps []time.Time
Slope float64
RSquared float64
LastUpdate time.Time
}
// AnomalyDetector 异常检测器
type AnomalyDetector struct {
baselines map[string]*Baseline
mutex sync.RWMutex
}
// Baseline 基线数据
type Baseline struct {
Mean float64
StdDev float64
Min float64
Max float64
SampleSize int
LastUpdate time.Time
}
// CorrelationAnalyzer 相关性分析器
type CorrelationAnalyzer struct {
correlationMatrix map[string]map[string]float64
mutex sync.RWMutex
}
// NewPerformanceAnalyzer 创建性能分析器
func NewPerformanceAnalyzer() *PerformanceAnalyzer {
return &PerformanceAnalyzer{
trendAnalyzer: &TrendAnalyzer{
trends: make(map[string]*TrendData),
},
anomalyDetector: &AnomalyDetector{
baselines: make(map[string]*Baseline),
},
correlationAnalyzer: &CorrelationAnalyzer{
correlationMatrix: make(map[string]map[string]float64),
},
currentTrends: make(map[string]Trend),
detectedAnomalies: make([]Anomaly, 0),
correlations: make(map[string]float64),
}
}
// AnalyzeTrends 分析趋势
func (p *PerformanceAnalyzer) AnalyzeTrends(history []MetricSnapshot) {
if len(history) < 3 {
return // 数据点太少，无法分析趋势
}
p.mutex.Lock()
defer p.mutex.Unlock()
// 分析各个指标的趋势
metrics := map[string][]float64{
"cpu_usage": make([]float64, len(history)),
"memory_usage": make([]float64, len(history)),
"response_time": make([]float64, len(history)),
"cache_hit_rate": make([]float64, len(history)),
"overall_performance": make([]float64, len(history)),
}
timestamps := make([]time.Time, len(history))
// 提取时间序列数据
for i, snapshot := range history {
metrics["cpu_usage"][i] = snapshot.System.CPUUsage
metrics["memory_usage"][i] = float64(snapshot.System.MemoryUsage) / 1024 / 1024 // MB
metrics["response_time"][i] = float64(snapshot.Application.ResponseTime) / float64(time.Millisecond)
metrics["cache_hit_rate"][i] = snapshot.Cache.HitRate
metrics["overall_performance"][i] = snapshot.OverallPerformance
timestamps[i] = snapshot.Timestamp
}
// 分析每个指标的趋势
for metricName, values := range metrics {
trend := p.calculateTrend(metricName, values, timestamps)
p.currentTrends[metricName] = trend
}
}
// calculateTrend 计算趋势
func (p *PerformanceAnalyzer) calculateTrend(metricName string, values []float64, timestamps []time.Time) Trend {
if len(values) < 2 {
return Trend{
Metric: metricName,
Direction: "stable",
Slope: 0,
Confidence: 0,
Duration: 0,
Prediction: values[len(values)-1],
}
}
// 线性回归计算趋势
slope, intercept, rSquared := p.linearRegression(values, timestamps)
// 确定趋势方向
direction := "stable"
if math.Abs(slope) > 0.01 { // 阈值
if slope > 0 {
direction = "increasing"
} else {
direction = "decreasing"
}
}
// 计算置信度
confidence := math.Min(rSquared, 1.0)
// 预测未来值
futureTime := timestamps[len(timestamps)-1].Add(5 * time.Minute)
prediction := intercept + slope*float64(futureTime.Unix())
return Trend{
Metric: metricName,
Direction: direction,
Slope: slope,
Confidence: confidence,
Duration: timestamps[len(timestamps)-1].Sub(timestamps[0]),
Prediction: prediction,
}
}
// linearRegression 线性回归
func (p *PerformanceAnalyzer) linearRegression(y []float64, timestamps []time.Time) (slope, intercept, rSquared float64) {
n := float64(len(y))
if n < 2 {
return 0, y[0], 0
}
// 转换时间戳为数值
x := make([]float64, len(timestamps))
for i, t := range timestamps {
x[i] = float64(t.Unix())
}
// 计算均值
var sumX, sumY float64
for i := 0; i < len(x); i++ {
sumX += x[i]
sumY += y[i]
}
meanX := sumX / n
meanY := sumY / n
// 计算斜率和截距
var numerator, denominator float64
for i := 0; i < len(x); i++ {
numerator += (x[i] - meanX) * (y[i] - meanY)
denominator += (x[i] - meanX) * (x[i] - meanX)
}
if denominator == 0 {
return 0, meanY, 0
}
slope = numerator / denominator
intercept = meanY - slope*meanX
// 计算R²
var ssRes, ssTot float64
for i := 0; i < len(y); i++ {
predicted := intercept + slope*x[i]
ssRes += (y[i] - predicted) * (y[i] - predicted)
ssTot += (y[i] - meanY) * (y[i] - meanY)
}
if ssTot == 0 {
rSquared = 1.0
} else {
rSquared = 1.0 - ssRes/ssTot
}
return slope, intercept, math.Max(0, rSquared)
}
// DetectAnomalies 检测异常
func (p *PerformanceAnalyzer) DetectAnomalies(currentMetrics *MetricSnapshot) {
if currentMetrics == nil {
return
}
p.mutex.Lock()
defer p.mutex.Unlock()
// 清空之前的异常
p.detectedAnomalies = make([]Anomaly, 0)
// 检测各个指标的异常
metrics := map[string]float64{
"cpu_usage": currentMetrics.System.CPUUsage,
"memory_usage": float64(currentMetrics.System.MemoryUsage) / 1024 / 1024,
"response_time": float64(currentMetrics.Application.ResponseTime) / float64(time.Millisecond),
"error_rate": currentMetrics.Application.ErrorRate,
"cache_hit_rate": currentMetrics.Cache.HitRate,
"overall_performance": currentMetrics.OverallPerformance,
}
for metricName, value := range metrics {
anomaly := p.detectMetricAnomaly(metricName, value, currentMetrics.Timestamp)
if anomaly != nil {
p.detectedAnomalies = append(p.detectedAnomalies, *anomaly)
}
}
}
// detectMetricAnomaly 检测单个指标异常
func (p *PerformanceAnalyzer) detectMetricAnomaly(metricName string, value float64, timestamp time.Time) *Anomaly {
baseline := p.anomalyDetector.baselines[metricName]
if baseline == nil {
// 创建新基线
p.anomalyDetector.baselines[metricName] = &Baseline{
Mean: value,
StdDev: 0,
Min: value,
Max: value,
SampleSize: 1,
LastUpdate: timestamp,
}
return nil
}
// 更新基线
p.updateBaseline(baseline, value, timestamp)
// 使用3-sigma规则检测异常
if baseline.StdDev > 0 {
zScore := math.Abs(value-baseline.Mean) / baseline.StdDev
var severity string
var impact float64
if zScore > 3.0 {
severity = "high"
impact = 0.8
} else if zScore > 2.0 {
severity = "medium"
impact = 0.5
} else if zScore > 1.5 {
severity = "low"
impact = 0.2
} else {
return nil // 无异常
}
// 确定异常描述
description := fmt.Sprintf("%s异常: 当前值%.2f, 期望范围[%.2f, %.2f]",
metricName, value,
baseline.Mean-2*baseline.StdDev,
baseline.Mean+2*baseline.StdDev)
return &Anomaly{
Metric: metricName,
Timestamp: timestamp,
Severity: severity,
Value: value,
ExpectedRange: [2]float64{baseline.Mean - 2*baseline.StdDev, baseline.Mean + 2*baseline.StdDev},
Description: description,
Impact: impact,
}
}
return nil
}
// updateBaseline 更新基线
func (p *PerformanceAnalyzer) updateBaseline(baseline *Baseline, newValue float64, timestamp time.Time) {
// 增量更新均值和标准差
oldMean := baseline.Mean
baseline.SampleSize++
baseline.Mean += (newValue - baseline.Mean) / float64(baseline.SampleSize)
// 更新方差（Welford算法）
if baseline.SampleSize > 1 {
variance := (float64(baseline.SampleSize-2)*baseline.StdDev*baseline.StdDev +
(newValue-oldMean)*(newValue-baseline.Mean)) / float64(baseline.SampleSize-1)
baseline.StdDev = math.Sqrt(math.Max(0, variance))
}
// 更新最值
if newValue < baseline.Min {
baseline.Min = newValue
}
if newValue > baseline.Max {
baseline.Max = newValue
}
baseline.LastUpdate = timestamp
}
// AnalyzeCorrelations 分析相关性
func (p *PerformanceAnalyzer) AnalyzeCorrelations(history []MetricSnapshot) {
if len(history) < 3 {
return
}
p.mutex.Lock()
defer p.mutex.Unlock()
// 提取指标数据
metrics := map[string][]float64{
"cpu_usage": make([]float64, len(history)),
"memory_usage": make([]float64, len(history)),
"response_time": make([]float64, len(history)),
"cache_hit_rate": make([]float64, len(history)),
"overall_performance": make([]float64, len(history)),
}
for i, snapshot := range history {
metrics["cpu_usage"][i] = snapshot.System.CPUUsage
metrics["memory_usage"][i] = float64(snapshot.System.MemoryUsage) / 1024 / 1024
metrics["response_time"][i] = float64(snapshot.Application.ResponseTime) / float64(time.Millisecond)
metrics["cache_hit_rate"][i] = snapshot.Cache.HitRate
metrics["overall_performance"][i] = snapshot.OverallPerformance
}
// 计算相关性矩阵
metricNames := make([]string, 0, len(metrics))
for name := range metrics {
metricNames = append(metricNames, name)
}
sort.Strings(metricNames)
for i, metric1 := range metricNames {
if p.correlationAnalyzer.correlationMatrix[metric1] == nil {
p.correlationAnalyzer.correlationMatrix[metric1] = make(map[string]float64)
}
for j, metric2 := range metricNames {
if i <= j {
correlation := p.calculateCorrelation(metrics[metric1], metrics[metric2])
p.correlationAnalyzer.correlationMatrix[metric1][metric2] = correlation
p.correlationAnalyzer.correlationMatrix[metric2][metric1] = correlation
// 保存重要相关性
if math.Abs(correlation) > 0.5 && metric1 != metric2 {
p.correlations[fmt.Sprintf("%s_%s", metric1, metric2)] = correlation
}
}
}
}
}
// calculateCorrelation 计算皮尔逊相关系数
func (p *PerformanceAnalyzer) calculateCorrelation(x, y []float64) float64 {
if len(x) != len(y) || len(x) < 2 {
return 0
}
n := float64(len(x))
// 计算均值
var sumX, sumY float64
for i := 0; i < len(x); i++ {
sumX += x[i]
sumY += y[i]
}
meanX := sumX / n
meanY := sumY / n
// 计算协方差和方差
var covariance, varianceX, varianceY float64
for i := 0; i < len(x); i++ {
dx := x[i] - meanX
dy := y[i] - meanY
covariance += dx * dy
varianceX += dx * dx
varianceY += dy * dy
}
// 计算相关系数
if varianceX == 0 || varianceY == 0 {
return 0
}
correlation := covariance / math.Sqrt(varianceX*varianceY)
return correlation
}
// AnalyzeIssues 分析性能问题
func (p *PerformanceAnalyzer) AnalyzeIssues(currentMetrics *MetricSnapshot) []string {
if currentMetrics == nil {
return nil
}
issues := make([]string, 0)
// CPU使用率过高
if currentMetrics.System.CPUUsage > 0.8 {
issues = append(issues, "high_cpu_usage")
}
// 内存使用率过高
memoryUsageRatio := float64(currentMetrics.System.MemoryUsage) / float64(currentMetrics.System.MemoryTotal)
if memoryUsageRatio > 0.85 {
issues = append(issues, "high_memory_usage")
}
// 响应时间过长
if currentMetrics.Application.ResponseTime > 1*time.Second {
issues = append(issues, "high_response_time")
}
// 错误率过高
if currentMetrics.Application.ErrorRate > 0.05 {
issues = append(issues, "high_error_rate")
}
// 缓存命中率过低
if currentMetrics.Cache.HitRate < 0.7 {
issues = append(issues, "low_cache_hit_rate")
}
// 整体性能过低
if currentMetrics.OverallPerformance < 60 {
issues = append(issues, "low_overall_performance")
}
return issues
}
// GetCurrentTrends 获取当前趋势
func (p *PerformanceAnalyzer) GetCurrentTrends() map[string]Trend {
p.mutex.RLock()
defer p.mutex.RUnlock()
trends := make(map[string]Trend)
for k, v := range p.currentTrends {
trends[k] = v
}
return trends
}
// GetDetectedAnomalies 获取检测到的异常
func (p *PerformanceAnalyzer) GetDetectedAnomalies() []Anomaly {
p.mutex.RLock()
defer p.mutex.RUnlock()
anomalies := make([]Anomaly, len(p.detectedAnomalies))
copy(anomalies, p.detectedAnomalies)
return anomalies
}
// GetCorrelations 获取相关性
func (p *PerformanceAnalyzer) GetCorrelations() map[string]float64 {
p.mutex.RLock()
defer p.mutex.RUnlock()
correlations := make(map[string]float64)
for k, v := range p.correlations {
correlations[k] = v
}
return correlations
}

396
util/cache/predictive_model.go vendored
View File

@@ -1,396 +0,0 @@
package cache
import (
"fmt"
"math"
"time"
)
// NewPredictiveModel 创建预测模型
func NewPredictiveModel() *PredictiveModel {
return &PredictiveModel{
modelType: "linear_regression",
coefficients: make([]float64, 0),
seasonalFactors: make([]float64, 0),
trainingData: make([]DataPoint, 0),
testData: make([]DataPoint, 0),
predictions: make(map[string]Prediction),
}
}
// Train 训练模型
func (p *PredictiveModel) Train(dataset *LearningDataset) error {
p.mutex.Lock()
defer p.mutex.Unlock()
if dataset == nil || len(dataset.Features) == 0 {
return fmt.Errorf("训练数据集为空")
}
// 准备训练数据
p.prepareTrainingData(dataset)
// 根据模型类型进行训练
switch p.modelType {
case "linear_regression":
return p.trainLinearRegression(dataset)
case "exponential_smoothing":
return p.trainExponentialSmoothing(dataset)
default:
return fmt.Errorf("不支持的模型类型: %s", p.modelType)
}
}
// prepareTrainingData 准备训练数据
func (p *PredictiveModel) prepareTrainingData(dataset *LearningDataset) {
dataset.mutex.RLock()
defer dataset.mutex.RUnlock()
totalSamples := len(dataset.Features)
if totalSamples == 0 {
return
}
// 数据分割
trainSize := int(float64(totalSamples) * 0.8) // 80%用于训练
p.trainingData = make([]DataPoint, trainSize)
p.testData = make([]DataPoint, totalSamples-trainSize)
// 填充训练数据
for i := 0; i < trainSize; i++ {
p.trainingData[i] = DataPoint{
Timestamp: time.Now().Add(-time.Duration(totalSamples-i) * time.Minute),
Values: make(map[string]float64),
}
// 转换特征为命名值
if len(dataset.Features[i]) >= 5 {
p.trainingData[i].Values["cpu_usage"] = dataset.Features[i][0]
p.trainingData[i].Values["memory_usage"] = dataset.Features[i][1]
p.trainingData[i].Values["response_time"] = dataset.Features[i][2]
p.trainingData[i].Values["cache_hit_rate"] = dataset.Features[i][3]
p.trainingData[i].Values["compression_ratio"] = dataset.Features[i][4]
}
}
// 填充测试数据
for i := 0; i < len(p.testData); i++ {
testIndex := trainSize + i
p.testData[i] = DataPoint{
Timestamp: time.Now().Add(-time.Duration(totalSamples-testIndex) * time.Minute),
Values: make(map[string]float64),
}
if len(dataset.Features[testIndex]) >= 5 {
p.testData[i].Values["cpu_usage"] = dataset.Features[testIndex][0]
p.testData[i].Values["memory_usage"] = dataset.Features[testIndex][1]
p.testData[i].Values["response_time"] = dataset.Features[testIndex][2]
p.testData[i].Values["cache_hit_rate"] = dataset.Features[testIndex][3]
p.testData[i].Values["compression_ratio"] = dataset.Features[testIndex][4]
}
}
}
// trainLinearRegression 训练线性回归模型
func (p *PredictiveModel) trainLinearRegression(dataset *LearningDataset) error {
dataset.mutex.RLock()
defer dataset.mutex.RUnlock()
if len(dataset.Features) == 0 || len(dataset.Labels) != len(dataset.Features) {
return fmt.Errorf("训练数据不匹配")
}
featuresCount := len(dataset.Features[0])
samplesCount := len(dataset.Features)
// 初始化系数（包括偏置项）
p.coefficients = make([]float64, featuresCount+1)
// 使用梯度下降训练
learningRate := 0.01
iterations := 1000
for iter := 0; iter < iterations; iter++ {
// 计算预测值和误差
totalLoss := 0.0
gradients := make([]float64, len(p.coefficients))
for i := 0; i < samplesCount; i++ {
// 计算预测值
predicted := p.coefficients[0] // 偏置项
for j := 0; j < featuresCount; j++ {
predicted += p.coefficients[j+1] * dataset.Features[i][j]
}
// 计算误差
error := predicted - dataset.Labels[i]
totalLoss += error * error
// 计算梯度
gradients[0] += error // 偏置项梯度
for j := 0; j < featuresCount; j++ {
gradients[j+1] += error * dataset.Features[i][j]
}
}
// 更新参数
for j := 0; j < len(p.coefficients); j++ {
p.coefficients[j] -= learningRate * gradients[j] / float64(samplesCount)
}
// 计算平均损失
avgLoss := totalLoss / float64(samplesCount)
// 早停条件
if avgLoss < 0.001 {
break
}
}
return nil
}
// trainExponentialSmoothing 训练指数平滑模型
func (p *PredictiveModel) trainExponentialSmoothing(dataset *LearningDataset) error {
dataset.mutex.RLock()
defer dataset.mutex.RUnlock()
if len(dataset.Labels) < 2 {
return fmt.Errorf("指数平滑需要至少2个数据点")
}
// 简单指数平滑参数
alpha := 0.3 // 平滑参数
// 初始化
p.coefficients = make([]float64, 2)
p.coefficients[0] = dataset.Labels[0] // 初始水平
p.coefficients[1] = alpha // 平滑参数
// 计算趋势组件
if len(dataset.Labels) > 1 {
p.trendComponent = dataset.Labels[1] - dataset.Labels[0]
}
return nil
}
// Predict 进行预测
func (p *PredictiveModel) Predict(features []float64, horizon time.Duration) (*Prediction, error) {
p.mutex.RLock()
defer p.mutex.RUnlock()
if len(p.coefficients) == 0 {
return nil, fmt.Errorf("模型尚未训练")
}
var predictedValue float64
var confidence float64
switch p.modelType {
case "linear_regression":
if len(features) != len(p.coefficients)-1 {
return nil, fmt.Errorf("特征维度不匹配")
}
// 线性回归预测
predictedValue = p.coefficients[0] // 偏置项
for i, feature := range features {
predictedValue += p.coefficients[i+1] * feature
}
// 置信度基于训练数据的拟合程度
confidence = math.Max(0.5, p.rmse) // 简化的置信度计算
case "exponential_smoothing":
// 指数平滑预测
predictedValue = p.coefficients[0] + p.trendComponent*float64(horizon/time.Minute)
confidence = 0.7 // 固定置信度
default:
return nil, fmt.Errorf("不支持的模型类型: %s", p.modelType)
}
return &Prediction{
Metric: "overall_performance",
FutureValue: predictedValue,
Confidence: confidence,
TimeHorizon: horizon,
PredictedAt: time.Now(),
}, nil
}
// Validate 验证模型性能
func (p *PredictiveModel) Validate() float64 {
p.mutex.Lock()
defer p.mutex.Unlock()
if len(p.testData) == 0 {
return 0
}
correctPredictions := 0
totalPredictions := len(p.testData)
for _, testPoint := range p.testData {
// 提取特征
features := []float64{
testPoint.Values["cpu_usage"],
testPoint.Values["memory_usage"],
testPoint.Values["response_time"],
testPoint.Values["cache_hit_rate"],
testPoint.Values["compression_ratio"],
}
// 进行预测
prediction, err := p.Predict(features, 5*time.Minute)
if err != nil {
continue
}
// 计算实际性能分数（简化）
actualPerformance := p.calculatePerformanceScore(testPoint.Values)
// 判断预测是否准确（容忍误差10%）
errorRate := math.Abs(prediction.FutureValue-actualPerformance) / actualPerformance
if errorRate < 0.1 {
correctPredictions++
}
}
accuracy := float64(correctPredictions) / float64(totalPredictions)
p.accuracy = accuracy
// 计算RMSE
p.calculateRMSE()
return accuracy
}
// calculatePerformanceScore 计算性能分数
func (p *PredictiveModel) calculatePerformanceScore(values map[string]float64) float64 {
// 简化的性能分数计算
cpuScore := (1.0 - values["cpu_usage"]) * 30
memoryScore := (1.0 - values["memory_usage"]/1000) * 25 // 假设内存以MB为单位
responseScore := (1.0 - values["response_time"]/1000) * 25 // 假设响应时间以ms为单位
cacheScore := values["cache_hit_rate"] * 20
return math.Max(0, cpuScore+memoryScore+responseScore+cacheScore)
}
// calculateRMSE 计算均方根误差
func (p *PredictiveModel) calculateRMSE() {
if len(p.testData) == 0 {
return
}
var sumSquaredErrors float64
validPredictions := 0
for _, testPoint := range p.testData {
features := []float64{
testPoint.Values["cpu_usage"],
testPoint.Values["memory_usage"],
testPoint.Values["response_time"],
testPoint.Values["cache_hit_rate"],
testPoint.Values["compression_ratio"],
}
prediction, err := p.Predict(features, 5*time.Minute)
if err != nil {
continue
}
actualPerformance := p.calculatePerformanceScore(testPoint.Values)
error := prediction.FutureValue - actualPerformance
sumSquaredErrors += error * error
validPredictions++
}
if validPredictions > 0 {
p.rmse = math.Sqrt(sumSquaredErrors / float64(validPredictions))
}
}
// PredictMultiple 预测多个指标
func (p *PredictiveModel) PredictMultiple(currentMetrics *MetricSnapshot, horizons []time.Duration) map[string]Prediction {
if currentMetrics == nil {
return nil
}
features := []float64{
currentMetrics.System.CPUUsage,
float64(currentMetrics.System.MemoryUsage) / 1024 / 1024,
float64(currentMetrics.Application.ResponseTime) / float64(time.Millisecond),
currentMetrics.Cache.HitRate,
currentMetrics.Cache.CompressionRatio,
}
predictions := make(map[string]Prediction)
for _, horizon := range horizons {
predictionKey := fmt.Sprintf("performance_%s", horizon.String())
prediction, err := p.Predict(features, horizon)
if err == nil {
predictions[predictionKey] = *prediction
}
}
return predictions
}
// GetAccuracy 获取模型准确率
func (p *PredictiveModel) GetAccuracy() float64 {
p.mutex.RLock()
defer p.mutex.RUnlock()
return p.accuracy
}
// GetPredictions 获取所有预测结果
func (p *PredictiveModel) GetPredictions() map[string]Prediction {
p.mutex.RLock()
defer p.mutex.RUnlock()
predictions := make(map[string]Prediction)
for k, v := range p.predictions {
predictions[k] = v
}
return predictions
}
// UpdatePredictions 更新预测结果
func (p *PredictiveModel) UpdatePredictions(currentMetrics *MetricSnapshot) {
if currentMetrics == nil {
return
}
// 预测未来1小时的性能
horizons := []time.Duration{
5 * time.Minute,
15 * time.Minute,
30 * time.Minute,
1 * time.Hour,
}
newPredictions := p.PredictMultiple(currentMetrics, horizons)
p.mutex.Lock()
defer p.mutex.Unlock()
// 更新预测结果
for k, v := range newPredictions {
p.predictions[k] = v
}
// 清理过期预测
now := time.Now()
for k, v := range p.predictions {
if now.Sub(v.PredictedAt) > 2*time.Hour {
delete(p.predictions, k)
}
}
}

431
util/cache/search_pattern_analyzer.go vendored
View File

@@ -1,431 +0,0 @@
package cache
import (
"crypto/md5"
"fmt"
"regexp"
"sort"
"strings"
"sync"
"time"
)
// SearchPatternAnalyzer 搜索模式分析器
type SearchPatternAnalyzer struct {
// 模式缓存
patternCache map[string]*SearchPattern
cacheMutex sync.RWMutex
// 分析规则
keywordRules []*KeywordRule
// 统计信息
analysisCount int64
cacheHitCount int64
// 配置
maxCacheSize int
cacheExpiry time.Duration
}
// KeywordRule 关键词规则
type KeywordRule struct {
Name string
Pattern *regexp.Regexp
Priority int
Description string
}
// NewSearchPatternAnalyzer 创建搜索模式分析器
func NewSearchPatternAnalyzer() *SearchPatternAnalyzer {
analyzer := &SearchPatternAnalyzer{
patternCache: make(map[string]*SearchPattern),
maxCacheSize: 1000, // 最大缓存1000个模式
cacheExpiry: 1 * time.Hour, // 1小时过期
}
// 初始化关键词规则
analyzer.initializeKeywordRules()
return analyzer
}
// initializeKeywordRules 初始化关键词规则
func (s *SearchPatternAnalyzer) initializeKeywordRules() {
s.keywordRules = []*KeywordRule{
{
Name: "电影资源",
Pattern: regexp.MustCompile(`(?i)(电影|movie|film|影片|HD|4K|蓝光|BluRay)`),
Priority: 1,
Description: "电影相关搜索",
},
{
Name: "电视剧资源",
Pattern: regexp.MustCompile(`(?i)(电视剧|TV|series|连续剧|美剧|韩剧|日剧)`),
Priority: 1,
Description: "电视剧相关搜索",
},
{
Name: "动漫资源",
Pattern: regexp.MustCompile(`(?i)(动漫|anime|动画|漫画|manga)`),
Priority: 1,
Description: "动漫相关搜索",
},
{
Name: "音乐资源",
Pattern: regexp.MustCompile(`(?i)(音乐|music|歌曲|专辑|album|MP3|FLAC)`),
Priority: 2,
Description: "音乐相关搜索",
},
{
Name: "游戏资源",
Pattern: regexp.MustCompile(`(?i)(游戏|game|单机|网游|手游|steam)`),
Priority: 2,
Description: "游戏相关搜索",
},
{
Name: "软件资源",
Pattern: regexp.MustCompile(`(?i)(软件|software|app|应用|工具|破解)`),
Priority: 2,
Description: "软件相关搜索",
},
{
Name: "学习资源",
Pattern: regexp.MustCompile(`(?i)(教程|tutorial|课程|学习|教学|资料)`),
Priority: 3,
Description: "学习资源搜索",
},
{
Name: "文档资源",
Pattern: regexp.MustCompile(`(?i)(文档|doc|pdf|txt|电子书|ebook)`),
Priority: 3,
Description: "文档资源搜索",
},
{
Name: "通用搜索",
Pattern: regexp.MustCompile(`.*`), // 匹配所有
Priority: 4,
Description: "通用搜索模式",
},
}
}
// AnalyzePattern 分析搜索模式
func (s *SearchPatternAnalyzer) AnalyzePattern(op *CacheOperation) *SearchPattern {
s.analysisCount++
// 🔧 生成缓存键
cacheKey := s.generateCacheKey(op)
// 🚀 检查缓存
s.cacheMutex.RLock()
if cached, exists := s.patternCache[cacheKey]; exists {
// 检查是否过期
if time.Since(cached.LastAccessTime) < s.cacheExpiry {
cached.LastAccessTime = time.Now()
cached.Frequency++
s.cacheMutex.RUnlock()
s.cacheHitCount++
return cached
}
}
s.cacheMutex.RUnlock()
// 🎯 分析新模式
pattern := s.analyzeNewPattern(op)
// 🗄️ 缓存结果
s.cachePattern(cacheKey, pattern)
return pattern
}
// generateCacheKey 生成缓存键
func (s *SearchPatternAnalyzer) generateCacheKey(op *CacheOperation) string {
// 使用关键词和插件名生成缓存键
source := fmt.Sprintf("%s_%s",
s.normalizeKeyword(op.Keyword),
op.PluginName)
// MD5哈希以节省内存
hash := md5.Sum([]byte(source))
return fmt.Sprintf("%x", hash)
}
// normalizeKeyword 标准化关键词
func (s *SearchPatternAnalyzer) normalizeKeyword(keyword string) string {
// 转换为小写
normalized := strings.ToLower(keyword)
// 移除特殊字符和多余空格
normalized = regexp.MustCompile(`[^\w\s\u4e00-\u9fff]`).ReplaceAllString(normalized, " ")
normalized = regexp.MustCompile(`\s+`).ReplaceAllString(normalized, " ")
normalized = strings.TrimSpace(normalized)
return normalized
}
// analyzeNewPattern 分析新模式
func (s *SearchPatternAnalyzer) analyzeNewPattern(op *CacheOperation) *SearchPattern {
pattern := &SearchPattern{
KeywordPattern: s.classifyKeyword(op.Keyword),
PluginSet: []string{op.PluginName},
TimeWindow: s.determineTimeWindow(op),
Frequency: 1,
LastAccessTime: time.Now(),
Metadata: make(map[string]interface{}),
}
// 🔍 关键词分析
s.analyzeKeywordCharacteristics(pattern, op.Keyword)
// 🔍 插件分析
s.analyzePluginCharacteristics(pattern, op.PluginName)
// 🔍 时间模式分析
s.analyzeTimePattern(pattern, op.Timestamp)
return pattern
}
// classifyKeyword 分类关键词
func (s *SearchPatternAnalyzer) classifyKeyword(keyword string) string {
// 按优先级检查规则
for _, rule := range s.keywordRules {
if rule.Pattern.MatchString(keyword) {
return rule.Name
}
}
return "通用搜索"
}
// analyzeKeywordCharacteristics 分析关键词特征
func (s *SearchPatternAnalyzer) analyzeKeywordCharacteristics(pattern *SearchPattern, keyword string) {
metadata := pattern.Metadata
// 分析关键词长度
metadata["keyword_length"] = len(keyword)
// 分析关键词复杂度（包含的词数）
words := strings.Fields(keyword)
metadata["word_count"] = len(words)
// 分析是否包含特殊字符
hasSpecialChars := regexp.MustCompile(`[^\w\s\u4e00-\u9fff]`).MatchString(keyword)
metadata["has_special_chars"] = hasSpecialChars
// 分析是否包含数字
hasNumbers := regexp.MustCompile(`\d`).MatchString(keyword)
metadata["has_numbers"] = hasNumbers
// 分析语言类型
hasChinese := regexp.MustCompile(`[\u4e00-\u9fff]`).MatchString(keyword)
hasEnglish := regexp.MustCompile(`[a-zA-Z]`).MatchString(keyword)
if hasChinese && hasEnglish {
metadata["language"] = "mixed"
} else if hasChinese {
metadata["language"] = "chinese"
} else if hasEnglish {
metadata["language"] = "english"
} else {
metadata["language"] = "other"
}
// 预测搜索频率（基于关键词特征）
complexity := len(words)
if hasSpecialChars {
complexity++
}
if hasNumbers {
complexity++
}
// 复杂度越低，搜索频率可能越高
predictedFrequency := "medium"
if complexity <= 2 {
predictedFrequency = "high"
} else if complexity >= 5 {
predictedFrequency = "low"
}
metadata["predicted_frequency"] = predictedFrequency
}
// analyzePluginCharacteristics 分析插件特征
func (s *SearchPatternAnalyzer) analyzePluginCharacteristics(pattern *SearchPattern, pluginName string) {
metadata := pattern.Metadata
// 插件类型分析（基于名称推断）
pluginType := "general"
if strings.Contains(strings.ToLower(pluginName), "4k") {
pluginType = "high_quality"
} else if strings.Contains(strings.ToLower(pluginName), "pan") {
pluginType = "cloud_storage"
} else if strings.Contains(strings.ToLower(pluginName), "search") {
pluginType = "search_engine"
}
metadata["plugin_type"] = pluginType
metadata["plugin_name"] = pluginName
}
// analyzeTimePattern 分析时间模式
func (s *SearchPatternAnalyzer) analyzeTimePattern(pattern *SearchPattern, timestamp time.Time) {
metadata := pattern.Metadata
// 时间段分析
hour := timestamp.Hour()
var timePeriod string
switch {
case hour >= 6 && hour < 12:
timePeriod = "morning"
case hour >= 12 && hour < 18:
timePeriod = "afternoon"
case hour >= 18 && hour < 22:
timePeriod = "evening"
default:
timePeriod = "night"
}
metadata["time_period"] = timePeriod
// 工作日/周末分析
weekday := timestamp.Weekday()
isWeekend := weekday == time.Saturday || weekday == time.Sunday
metadata["is_weekend"] = isWeekend
// 预测最佳缓存时间（基于时间模式）
if isWeekend || timePeriod == "evening" {
pattern.TimeWindow = 30 * time.Minute // 高峰期，较长缓存
} else {
pattern.TimeWindow = 15 * time.Minute // 非高峰期，较短缓存
}
}
// determineTimeWindow 确定时间窗口
func (s *SearchPatternAnalyzer) determineTimeWindow(op *CacheOperation) time.Duration {
// 基本时间窗口：15分钟
baseWindow := 15 * time.Minute
// 根据优先级调整
switch op.Priority {
case 1: // 高优先级插件
return baseWindow * 2 // 30分钟
case 2: // 中高优先级插件
return baseWindow * 3 / 2 // 22.5分钟
case 3: // 中等优先级插件
return baseWindow // 15分钟
case 4: // 低优先级插件
return baseWindow / 2 // 7.5分钟
default:
return baseWindow
}
}
// cachePattern 缓存模式
func (s *SearchPatternAnalyzer) cachePattern(cacheKey string, pattern *SearchPattern) {
s.cacheMutex.Lock()
defer s.cacheMutex.Unlock()
// 检查缓存大小，必要时清理
if len(s.patternCache) >= s.maxCacheSize {
s.cleanupCache()
}
s.patternCache[cacheKey] = pattern
}
// cleanupCache 清理缓存
func (s *SearchPatternAnalyzer) cleanupCache() {
now := time.Now()
// 收集需要删除的键
toDelete := make([]string, 0)
for key, pattern := range s.patternCache {
if now.Sub(pattern.LastAccessTime) > s.cacheExpiry {
toDelete = append(toDelete, key)
}
}
// 如果过期删除不够，按使用频率删除
if len(toDelete) < len(s.patternCache)/4 { // 删除不到25%
// 按频率排序，删除使用频率最低的
type patternFreq struct {
key string
frequency int
lastAccess time.Time
}
patterns := make([]patternFreq, 0, len(s.patternCache))
for key, pattern := range s.patternCache {
patterns = append(patterns, patternFreq{
key: key,
frequency: pattern.Frequency,
lastAccess: pattern.LastAccessTime,
})
}
// 按频率排序（频率低的在前）
sort.Slice(patterns, func(i, j int) bool {
if patterns[i].frequency == patterns[j].frequency {
return patterns[i].lastAccess.Before(patterns[j].lastAccess)
}
return patterns[i].frequency < patterns[j].frequency
})
// 删除前25%
deleteCount := len(patterns) / 4
for i := 0; i < deleteCount; i++ {
toDelete = append(toDelete, patterns[i].key)
}
}
// 执行删除
for _, key := range toDelete {
delete(s.patternCache, key)
}
}
// GetCacheStats 获取缓存统计
func (s *SearchPatternAnalyzer) GetCacheStats() map[string]interface{} {
s.cacheMutex.RLock()
defer s.cacheMutex.RUnlock()
hitRate := float64(0)
if s.analysisCount > 0 {
hitRate = float64(s.cacheHitCount) / float64(s.analysisCount)
}
return map[string]interface{}{
"cache_size": len(s.patternCache),
"max_cache_size": s.maxCacheSize,
"analysis_count": s.analysisCount,
"cache_hit_count": s.cacheHitCount,
"hit_rate": hitRate,
"cache_expiry": s.cacheExpiry,
}
}
// GetPopularPatterns 获取热门模式
func (s *SearchPatternAnalyzer) GetPopularPatterns(limit int) []*SearchPattern {
s.cacheMutex.RLock()
defer s.cacheMutex.RUnlock()
patterns := make([]*SearchPattern, 0, len(s.patternCache))
for _, pattern := range s.patternCache {
patterns = append(patterns, pattern)
}
// 按频率排序
sort.Slice(patterns, func(i, j int) bool {
return patterns[i].Frequency > patterns[j].Frequency
})
if limit > 0 && limit < len(patterns) {
patterns = patterns[:limit]
}
return patterns
}

14
util/cache/sharded_memory_cache.go vendored
View File

@@ -8,7 +8,7 @@ import (
"time"
)
// 🔥 全局清理任务相关变量（单例模式）
// 全局清理任务相关变量（单例模式）
var (
globalCleanupTicker *time.Ticker
globalCleanupOnce sync.Once
@@ -16,7 +16,7 @@ var (
cacheRegistryMutex sync.RWMutex
)
// 🔥 清理目标接口
// 清理目标接口
type cleanupTarget interface {
CleanExpired()
}
@@ -45,8 +45,8 @@ type ShardedMemoryCache struct {
maxSize int64
itemsPerShard int
sizePerShard int64
diskCache *ShardedDiskCache // 🔥 新增：磁盘缓存引用
diskCacheMutex sync.RWMutex // 🔥 新增：磁盘缓存引用的保护锁
diskCache *ShardedDiskCache // 磁盘缓存引用
diskCacheMutex sync.RWMutex // 磁盘缓存引用的保护锁
}
// 创建新的分片内存缓存
@@ -237,7 +237,7 @@ func (c *ShardedMemoryCache) evictFromShard(shard *memoryCacheShard) {
go func(key string, data []byte, expiry time.Time) {
ttl := time.Until(expiry)
if ttl > 0 {
diskCache.Set(key, data, ttl) // 🔥 保持相同TTL
diskCache.Set(key, data, ttl) // 保持相同TTL
}
}(oldestKey, oldestItem.data, oldestItem.expiry)
}
@@ -302,7 +302,7 @@ func (c *ShardedMemoryCache) Clear() {
wg.Wait()
}
// 🔥 启动全局清理任务（单例模式）
// 启动全局清理任务（单例模式）
func startGlobalCleanupTask() {
globalCleanupOnce.Do(func() {
globalCleanupTicker = time.NewTicker(5 * time.Minute)
@@ -322,7 +322,7 @@ func startGlobalCleanupTask() {
})
}
// 🔥 注册缓存到全局清理任务
// 注册缓存到全局清理任务
func registerForCleanup(cache cleanupTarget) {
cacheRegistryMutex.Lock()
defer cacheRegistryMutex.Unlock()

463
util/cache/tuning_strategy.go vendored
View File

@@ -1,463 +0,0 @@
package cache
import (
"sort"
"time"
)
// NewTuningStrategy 创建调优策略
func NewTuningStrategy() *TuningStrategy {
strategy := &TuningStrategy{
strategyType: "balanced",
rules: make([]*TuningRule, 0),
parameterAdjustments: make(map[string]ParameterAdjustment),
executionHistory: make([]*StrategyExecution, 0),
}
// 初始化调优规则
strategy.initializeRules()
return strategy
}
// NewLearningDataset 创建学习数据集
func NewLearningDataset() *LearningDataset {
return &LearningDataset{
Features: make([][]float64, 0),
Labels: make([]float64, 0),
Weights: make([]float64, 0),
FeatureStats: make([]FeatureStatistics, 0),
TrainingSplit: 0.8,
ValidationSplit: 0.1,
TestSplit: 0.1,
}
}
// initializeRules 初始化调优规则
func (t *TuningStrategy) initializeRules() {
t.rules = []*TuningRule{
{
Name: "高CPU使用率调优",
Priority: 1,
Enabled: true,
Condition: func(metrics *MetricSnapshot) bool {
return metrics.System.CPUUsage > 0.8
},
Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
return t.createCPUOptimizationDecision(engine)
},
},
{
Name: "高内存使用调优",
Priority: 2,
Enabled: true,
Condition: func(metrics *MetricSnapshot) bool {
memoryRatio := float64(metrics.System.MemoryUsage) / float64(metrics.System.MemoryTotal)
return memoryRatio > 0.85
},
Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
return t.createMemoryOptimizationDecision(engine)
},
},
{
Name: "响应时间过长调优",
Priority: 3,
Enabled: true,
Condition: func(metrics *MetricSnapshot) bool {
return metrics.Application.ResponseTime > 500*time.Millisecond
},
Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
return t.createResponseTimeOptimizationDecision(engine)
},
},
{
Name: "缓存命中率低调优",
Priority: 4,
Enabled: true,
Condition: func(metrics *MetricSnapshot) bool {
return metrics.Cache.HitRate < 0.7
},
Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
return t.createCacheOptimizationDecision(engine)
},
},
{
Name: "整体性能低调优",
Priority: 5,
Enabled: true,
Condition: func(metrics *MetricSnapshot) bool {
return metrics.OverallPerformance < 60
},
Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
return t.createOverallPerformanceDecision(engine)
},
},
{
Name: "预防性调优",
Priority: 10,
Enabled: true,
Condition: func(metrics *MetricSnapshot) bool {
// 基于趋势的预防性调优
return false // 暂时禁用，需要趋势数据
},
Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
return t.createPreventiveDecision(engine)
},
},
}
}
// GenerateDecision 生成调优决策
func (t *TuningStrategy) GenerateDecision(metrics *MetricSnapshot, issues []string) *TuningDecision {
if metrics == nil {
return nil
}
t.mutex.Lock()
defer t.mutex.Unlock()
// 按优先级排序规则
sort.Slice(t.rules, func(i, j int) bool {
return t.rules[i].Priority < t.rules[j].Priority
})
// 检查规则并生成决策
for _, rule := range t.rules {
if !rule.Enabled {
continue
}
// 检查冷却时间（防止频繁调优）
if time.Since(rule.LastTriggered) < 5*time.Minute {
continue
}
// 检查条件
if rule.Condition(metrics) {
decision, err := rule.Action(nil) // 简化实现，不传递engine
if err != nil {
continue
}
// 更新规则状态
rule.LastTriggered = time.Now()
rule.TriggerCount++
// 设置决策基本信息
decision.Timestamp = time.Now()
decision.Trigger = rule.Name
return decision
}
}
return nil
}
// createCPUOptimizationDecision 创建CPU优化决策
func (t *TuningStrategy) createCPUOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
adjustments := []ParameterAdjustment{
{
ParameterName: "batch_interval",
CurrentValue: "60s",
ProposedValue: "90s",
AdjustmentRatio: 0.5, // 增加50%
Reason: "减少CPU负载",
ExpectedImpact: "降低CPU使用率",
Risk: "medium",
},
{
ParameterName: "batch_size",
CurrentValue: 100,
ProposedValue: 150,
AdjustmentRatio: 0.5,
Reason: "减少处理频率",
ExpectedImpact: "降低CPU负载",
Risk: "low",
},
}
return &TuningDecision{
Adjustments: adjustments,
Confidence: 0.8,
ExpectedImprovement: 0.15,
Risk: 0.3,
AutoExecute: true,
}, nil
}
// createMemoryOptimizationDecision 创建内存优化决策
func (t *TuningStrategy) createMemoryOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
adjustments := []ParameterAdjustment{
{
ParameterName: "max_buffer_size",
CurrentValue: 1000,
ProposedValue: 700,
AdjustmentRatio: -0.3, // 减少30%
Reason: "减少内存占用",
ExpectedImpact: "降低内存使用率",
Risk: "medium",
},
{
ParameterName: "cache_cleanup_frequency",
CurrentValue: "5m",
ProposedValue: "3m",
AdjustmentRatio: -0.4,
Reason: "更频繁清理缓存",
ExpectedImpact: "释放内存空间",
Risk: "low",
},
}
return &TuningDecision{
Adjustments: adjustments,
Confidence: 0.85,
ExpectedImprovement: 0.2,
Risk: 0.25,
AutoExecute: true,
}, nil
}
// createResponseTimeOptimizationDecision 创建响应时间优化决策
func (t *TuningStrategy) createResponseTimeOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
adjustments := []ParameterAdjustment{
{
ParameterName: "batch_interval",
CurrentValue: "60s",
ProposedValue: "30s",
AdjustmentRatio: -0.5, // 减少50%
Reason: "更快的数据写入",
ExpectedImpact: "降低响应时间",
Risk: "medium",
},
{
ParameterName: "concurrent_workers",
CurrentValue: 4,
ProposedValue: 6,
AdjustmentRatio: 0.5,
Reason: "增加并发处理",
ExpectedImpact: "提高处理速度",
Risk: "high",
},
}
return &TuningDecision{
Adjustments: adjustments,
Confidence: 0.75,
ExpectedImprovement: 0.25,
Risk: 0.4,
AutoExecute: false, // 高风险，需要手动确认
}, nil
}
// createCacheOptimizationDecision 创建缓存优化决策
func (t *TuningStrategy) createCacheOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
adjustments := []ParameterAdjustment{
{
ParameterName: "cache_ttl",
CurrentValue: "1h",
ProposedValue: "2h",
AdjustmentRatio: 1.0, // 增加100%
Reason: "延长缓存生存时间",
ExpectedImpact: "提高缓存命中率",
Risk: "low",
},
{
ParameterName: "cache_size_limit",
CurrentValue: 1000,
ProposedValue: 1500,
AdjustmentRatio: 0.5,
Reason: "增加缓存容量",
ExpectedImpact: "减少缓存驱逐",
Risk: "medium",
},
}
return &TuningDecision{
Adjustments: adjustments,
Confidence: 0.9,
ExpectedImprovement: 0.3,
Risk: 0.2,
AutoExecute: true,
}, nil
}
// createOverallPerformanceDecision 创建整体性能优化决策
func (t *TuningStrategy) createOverallPerformanceDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
adjustments := []ParameterAdjustment{
{
ParameterName: "global_optimization",
CurrentValue: false,
ProposedValue: true,
AdjustmentRatio: 1.0,
Reason: "启用全局优化",
ExpectedImpact: "整体性能提升",
Risk: "medium",
},
{
ParameterName: "compression_level",
CurrentValue: "standard",
ProposedValue: "high",
AdjustmentRatio: 0.3,
Reason: "提高压缩效率",
ExpectedImpact: "减少存储开销",
Risk: "low",
},
}
return &TuningDecision{
Adjustments: adjustments,
Confidence: 0.7,
ExpectedImprovement: 0.2,
Risk: 0.35,
AutoExecute: true,
}, nil
}
// createPreventiveDecision 创建预防性调优决策
func (t *TuningStrategy) createPreventiveDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) {
// 基于趋势预测的预防性调优
adjustments := []ParameterAdjustment{
{
ParameterName: "preventive_scaling",
CurrentValue: 1.0,
ProposedValue: 1.1,
AdjustmentRatio: 0.1,
Reason: "预防性资源扩展",
ExpectedImpact: "避免性能下降",
Risk: "low",
},
}
return &TuningDecision{
Adjustments: adjustments,
Confidence: 0.6,
ExpectedImprovement: 0.1,
Risk: 0.15,
AutoExecute: true,
}, nil
}
// ExecuteDecision 执行决策
func (t *TuningStrategy) ExecuteDecision(decision *TuningDecision) *StrategyExecution {
execution := &StrategyExecution{
Timestamp: time.Now(),
Decision: decision,
Executed: false,
}
// 简化的执行逻辑
execution.Executed = true
execution.Result = &ExecutionResult{
Success: true,
PerformanceBefore: 70.0, // 模拟值
PerformanceAfter: 85.0, // 模拟值
Improvement: 0.15,
SideEffects: []string{},
}
// 记录执行历史
t.mutex.Lock()
t.executionHistory = append(t.executionHistory, execution)
// 限制历史记录大小
if len(t.executionHistory) > 100 {
t.executionHistory = t.executionHistory[1:]
}
t.mutex.Unlock()
return execution
}
// GetExecutionHistory 获取执行历史
func (t *TuningStrategy) GetExecutionHistory(limit int) []*StrategyExecution {
t.mutex.RLock()
defer t.mutex.RUnlock()
if limit <= 0 || limit > len(t.executionHistory) {
limit = len(t.executionHistory)
}
history := make([]*StrategyExecution, limit)
startIndex := len(t.executionHistory) - limit
copy(history, t.executionHistory[startIndex:])
return history
}
// UpdateStrategy 更新策略
func (t *TuningStrategy) UpdateStrategy(strategyType string) {
t.mutex.Lock()
defer t.mutex.Unlock()
t.strategyType = strategyType
// 根据策略类型调整规则优先级和启用状态
switch strategyType {
case "conservative":
// 保守策略：只启用低风险规则
for _, rule := range t.rules {
rule.Enabled = rule.Priority <= 5
}
case "aggressive":
// 激进策略：启用所有规则
for _, rule := range t.rules {
rule.Enabled = true
}
case "balanced":
// 平衡策略：默认设置
for _, rule := range t.rules {
rule.Enabled = true
}
}
}
// GetStrategyStats 获取策略统计
func (t *TuningStrategy) GetStrategyStats() map[string]interface{} {
t.mutex.RLock()
defer t.mutex.RUnlock()
stats := map[string]interface{}{
"strategy_type": t.strategyType,
"total_executions": len(t.executionHistory),
"enabled_rules": 0,
"rule_statistics": make(map[string]interface{}),
}
enabledRules := 0
ruleStats := make(map[string]interface{})
for _, rule := range t.rules {
if rule.Enabled {
enabledRules++
}
ruleStats[rule.Name] = map[string]interface{}{
"enabled": rule.Enabled,
"priority": rule.Priority,
"trigger_count": rule.TriggerCount,
"last_triggered": rule.LastTriggered,
}
}
stats["enabled_rules"] = enabledRules
stats["rule_statistics"] = ruleStats
// 计算成功率
successfulExecutions := 0
for _, execution := range t.executionHistory {
if execution.Result != nil && execution.Result.Success {
successfulExecutions++
}
}
if len(t.executionHistory) > 0 {
stats["success_rate"] = float64(successfulExecutions) / float64(len(t.executionHistory))
} else {
stats["success_rate"] = 0.0
}
return stats
}

12
util/parser_util.go
View File

@@ -135,18 +135,6 @@ func ParseSearchResults(html string, channel string) ([]model.SearchResult, stri
var results []model.SearchResult
var nextPageParam string
// 查找分页链接 - 使用next而不是prev来获取下一页
// doc.Find("link[rel='next']").Each(func(i int, s *goquery.Selection) {
// href, exists := s.Attr("href")
// if exists {
// // 从href中提取before参数
// parts := strings.Split(href, "before=")
// if len(parts) > 1 {
// nextPageParam = strings.Split(parts[1], "&")[0]
// }
// }
// })
// 查找消息块
doc.Find(".tgme_widget_message_wrap").Each(func(i int, s *goquery.Selection) {
messageDiv := s.Find(".tgme_widget_message")