diff --git a/README.md b/README.md index 811e97c..aee0fc5 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ export ENABLED_PLUGINS=hunhepan,jikepan,panwiki,pansearch,panta,qupansou,susu,th | CACHE_WRITE_STRATEGY | 缓存写入策略(immediate/hybrid) | `hybrid` | | ENABLE_COMPRESSION | 是否启用压缩 | `false` | | MIN_SIZE_TO_COMPRESS | 最小压缩阈值(字节) | `1024` | -| GC_PERCENT | Go GC触发百分比 | `100` | +| GC_PERCENT | Go GC触发百分比 | `50` | | ASYNC_MAX_BACKGROUND_WORKERS | 最大后台工作者数量 | CPU核心数×5 | | ASYNC_MAX_BACKGROUND_TASKS | 最大后台任务数量 | 工作者数×5 | | ASYNC_CACHE_TTL_HOURS | 异步缓存有效期(小时) | `1` | @@ -153,7 +153,7 @@ CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -extldflags '-sta ``` [program:pansou] -environment=PORT=8888,CHANNELS="tgsearchers3,yunpanxunlei,tianyifc,BaiduCloudDisk,txtyzy,peccxinpd,gotopan,xingqiump4,yunpanqk,PanjClub,kkxlzy,baicaoZY,MCPH01,share_aliyun,bdwpzhpd,ysxb48,jdjdn1111,yggpan,MCPH086,zaihuayun,Q66Share,NewAliPan,ypquark,Oscar_4Kmovies,ucwpzy,alyp_TV,alyp_4K_Movies,shareAliyun,alyp_1,dianyingshare,Quark_Movies,XiangxiuNBB,NewQuark,ydypzyfx,kuakeyun,ucquark,xx123pan,yingshifenxiang123,zyfb123,tyypzhpd,tianyirigeng,cloudtianyi,hdhhd21,Lsp115,oneonefivewpfx,Maidanglaocom,qixingzhenren,taoxgzy,tgsearchers115,Channel_Shares_115,tyysypzypd,vip115hot,wp123zy,yunpan139,yunpan189,yunpanuc,yydf_hzl,alyp_Animation,alyp_JLP,leoziyuan",ENABLED_PLUGINS="labi,zhizhen,shandian,duoduo,muou" +environment=PORT=8888,CHANNELS="tgsearchers3,Aliyun_4K_Movies,bdbdndn11,yunpanx,bsbdbfjfjff,yp123pan,sbsbsnsqq,yunpanxunlei,tianyifc,BaiduCloudDisk,txtyzy,peccxinpd,gotopan,PanjClub,kkxlzy,baicaoZY,MCPH01,bdwpzhpd,ysxb48,jdjdn1111,yggpan,MCPH086,zaihuayun,Q66Share,Oscar_4Kmovies,ucwpzy,shareAliyun,alyp_1,dianyingshare,Quark_Movies,XiangxiuNBB,ydypzyfx,ucquark,xx123pan,yingshifenxiang123,zyfb123,tyypzhpd,tianyirigeng,cloudtianyi,hdhhd21,Lsp115,oneonefivewpfx,qixingzhenren,taoxgzy,Channel_Shares_115,tyysypzypd,vip115hot,wp123zy,yunpan139,yunpan189,yunpanuc,yydf_hzl,leoziyuan,pikpakpan,Q_dongman,yoyokuakeduanju",ENABLED_PLUGINS="labi,zhizhen,shandian,duoduo,muou" command=/home/work/pansou/pansou directory=/home/work/pansou autostart=true diff --git a/config/config.go b/config/config.go index e67ff02..a69fac1 100644 --- a/config/config.go +++ b/config/config.go @@ -260,11 +260,11 @@ func getMinSizeToCompress() int { func getGCPercent() int { percentEnv := os.Getenv("GC_PERCENT") if percentEnv == "" { - return 100 // 默认100% + return 50 // 默认50% - 优化内存管理，更频繁的GC避免内存暴涨 } percent, err := strconv.Atoi(percentEnv) if err != nil || percent <= 0 { - return 100 + return 50 // 错误时也使用优化后的默认值 } return percent } diff --git a/docker-compose.yml b/docker-compose.yml index 2c333c4..f83dc96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: - "8888:8888" environment: - PORT=8888 - - CHANNELS=tgsearchers3,SharePanBaidu,yunpanxunlei,tianyifc,BaiduCloudDisk + - CHANNELS=tgsearchers3,Aliyun_4K_Movies,bdbdndn11,yunpanx,bsbdbfjfjff,yp123pan,sbsbsnsqq,yunpanxunlei,tianyifc,BaiduCloudDisk,txtyzy,peccxinpd,gotopan,PanjClub,kkxlzy,baicaoZY,MCPH01,bdwpzhpd,ysxb48,jdjdn1111,yggpan,MCPH086,zaihuayun,Q66Share,Oscar_4Kmovies,ucwpzy,shareAliyun,alyp_1,dianyingshare,Quark_Movies,XiangxiuNBB,ydypzyfx,ucquark,xx123pan,yingshifenxiang123,zyfb123,tyypzhpd,tianyirigeng,cloudtianyi,hdhhd21,Lsp115,oneonefivewpfx,qixingzhenren,taoxgzy,Channel_Shares_115,tyysypzypd,vip115hot,wp123zy,yunpan139,yunpan189,yunpanuc,yydf_hzl,leoziyuan,pikpakpan,Q_dongman,yoyokuakeduanju # 必须指定启用的插件，多个插件用逗号分隔 - ENABLED_PLUGINS=labi,zhizhen,shandian,duoduo,muou,wanou - CACHE_ENABLED=true diff --git a/main.go b/main.go index 15a0806..c598360 100644 --- a/main.go +++ b/main.go @@ -90,7 +90,7 @@ func initApp() { // 初始化HTTP客户端 util.InitHTTPClient() - // 🔥 初始化缓存写入管理器 + // 初始化缓存写入管理器 var err error globalCacheWriteManager, err = cache.NewDelayedBatchWriteManager() if err != nil { @@ -188,20 +188,20 @@ func startServer() { <-quit fmt.Println("正在关闭服务器...") - // 🔥 优先保存缓存数据到磁盘（数据安全第一） + // 优先保存缓存数据到磁盘（数据安全第一） // 增加关闭超时时间，确保数据有足够时间保存 shutdownTimeout := 10 * time.Second if globalCacheWriteManager != nil { if err := globalCacheWriteManager.Shutdown(shutdownTimeout); err != nil { - log.Printf("❌ 缓存数据保存失败: %v", err) + log.Printf("缓存数据保存失败: %v", err) } } // 额外确保内存缓存也被保存（双重保障） if mainCache := service.GetEnhancedTwoLevelCache(); mainCache != nil { if err := mainCache.FlushMemoryToDisk(); err != nil { - log.Printf("❌ 内存缓存同步失败: %v", err) + log.Printf("内存缓存同步失败: %v", err) } } diff --git a/plugin/huban/huban.go b/plugin/huban/huban.go index 2615299..65c8633 100644 --- a/plugin/huban/huban.go +++ b/plugin/huban/huban.go @@ -176,7 +176,7 @@ func (p *HubanAsyncPlugin) searchImpl(client *http.Client, keyword string, ext m return results, nil } else if i == 0 { // 第一个域名失败，记录日志但继续尝试第二个 - fmt.Printf("[%s] 域名1失败，尝试域名2: %v\n", p.Name(), err) + // fmt.Printf("[%s] 域名1失败，尝试域名2: %v\n", p.Name(), err) } } @@ -209,10 +209,7 @@ func (p *HubanAsyncPlugin) tryRequest(searchURL string, client *http.Client) ([] defer resp.Body.Close() // 解析JSON响应 - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("读取响应失败: %w", err) - } + body, _ := io.ReadAll(resp.Body) var apiResponse HubanAPIResponse if err := json.Unmarshal(body, &apiResponse); err != nil { diff --git a/service/cache_integration.go b/service/cache_integration.go index 161b81d..03709a3 100644 --- a/service/cache_integration.go +++ b/service/cache_integration.go @@ -40,7 +40,7 @@ func NewCacheWriteIntegration(mainCache *cache.EnhancedTwoLevelCache) (*CacheWri integration.initialized = true - fmt.Printf("✅ [缓存写入集成] 初始化完成\n") + fmt.Printf("[缓存写入集成] 初始化完成\n") return integration, nil } diff --git a/service/search_service.go b/service/search_service.go index 6a9396c..efca324 100644 --- a/service/search_service.go +++ b/service/search_service.go @@ -236,7 +236,7 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach return } - // 🔧 设置全局序列化器，确保异步插件与主程序使用相同的序列化格式 + // 设置全局序列化器，确保异步插件与主程序使用相同的序列化格式 serializer := mainCache.GetSerializer() if serializer != nil { plugin.SetGlobalCacheSerializer(serializer) @@ -244,12 +244,12 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach // 创建缓存更新函数（支持IsFinal参数）- 接收原始数据并与现有缓存合并 cacheUpdater := func(key string, newResults []model.SearchResult, ttl time.Duration, isFinal bool, keyword string, pluginName string) error { - // 🚀 优化：如果新结果为空，跳过缓存更新（避免无效操作） + // 优化：如果新结果为空，跳过缓存更新（避免无效操作） if len(newResults) == 0 { return nil } - // 🔧 获取现有缓存数据进行合并 + // 获取现有缓存数据进行合并 var finalResults []model.SearchResult if existingData, hit, err := mainCache.Get(key); err == nil && hit { var existingResults []model.SearchResult @@ -268,9 +268,9 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach if config.AppConfig != nil && config.AppConfig.AsyncLogEnabled { displayKey := key[:8] + "..." if keyword != "" { - fmt.Printf("⚠️ [异步插件 %s] 缓存反序列化失败，使用新结果: %s(关键词:%s) | 结果数: %d\n", pluginName, displayKey, keyword, len(newResults)) + fmt.Printf("[异步插件 %s] 缓存反序列化失败，使用新结果: %s(关键词:%s) | 结果数: %d\n", pluginName, displayKey, keyword, len(newResults)) } else { - fmt.Printf("⚠️ [异步插件 %s] 缓存反序列化失败，使用新结果: %s | 结果数: %d\n", pluginName, key, len(newResults)) + fmt.Printf("[异步插件 %s] 缓存反序列化失败，使用新结果: %s | 结果数: %d\n", pluginName, key, len(newResults)) } } } @@ -287,18 +287,13 @@ func injectMainCacheToAsyncPlugins(pluginManager *plugin.PluginManager, mainCach } } - // 🔧 序列化合并后的结果 + // 序列化合并后的结果 data, err := mainCache.GetSerializer().Serialize(finalResults) if err != nil { - fmt.Printf("❌ [缓存更新] 序列化失败: %s | 错误: %v\n", key, err) + fmt.Printf("[缓存更新] 序列化失败: %s | 错误: %v\n", key, err) return err } - // 🔥 使用新的缓存写入管理器 - // 注意：获取外部引用需要导入main包 - // 为了避免循环依赖，我们暂时通过全局变量访问 - // TODO: 优化架构，使用依赖注入方式 - // 先更新内存缓存（立即可见） if err := mainCache.SetMemoryOnly(key, data, ttl); err != nil { return fmt.Errorf("内存缓存更新失败: %v", err) @@ -942,7 +937,7 @@ func isEmpty(line string) bool { // 将搜索结果按网盘类型分组 func mergeResultsByType(results []model.SearchResult, keyword string, cloudTypes []string) model.MergedLinks { // 创建合并结果的映射 - mergedLinks := make(model.MergedLinks, 10) // 预分配容量，假设有10种不同的网盘类型 + mergedLinks := make(model.MergedLinks, 12) // 预分配容量，假设有12种不同的网盘类型 // 用于去重的映射，键为URL uniqueLinks := make(map[string]model.MergedLink) @@ -1123,17 +1118,6 @@ func mergeResultsByType(results []model.SearchResult, keyword string, cloudTypes mergedLinks[linkType] = append(mergedLinks[linkType], mergedLink) } - // 注意：不再重新排序，保持SearchResult阶段的权重排序结果 - // 原来的时间排序会覆盖权重排序，现在注释掉 - /* - // 对每种类型的链接按时间排序（新的在前面） - for linkType, links := range mergedLinks { - sort.Slice(links, func(i, j int) bool { - return links[i].Datetime.After(links[j].Datetime) - }) - mergedLinks[linkType] = links - } - */ // 如果指定了cloudTypes，则过滤结果 if len(cloudTypes) > 0 { @@ -1263,7 +1247,7 @@ func (s *SearchService) searchPlugins(keyword string, plugins []string, forceRef return results, nil } else { displayKey := cacheKey[:8] + "..." - fmt.Printf("❌ [主服务] 缓存反序列化失败: %s(关键词:%s) | 错误: %v\n", displayKey, keyword, err) + fmt.Printf("[主服务] 缓存反序列化失败: %s(关键词:%s) | 错误: %v\n", displayKey, keyword, err) } } } @@ -1354,7 +1338,7 @@ func (s *SearchService) searchPlugins(keyword string, plugins []string, forceRef } } - // 🔧 恢复主程序缓存更新：确保最终合并结果被正确缓存 + // 恢复主程序缓存更新：确保最终合并结果被正确缓存 if cacheInitialized && config.AppConfig.CacheEnabled { go func(res []model.SearchResult, kw string, key string) { ttl := time.Duration(config.AppConfig.CacheTTLMinutes) * time.Minute @@ -1363,12 +1347,12 @@ func (s *SearchService) searchPlugins(keyword string, plugins []string, forceRef if enhancedTwoLevelCache != nil { data, err := enhancedTwoLevelCache.GetSerializer().Serialize(res) if err != nil { - fmt.Printf("❌ [主程序] 缓存序列化失败: %s | 错误: %v\n", key, err) + fmt.Printf("[主程序] 缓存序列化失败: %s | 错误: %v\n", key, err) return } - // 主程序最后更新，覆盖可能有问题的异步插件缓存 - // 🔥 修复：使用同步方式确保数据写入磁盘 + // 主程序最后更新，覆盖可能有问题的异步插件缓存 + // 使用同步方式确保数据写入磁盘 enhancedTwoLevelCache.SetBothLevels(key, data, ttl) if config.AppConfig != nil && config.AppConfig.AsyncLogEnabled { fmt.Printf("[主程序] 缓存更新完成: %s | 结果数: %d", diff --git a/util/cache/adaptive_tuning_engine.go b/util/cache/adaptive_tuning_engine.go deleted file mode 100644 index 29e21ab..0000000 --- a/util/cache/adaptive_tuning_engine.go +++ /dev/null @@ -1,884 +0,0 @@ -package cache - -import ( - "fmt" - "math" - "sync" - "sync/atomic" - "time" -) - -// AdaptiveTuningEngine 自适应调优引擎 -type AdaptiveTuningEngine struct { - // 核心组件 - metricCollector *MetricCollector - performanceAnalyzer *PerformanceAnalyzer - predictiveModel *PredictiveModel - tuningStrategy *TuningStrategy - - // 配置参数 - config *AdaptiveTuningConfig - - // 运行状态 - isRunning int32 - shutdownChan chan struct{} - - // 调优历史 - tuningHistory []*TuningRecord - historyMutex sync.RWMutex - maxHistorySize int - - // 学习数据 - learningData *LearningDataset - - // 统计信息 - stats *TuningEngineStats - - mutex sync.RWMutex -} - -// AdaptiveTuningConfig 自适应调优配置 -type AdaptiveTuningConfig struct { - // 调优间隔 - TuningInterval time.Duration - MetricInterval time.Duration - - // 性能阈值 - CPUUsageThreshold float64 - MemoryThreshold int64 - LatencyThreshold time.Duration - - // 学习参数 - LearningRate float64 - AdaptationSpeed float64 - StabilityFactor float64 - - // 调优范围 - MinBatchInterval time.Duration - MaxBatchInterval time.Duration - MinBatchSize int - MaxBatchSize int - - // 安全参数 - MaxAdjustmentRatio float64 // 最大调整幅度 - RollbackThreshold float64 // 回滚阈值 - - // 预测参数 - PredictionWindow time.Duration - ConfidenceThreshold float64 -} - -// MetricCollector 指标收集器 -type MetricCollector struct { - // 系统指标 - systemMetrics *SystemMetrics - - // 应用指标 - applicationMetrics *ApplicationMetrics - - // 缓存指标 - cacheMetrics *CacheMetrics - - // 历史数据 - metricsHistory []MetricSnapshot - historyMutex sync.RWMutex - maxHistorySize int - - // 采集状态 - isCollecting int32 - collectionChan chan struct{} -} - -// SystemMetrics 系统指标 -type SystemMetrics struct { - Timestamp time.Time - CPUUsage float64 - MemoryUsage int64 - MemoryTotal int64 - DiskIORate float64 - NetworkIORate float64 - GoroutineCount int - GCPauseDuration time.Duration - HeapSize int64 - AllocRate float64 -} - -// ApplicationMetrics 应用指标 -type ApplicationMetrics struct { - Timestamp time.Time - RequestRate float64 - ResponseTime time.Duration - ErrorRate float64 - ThroughputMBps float64 - ConcurrentUsers int - QueueDepth int - ProcessingRate float64 -} - -// CacheMetrics 缓存指标 -type CacheMetrics struct { - Timestamp time.Time - HitRate float64 - WriteRate float64 - ReadRate float64 - EvictionRate float64 - CompressionRatio float64 - StorageUsage int64 - BufferUtilization float64 - BatchEfficiency float64 -} - -// MetricSnapshot 指标快照 -type MetricSnapshot struct { - Timestamp time.Time - System SystemMetrics - Application ApplicationMetrics - Cache CacheMetrics - - // 综合指标 - OverallPerformance float64 - Efficiency float64 - Stability float64 -} - -// PerformanceAnalyzer 性能分析器 -type PerformanceAnalyzer struct { - // 分析算法 - trendAnalyzer *TrendAnalyzer - anomalyDetector *AnomalyDetector - correlationAnalyzer *CorrelationAnalyzer - - // 分析结果 - currentTrends map[string]Trend - detectedAnomalies []Anomaly - correlations map[string]float64 - - mutex sync.RWMutex -} - -// Trend 趋势 -type Trend struct { - Metric string - Direction string // increasing, decreasing, stable - Slope float64 - Confidence float64 - Duration time.Duration - Prediction float64 -} - -// Anomaly 异常 -type Anomaly struct { - Metric string - Timestamp time.Time - Severity string // low, medium, high - Value float64 - ExpectedRange [2]float64 - Description string - Impact float64 -} - -// PredictiveModel 预测模型 -type PredictiveModel struct { - // 模型类型 - modelType string // linear_regression, exponential_smoothing, arima - - // 模型参数 - coefficients []float64 - seasonalFactors []float64 - trendComponent float64 - - // 训练数据 - trainingData []DataPoint - testData []DataPoint - - // 模型性能 - accuracy float64 - rmse float64 - mae float64 - - // 预测结果 - predictions map[string]Prediction - - mutex sync.RWMutex -} - -// DataPoint 数据点 -type DataPoint struct { - Timestamp time.Time - Values map[string]float64 - Label string -} - -// Prediction 预测 -type Prediction struct { - Metric string - FutureValue float64 - Confidence float64 - TimeHorizon time.Duration - PredictedAt time.Time - ActualValue *float64 // 用于验证预测准确性 -} - -// TuningStrategy 调优策略 -type TuningStrategy struct { - // 策略类型 - strategyType string // conservative, aggressive, balanced - - // 调优规则 - rules []*TuningRule - - // 参数调整 - parameterAdjustments map[string]ParameterAdjustment - - // 执行历史 - executionHistory []*StrategyExecution - - mutex sync.RWMutex -} - -// TuningRule 调优规则 -type TuningRule struct { - Name string - Condition func(*MetricSnapshot) bool - Action func(*AdaptiveTuningEngine) (*TuningDecision, error) - Priority int - Enabled bool - LastTriggered time.Time - TriggerCount int64 -} - -// ParameterAdjustment 参数调整 -type ParameterAdjustment struct { - ParameterName string - CurrentValue interface{} - ProposedValue interface{} - AdjustmentRatio float64 - Reason string - ExpectedImpact string - Risk string -} - -// TuningDecision 调优决策 -type TuningDecision struct { - Timestamp time.Time - Trigger string - Adjustments []ParameterAdjustment - Confidence float64 - ExpectedImprovement float64 - Risk float64 - AutoExecute bool -} - -// StrategyExecution 策略执行 -type StrategyExecution struct { - Timestamp time.Time - Decision *TuningDecision - Executed bool - Result *ExecutionResult -} - -// ExecutionResult 执行结果 -type ExecutionResult struct { - Success bool - Error error - PerformanceBefore float64 - PerformanceAfter float64 - Improvement float64 - SideEffects []string -} - -// TuningRecord 调优记录 -type TuningRecord struct { - Timestamp time.Time - Type string // automatic, manual, rollback - Parameters map[string]interface{} - Reason string - Result *TuningResult -} - -// TuningResult 调优结果 -type TuningResult struct { - Success bool - PerformanceGain float64 - ResourceUsageChange float64 - StabilityImpact float64 - UserExperienceChange float64 - Duration time.Duration -} - -// LearningDataset 学习数据集 -type LearningDataset struct { - Features [][]float64 - Labels []float64 - Weights []float64 - - // 数据统计 - FeatureStats []FeatureStatistics - LabelStats LabelStatistics - - // 数据划分 - TrainingSplit float64 - ValidationSplit float64 - TestSplit float64 - - mutex sync.RWMutex -} - -// FeatureStatistics 特征统计 -type FeatureStatistics struct { - Name string - Mean float64 - Std float64 - Min float64 - Max float64 - Correlation float64 -} - -// LabelStatistics 标签统计 -type LabelStatistics struct { - Mean float64 - Std float64 - Min float64 - Max float64 - Distribution map[string]int -} - -// TuningEngineStats 调优引擎统计 -type TuningEngineStats struct { - // 基础统计 - TotalAdjustments int64 - SuccessfulAdjustments int64 - FailedAdjustments int64 - RollbackCount int64 - - // 性能统计 - AverageImprovement float64 - MaxImprovement float64 - TotalImprovement float64 - - // 学习统计 - ModelAccuracy float64 - PredictionAccuracy float64 - LearningIterations int64 - - // 时间统计 - AverageDecisionTime time.Duration - TotalTuningTime time.Duration - LastTuningTime time.Time - - // 系统影响 - CPUOverhead float64 - MemoryOverhead int64 - - mutex sync.RWMutex -} - -// NewAdaptiveTuningEngine 创建自适应调优引擎 -func NewAdaptiveTuningEngine() *AdaptiveTuningEngine { - config := &AdaptiveTuningConfig{ - TuningInterval: 5 * time.Minute, - MetricInterval: 30 * time.Second, - CPUUsageThreshold: 0.8, - MemoryThreshold: 500 * 1024 * 1024, // 500MB - LatencyThreshold: 10 * time.Second, - LearningRate: 0.01, - AdaptationSpeed: 0.1, - StabilityFactor: 0.9, - MinBatchInterval: 10 * time.Second, - MaxBatchInterval: 10 * time.Minute, - MinBatchSize: 10, - MaxBatchSize: 1000, - MaxAdjustmentRatio: 0.3, // 最大30%调整 - RollbackThreshold: 0.1, // 性能下降10%触发回滚 - PredictionWindow: 1 * time.Hour, - ConfidenceThreshold: 0.7, - } - - engine := &AdaptiveTuningEngine{ - config: config, - shutdownChan: make(chan struct{}), - maxHistorySize: 1000, - tuningHistory: make([]*TuningRecord, 0), - stats: &TuningEngineStats{ - LastTuningTime: time.Now(), - }, - } - - // 初始化组件 - engine.metricCollector = NewMetricCollector() - engine.performanceAnalyzer = NewPerformanceAnalyzer() - engine.predictiveModel = NewPredictiveModel() - engine.tuningStrategy = NewTuningStrategy() - engine.learningData = NewLearningDataset() - - return engine -} - -// Start 启动自适应调优引擎 -func (a *AdaptiveTuningEngine) Start() error { - if !atomic.CompareAndSwapInt32(&a.isRunning, 0, 1) { - return fmt.Errorf("调优引擎已在运行中") - } - - // 启动指标收集 - if err := a.metricCollector.Start(a.config.MetricInterval); err != nil { - return fmt.Errorf("启动指标收集失败: %v", err) - } - - // 启动主调优循环 - go a.tuningLoop() - - // 启动性能分析循环 - go a.analysisLoop() - - // 启动模型训练循环 - go a.learningLoop() - - fmt.Printf("🧠 [自适应调优引擎] 启动完成，调优间隔: %v\n", a.config.TuningInterval) - return nil -} - -// tuningLoop 调优循环 -func (a *AdaptiveTuningEngine) tuningLoop() { - ticker := time.NewTicker(a.config.TuningInterval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - a.performTuning() - - case <-a.shutdownChan: - return - } - } -} - -// analysisLoop 分析循环 -func (a *AdaptiveTuningEngine) analysisLoop() { - ticker := time.NewTicker(a.config.MetricInterval * 2) // 分析频率低于采集频率 - defer ticker.Stop() - - for { - select { - case <-ticker.C: - a.performAnalysis() - - case <-a.shutdownChan: - return - } - } -} - -// learningLoop 学习循环 -func (a *AdaptiveTuningEngine) learningLoop() { - ticker := time.NewTicker(15 * time.Minute) // 每15分钟学习一次 - defer ticker.Stop() - - for { - select { - case <-ticker.C: - a.performLearning() - - case <-a.shutdownChan: - return - } - } -} - -// performTuning 执行调优 -func (a *AdaptiveTuningEngine) performTuning() { - startTime := time.Now() - - // 获取当前指标 - currentMetrics := a.metricCollector.GetLatestMetrics() - if currentMetrics == nil { - return - } - - // 分析性能状态 - performanceIssues := a.performanceAnalyzer.AnalyzeIssues(currentMetrics) - - // 生成调优决策 - decision := a.tuningStrategy.GenerateDecision(currentMetrics, performanceIssues) - if decision == nil { - return - } - - // 验证决策合理性 - if !a.validateDecision(decision) { - fmt.Printf("⚠️ [调优引擎] 决策验证失败，跳过执行\n") - return - } - - // 执行调优 - result := a.executeDecision(decision) - - // 记录调优历史 - record := &TuningRecord{ - Timestamp: time.Now(), - Type: "automatic", - Parameters: a.extractParameters(decision), - Reason: decision.Trigger, - Result: result, - } - - a.addTuningRecord(record) - - // 更新统计 - a.updateTuningStats(result, time.Since(startTime)) - - if result.Success { - fmt.Printf("✅ [调优引擎] 自动调优完成，性能提升: %.2f%%\n", result.PerformanceGain*100) - } else { - fmt.Printf("❌ [调优引擎] 调优失败，考虑回滚\n") - a.considerRollback(decision, result) - } -} - -// performAnalysis 执行性能分析 -func (a *AdaptiveTuningEngine) performAnalysis() { - // 趋势分析 - a.performanceAnalyzer.AnalyzeTrends(a.metricCollector.GetMetricsHistory(100)) - - // 异常检测 - a.performanceAnalyzer.DetectAnomalies(a.metricCollector.GetLatestMetrics()) - - // 相关性分析 - a.performanceAnalyzer.AnalyzeCorrelations(a.metricCollector.GetMetricsHistory(50)) -} - -// performLearning 执行机器学习 -func (a *AdaptiveTuningEngine) performLearning() { - // 收集训练数据 - a.collectTrainingData() - - // 训练预测模型 - if err := a.predictiveModel.Train(a.learningData); err != nil { - fmt.Printf("⚠️ [调优引擎] 模型训练失败: %v\n", err) - return - } - - // 验证模型性能 - accuracy := a.predictiveModel.Validate() - - // 更新统计 - a.mutex.Lock() - a.stats.ModelAccuracy = accuracy - a.stats.LearningIterations++ - a.mutex.Unlock() - - fmt.Printf("🎓 [调优引擎] 模型训练完成，准确率: %.2f%%\n", accuracy*100) -} - -// validateDecision 验证调优决策 -func (a *AdaptiveTuningEngine) validateDecision(decision *TuningDecision) bool { - // 检查置信度 - if decision.Confidence < a.config.ConfidenceThreshold { - return false - } - - // 检查风险级别 - if decision.Risk > 0.7 { // 风险过高 - return false - } - - // 检查调整幅度 - for _, adj := range decision.Adjustments { - if math.Abs(adj.AdjustmentRatio) > a.config.MaxAdjustmentRatio { - return false - } - } - - return true -} - -// executeDecision 执行调优决策 -func (a *AdaptiveTuningEngine) executeDecision(decision *TuningDecision) *TuningResult { - startTime := time.Now() - - // 获取执行前性能基线 - beforeMetrics := a.metricCollector.GetLatestMetrics() - performanceBefore := a.calculateOverallPerformance(beforeMetrics) - - // 执行参数调整 - success := true - - for _, adjustment := range decision.Adjustments { - if err := a.applyParameterAdjustment(adjustment); err != nil { - success = false - break - } - } - - if !success { - return &TuningResult{ - Success: false, - PerformanceGain: 0, - ResourceUsageChange: 0, - StabilityImpact: 0, - Duration: time.Since(startTime), - } - } - - // 等待一段时间观察效果 - time.Sleep(30 * time.Second) - - // 获取执行后性能 - afterMetrics := a.metricCollector.GetLatestMetrics() - performanceAfter := a.calculateOverallPerformance(afterMetrics) - - performanceGain := (performanceAfter - performanceBefore) / performanceBefore - - // 计算资源使用变化 - resourceBefore := float64(beforeMetrics.System.MemoryUsage + int64(beforeMetrics.System.CPUUsage*1000)) - resourceAfter := float64(afterMetrics.System.MemoryUsage + int64(afterMetrics.System.CPUUsage*1000)) - resourceChange := (resourceAfter - resourceBefore) / resourceBefore - - return &TuningResult{ - Success: true, - PerformanceGain: performanceGain, - ResourceUsageChange: resourceChange, - StabilityImpact: a.calculateStabilityImpact(beforeMetrics, afterMetrics), - UserExperienceChange: performanceGain, // 简化假设 - Duration: time.Since(startTime), - } -} - -// calculateOverallPerformance 计算整体性能分数 -func (a *AdaptiveTuningEngine) calculateOverallPerformance(metrics *MetricSnapshot) float64 { - if metrics == nil { - return 0 - } - - // 性能分数计算（0-100分） - cpuScore := math.Max(0, (1.0-metrics.System.CPUUsage)*40) // CPU使用率越低越好，最高40分 - memoryScore := math.Max(0, (1.0-float64(metrics.System.MemoryUsage)/float64(metrics.System.MemoryTotal))*30) // 内存使用率越低越好，最高30分 - responseScore := math.Max(0, (1.0-math.Min(1.0, float64(metrics.Application.ResponseTime)/float64(time.Second)))*20) // 响应时间越短越好，最高20分 - cacheScore := metrics.Cache.HitRate * 10 // 缓存命中率越高越好，最高10分 - - return cpuScore + memoryScore + responseScore + cacheScore -} - -// calculateStabilityImpact 计算稳定性影响 -func (a *AdaptiveTuningEngine) calculateStabilityImpact(before, after *MetricSnapshot) float64 { - if before == nil || after == nil { - return 0 - } - - // 简化的稳定性计算：比较关键指标的变化 - cpuVariation := math.Abs(after.System.CPUUsage - before.System.CPUUsage) - memoryVariation := math.Abs(float64(after.System.MemoryUsage-before.System.MemoryUsage) / float64(before.System.MemoryUsage)) - - // 变化越小，稳定性越好 - stabilityScore := 1.0 - (cpuVariation*0.5 + memoryVariation*0.5) - return math.Max(0, stabilityScore) -} - -// applyParameterAdjustment 应用参数调整 -func (a *AdaptiveTuningEngine) applyParameterAdjustment(adjustment ParameterAdjustment) error { - // 这里应该调用具体的参数设置函数 - // 暂时模拟实现 - fmt.Printf("🔧 [调优引擎] 调整参数 %s: %v -> %v (%.1f%%)\n", - adjustment.ParameterName, - adjustment.CurrentValue, - adjustment.ProposedValue, - adjustment.AdjustmentRatio*100) - - return nil -} - -// collectTrainingData 收集训练数据 -func (a *AdaptiveTuningEngine) collectTrainingData() { - history := a.metricCollector.GetMetricsHistory(200) - _ = a.getTuningHistory(50) // 暂时不使用调优历史 - - // 构建特征和标签 - for i, metrics := range history { - if i < len(history)-1 { - // 特征：当前指标 - features := []float64{ - metrics.System.CPUUsage, - float64(metrics.System.MemoryUsage) / 1024 / 1024, // MB - float64(metrics.Application.ResponseTime) / float64(time.Millisecond), - metrics.Cache.HitRate, - metrics.Cache.CompressionRatio, - } - - // 标签：下一时刻的整体性能 - nextMetrics := history[i+1] - label := a.calculateOverallPerformance(&nextMetrics) - - // 添加到学习数据集 - a.learningData.mutex.Lock() - a.learningData.Features = append(a.learningData.Features, features) - a.learningData.Labels = append(a.learningData.Labels, label) - a.learningData.Weights = append(a.learningData.Weights, 1.0) - a.learningData.mutex.Unlock() - } - } - - // 限制数据集大小 - a.learningData.mutex.Lock() - maxSize := 1000 - if len(a.learningData.Features) > maxSize { - excess := len(a.learningData.Features) - maxSize - a.learningData.Features = a.learningData.Features[excess:] - a.learningData.Labels = a.learningData.Labels[excess:] - a.learningData.Weights = a.learningData.Weights[excess:] - } - a.learningData.mutex.Unlock() -} - -// considerRollback 考虑回滚 -func (a *AdaptiveTuningEngine) considerRollback(decision *TuningDecision, result *TuningResult) { - if result.PerformanceGain < -a.config.RollbackThreshold { - fmt.Printf("🔄 [调优引擎] 触发自动回滚，性能下降: %.2f%%\n", result.PerformanceGain*100) - a.performRollback(decision) - } -} - -// performRollback 执行回滚 -func (a *AdaptiveTuningEngine) performRollback(originalDecision *TuningDecision) { - // 创建回滚决策 - rollbackDecision := &TuningDecision{ - Timestamp: time.Now(), - Trigger: "automatic_rollback", - Adjustments: make([]ParameterAdjustment, 0), - Confidence: 1.0, - AutoExecute: true, - } - - // 反向调整所有参数 - for _, adjustment := range originalDecision.Adjustments { - rollbackAdjustment := ParameterAdjustment{ - ParameterName: adjustment.ParameterName, - CurrentValue: adjustment.ProposedValue, - ProposedValue: adjustment.CurrentValue, - AdjustmentRatio: -adjustment.AdjustmentRatio, - Reason: "rollback", - ExpectedImpact: "restore_stability", - Risk: "low", - } - rollbackDecision.Adjustments = append(rollbackDecision.Adjustments, rollbackAdjustment) - } - - // 执行回滚 - result := a.executeDecision(rollbackDecision) - - // 记录回滚 - record := &TuningRecord{ - Timestamp: time.Now(), - Type: "rollback", - Parameters: a.extractParameters(rollbackDecision), - Reason: "performance_degradation", - Result: result, - } - - a.addTuningRecord(record) - - // 更新统计 - atomic.AddInt64(&a.stats.RollbackCount, 1) -} - -// addTuningRecord 添加调优记录 -func (a *AdaptiveTuningEngine) addTuningRecord(record *TuningRecord) { - a.historyMutex.Lock() - defer a.historyMutex.Unlock() - - a.tuningHistory = append(a.tuningHistory, record) - - // 限制历史记录大小 - if len(a.tuningHistory) > a.maxHistorySize { - a.tuningHistory = a.tuningHistory[1:] - } -} - -// updateTuningStats 更新调优统计 -func (a *AdaptiveTuningEngine) updateTuningStats(result *TuningResult, decisionTime time.Duration) { - a.stats.mutex.Lock() - defer a.stats.mutex.Unlock() - - a.stats.TotalAdjustments++ - if result.Success { - a.stats.SuccessfulAdjustments++ - a.stats.TotalImprovement += result.PerformanceGain - a.stats.AverageImprovement = a.stats.TotalImprovement / float64(a.stats.SuccessfulAdjustments) - - if result.PerformanceGain > a.stats.MaxImprovement { - a.stats.MaxImprovement = result.PerformanceGain - } - } else { - a.stats.FailedAdjustments++ - } - - // 更新时间统计 - a.stats.TotalTuningTime += decisionTime - a.stats.AverageDecisionTime = time.Duration(int64(a.stats.TotalTuningTime) / a.stats.TotalAdjustments) - a.stats.LastTuningTime = time.Now() -} - -// extractParameters 提取决策参数 -func (a *AdaptiveTuningEngine) extractParameters(decision *TuningDecision) map[string]interface{} { - params := make(map[string]interface{}) - for _, adj := range decision.Adjustments { - params[adj.ParameterName] = adj.ProposedValue - } - return params -} - -// getTuningHistory 获取调优历史 -func (a *AdaptiveTuningEngine) getTuningHistory(limit int) []*TuningRecord { - a.historyMutex.RLock() - defer a.historyMutex.RUnlock() - - if limit <= 0 || limit > len(a.tuningHistory) { - limit = len(a.tuningHistory) - } - - history := make([]*TuningRecord, limit) - startIndex := len(a.tuningHistory) - limit - copy(history, a.tuningHistory[startIndex:]) - - return history -} - -// Stop 停止自适应调优引擎 -func (a *AdaptiveTuningEngine) Stop() error { - if !atomic.CompareAndSwapInt32(&a.isRunning, 1, 0) { - return nil - } - - // 停止指标收集 - a.metricCollector.Stop() - - // 停止所有循环 - close(a.shutdownChan) - - fmt.Printf("🧠 [自适应调优引擎] 已停止\n") - return nil -} - -// GetStats 获取调优引擎统计 -func (a *AdaptiveTuningEngine) GetStats() *TuningEngineStats { - a.stats.mutex.RLock() - defer a.stats.mutex.RUnlock() - - statsCopy := *a.stats - return &statsCopy -} - -// GetTuningReport 获取调优报告 -func (a *AdaptiveTuningEngine) GetTuningReport() map[string]interface{} { - stats := a.GetStats() - recentHistory := a.getTuningHistory(10) - - return map[string]interface{}{ - "engine_stats": stats, - "recent_history": recentHistory, - "current_trends": a.performanceAnalyzer.GetCurrentTrends(), - "anomalies": a.performanceAnalyzer.GetDetectedAnomalies(), - "predictions": a.predictiveModel.GetPredictions(), - "model_accuracy": a.predictiveModel.GetAccuracy(), - } -} \ No newline at end of file diff --git a/util/cache/advanced_data_merger.go b/util/cache/advanced_data_merger.go deleted file mode 100644 index 98b9c90..0000000 --- a/util/cache/advanced_data_merger.go +++ /dev/null @@ -1,620 +0,0 @@ -package cache - -import ( - "fmt" - "sync" - "time" - - "pansou/model" -) - -// AdvancedDataMerger 高级数据合并器 -type AdvancedDataMerger struct { - // 合并策略 - mergeStrategies map[string]MergeStrategy - - // 合并规则 - mergeRules []*MergeRule - - // 统计信息 - totalMerges int64 - successfulMerges int64 - failedMerges int64 - - // 缓存去重 - deduplicationMap map[string]*CacheOperation - dedupMutex sync.RWMutex - - // 性能监控 - mergeMetrics *MergeMetrics - - mutex sync.RWMutex -} - -// MergeStrategy 合并策略接口 -type MergeStrategy interface { - CanMerge(existing *CacheOperation, new *CacheOperation) bool - Merge(existing *CacheOperation, new *CacheOperation) (*CacheOperation, error) - GetPriority() int -} - -// MergeRule 合并规则 -type MergeRule struct { - Name string - Description string - Condition func(*CacheOperation, *CacheOperation) bool - MergeFunc func(*CacheOperation, *CacheOperation) (*CacheOperation, error) - Priority int - Enabled bool -} - -// MergeMetrics 合并指标 -type MergeMetrics struct { - // 时间统计 - AverageMergeTime time.Duration - MaxMergeTime time.Duration - TotalMergeTime time.Duration - - // 数据统计 - DataSizeBefore int64 - DataSizeAfter int64 - CompressionRatio float64 - - // 类型统计 - MergesByType map[string]int64 - MergesByPlugin map[string]int64 - MergesByKeyword map[string]int64 - - // 效率统计 - DuplicatesRemoved int64 - ResultsConsolidated int64 - StorageSaved int64 -} - -// NewAdvancedDataMerger 创建高级数据合并器 -func NewAdvancedDataMerger() *AdvancedDataMerger { - merger := &AdvancedDataMerger{ - mergeStrategies: make(map[string]MergeStrategy), - deduplicationMap: make(map[string]*CacheOperation), - mergeMetrics: &MergeMetrics{ - MergesByType: make(map[string]int64), - MergesByPlugin: make(map[string]int64), - MergesByKeyword: make(map[string]int64), - }, - } - - // 初始化合并策略 - merger.initializeMergeStrategies() - - // 初始化合并规则 - merger.initializeMergeRules() - - return merger -} - -// initializeMergeStrategies 初始化合并策略 -func (m *AdvancedDataMerger) initializeMergeStrategies() { - // 注册同键合并策略 - m.mergeStrategies["same_key"] = &SameKeyMergeStrategy{} - - // 注册同插件同关键词策略 - m.mergeStrategies["same_plugin_keyword"] = &SamePluginKeywordMergeStrategy{} - - // 注册结果去重策略 - m.mergeStrategies["deduplication"] = &DeduplicationMergeStrategy{} - - // 注册内容相似性策略 - m.mergeStrategies["content_similarity"] = &ContentSimilarityMergeStrategy{} -} - -// initializeMergeRules 初始化合并规则 -func (m *AdvancedDataMerger) initializeMergeRules() { - m.mergeRules = []*MergeRule{ - { - Name: "完全相同键合并", - Description: "合并具有完全相同缓存键的操作", - Condition: func(existing, new *CacheOperation) bool { - return existing.Key == new.Key - }, - MergeFunc: m.mergeSameKey, - Priority: 1, - Enabled: true, - }, - { - Name: "同插件同关键词合并", - Description: "合并同一插件对同一关键词的搜索结果", - Condition: func(existing, new *CacheOperation) bool { - return existing.PluginName == new.PluginName && - existing.Keyword == new.Keyword && - existing.Key != new.Key - }, - MergeFunc: m.mergeSamePluginKeyword, - Priority: 2, - Enabled: true, - }, - { - Name: "时间窗口内合并", - Description: "合并时间窗口内的相似操作", - Condition: func(existing, new *CacheOperation) bool { - timeDiff := new.Timestamp.Sub(existing.Timestamp) - return timeDiff >= 0 && timeDiff <= 5*time.Minute && - existing.PluginName == new.PluginName - }, - MergeFunc: m.mergeTimeWindow, - Priority: 3, - Enabled: true, - }, - { - Name: "结果去重合并", - Description: "去除重复的搜索结果", - Condition: func(existing, new *CacheOperation) bool { - return m.hasOverlapResults(existing, new) - }, - MergeFunc: m.mergeDeduplication, - Priority: 4, - Enabled: true, - }, - } -} - -// TryMergeOperation 尝试合并操作 -func (m *AdvancedDataMerger) TryMergeOperation(buffer *GlobalBuffer, newOp *CacheOperation) bool { - startTime := time.Now() - defer func() { - mergeTime := time.Since(startTime) - m.updateMergeMetrics(mergeTime) - }() - - m.totalMerges++ - - // 🔍 在缓冲区中寻找可合并的操作 - merged := false - - for i, existingOp := range buffer.Operations { - if m.canMergeOperations(existingOp, newOp) { - // 🚀 执行合并 - mergedOp, err := m.performMerge(existingOp, newOp) - if err != nil { - m.failedMerges++ - continue - } - - // 替换原操作 - buffer.Operations[i] = mergedOp - - // 更新统计 - m.successfulMerges++ - m.updateMergeStatistics(existingOp, newOp, mergedOp) - - merged = true - break - } - } - - return merged -} - -// canMergeOperations 检查是否可以合并操作 -func (m *AdvancedDataMerger) canMergeOperations(existing, new *CacheOperation) bool { - // 按优先级检查合并规则 - for _, rule := range m.mergeRules { - if rule.Enabled && rule.Condition(existing, new) { - return true - } - } - - return false -} - -// performMerge 执行合并 -func (m *AdvancedDataMerger) performMerge(existing, new *CacheOperation) (*CacheOperation, error) { - // 找到最高优先级的适用规则 - var bestRule *MergeRule - for _, rule := range m.mergeRules { - if rule.Enabled && rule.Condition(existing, new) { - if bestRule == nil || rule.Priority < bestRule.Priority { - bestRule = rule - } - } - } - - if bestRule == nil { - return nil, fmt.Errorf("未找到适用的合并规则") - } - - // 执行合并 - return bestRule.MergeFunc(existing, new) -} - -// mergeSameKey 合并相同键的操作 -func (m *AdvancedDataMerger) mergeSameKey(existing, new *CacheOperation) (*CacheOperation, error) { - // 合并搜索结果 - mergedResults := m.mergeSearchResults(existing.Data, new.Data) - - merged := &CacheOperation{ - Key: existing.Key, - Data: mergedResults, - TTL: m.chooseLongerTTL(existing.TTL, new.TTL), - PluginName: existing.PluginName, // 保持原插件名 - Keyword: existing.Keyword, // 保持原关键词 - Timestamp: new.Timestamp, // 使用最新时间戳 - Priority: m.chooseBetterPriority(existing.Priority, new.Priority), - DataSize: existing.DataSize + new.DataSize, // 累计数据大小 - IsFinal: existing.IsFinal || new.IsFinal, // 任一为最终结果则为最终结果 - } - - return merged, nil -} - -// mergeSamePluginKeyword 合并同插件同关键词操作 -func (m *AdvancedDataMerger) mergeSamePluginKeyword(existing, new *CacheOperation) (*CacheOperation, error) { - // 生成新的合并键 - mergedKey := fmt.Sprintf("merged_%s_%s_%d", - existing.PluginName, existing.Keyword, time.Now().Unix()) - - // 合并搜索结果 - mergedResults := m.mergeSearchResults(existing.Data, new.Data) - - merged := &CacheOperation{ - Key: mergedKey, - Data: mergedResults, - TTL: m.chooseLongerTTL(existing.TTL, new.TTL), - PluginName: existing.PluginName, - Keyword: existing.Keyword, - Timestamp: new.Timestamp, - Priority: m.chooseBetterPriority(existing.Priority, new.Priority), - DataSize: len(mergedResults) * 500, // 重新估算数据大小 - IsFinal: existing.IsFinal || new.IsFinal, - } - - return merged, nil -} - -// mergeTimeWindow 合并时间窗口内的操作 -func (m *AdvancedDataMerger) mergeTimeWindow(existing, new *CacheOperation) (*CacheOperation, error) { - // 时间窗口合并策略：保留最新的元信息，合并数据 - mergedResults := m.mergeSearchResults(existing.Data, new.Data) - - merged := &CacheOperation{ - Key: new.Key, // 使用新的键 - Data: mergedResults, - TTL: new.TTL, // 使用新的TTL - PluginName: new.PluginName, - Keyword: new.Keyword, - Timestamp: new.Timestamp, - Priority: new.Priority, - DataSize: len(mergedResults) * 500, - IsFinal: new.IsFinal, - } - - return merged, nil -} - -// mergeDeduplication 去重合并 -func (m *AdvancedDataMerger) mergeDeduplication(existing, new *CacheOperation) (*CacheOperation, error) { - // 执行深度去重 - deduplicatedResults := m.deduplicateSearchResults(existing.Data, new.Data) - - merged := &CacheOperation{ - Key: existing.Key, - Data: deduplicatedResults, - TTL: m.chooseLongerTTL(existing.TTL, new.TTL), - PluginName: existing.PluginName, - Keyword: existing.Keyword, - Timestamp: new.Timestamp, - Priority: m.chooseBetterPriority(existing.Priority, new.Priority), - DataSize: len(deduplicatedResults) * 500, - IsFinal: existing.IsFinal || new.IsFinal, - } - - return merged, nil -} - -// mergeSearchResults 合并搜索结果 -func (m *AdvancedDataMerger) mergeSearchResults(existing, new []model.SearchResult) []model.SearchResult { - // 使用map去重 - resultMap := make(map[string]model.SearchResult) - - // 添加现有结果 - for _, result := range existing { - key := m.generateResultKey(result) - resultMap[key] = result - } - - // 添加新结果，自动去重 - for _, result := range new { - key := m.generateResultKey(result) - if existingResult, exists := resultMap[key]; exists { - // 合并相同结果的信息 - mergedResult := m.mergeIndividualResults(existingResult, result) - resultMap[key] = mergedResult - } else { - resultMap[key] = result - } - } - - // 转换回切片 - merged := make([]model.SearchResult, 0, len(resultMap)) - for _, result := range resultMap { - merged = append(merged, result) - } - - return merged -} - -// deduplicateSearchResults 深度去重搜索结果 -func (m *AdvancedDataMerger) deduplicateSearchResults(existing, new []model.SearchResult) []model.SearchResult { - // 更严格的去重逻辑 - resultMap := make(map[string]model.SearchResult) - duplicateCount := 0 - - // 处理现有结果 - for _, result := range existing { - key := m.generateResultKey(result) - resultMap[key] = result - } - - // 处理新结果 - for _, result := range new { - key := m.generateResultKey(result) - if _, exists := resultMap[key]; !exists { - resultMap[key] = result - } else { - duplicateCount++ - } - } - - // 更新去重统计 - m.mergeMetrics.DuplicatesRemoved += int64(duplicateCount) - - // 转换回切片 - deduplicated := make([]model.SearchResult, 0, len(resultMap)) - for _, result := range resultMap { - deduplicated = append(deduplicated, result) - } - - return deduplicated -} - -// generateResultKey 生成结果键用于去重 -func (m *AdvancedDataMerger) generateResultKey(result model.SearchResult) string { - // 使用标题和主要链接生成唯一键 - key := result.Title - if len(result.Links) > 0 { - key += "_" + result.Links[0].URL - } - return key -} - -// mergeIndividualResults 合并单个结果 -func (m *AdvancedDataMerger) mergeIndividualResults(existing, new model.SearchResult) model.SearchResult { - merged := existing - - // 选择更完整的内容 - if len(new.Content) > len(existing.Content) { - merged.Content = new.Content - } - - // 合并链接 - linkMap := make(map[string]model.Link) - for _, link := range existing.Links { - linkMap[link.URL] = link - } - for _, link := range new.Links { - linkMap[link.URL] = link - } - - links := make([]model.Link, 0, len(linkMap)) - for _, link := range linkMap { - links = append(links, link) - } - merged.Links = links - - // 合并标签 - tagMap := make(map[string]bool) - for _, tag := range existing.Tags { - tagMap[tag] = true - } - for _, tag := range new.Tags { - tagMap[tag] = true - } - - tags := make([]string, 0, len(tagMap)) - for tag := range tagMap { - tags = append(tags, tag) - } - merged.Tags = tags - - // 使用更新的时间 - if new.Datetime.After(existing.Datetime) { - merged.Datetime = new.Datetime - } - - return merged -} - -// hasOverlapResults 检查是否有重叠结果 -func (m *AdvancedDataMerger) hasOverlapResults(existing, new *CacheOperation) bool { - if len(existing.Data) == 0 || len(new.Data) == 0 { - return false - } - - // 简单重叠检测：检查前几个结果的标题 - checkCount := 3 - if len(existing.Data) < checkCount { - checkCount = len(existing.Data) - } - if len(new.Data) < checkCount { - checkCount = len(new.Data) - } - - for i := 0; i < checkCount; i++ { - for j := 0; j < checkCount; j++ { - if existing.Data[i].Title == new.Data[j].Title { - return true - } - } - } - - return false -} - -// chooseLongerTTL 选择更长的TTL -func (m *AdvancedDataMerger) chooseLongerTTL(ttl1, ttl2 time.Duration) time.Duration { - if ttl1 > ttl2 { - return ttl1 - } - return ttl2 -} - -// chooseBetterPriority 选择更好的优先级 -func (m *AdvancedDataMerger) chooseBetterPriority(priority1, priority2 int) int { - if priority1 < priority2 { // 数字越小优先级越高 - return priority1 - } - return priority2 -} - -// updateMergeMetrics 更新合并指标 -func (m *AdvancedDataMerger) updateMergeMetrics(mergeTime time.Duration) { - m.mutex.Lock() - defer m.mutex.Unlock() - - m.mergeMetrics.TotalMergeTime += mergeTime - - // 更新平均时间 - if m.successfulMerges > 0 { - m.mergeMetrics.AverageMergeTime = time.Duration( - int64(m.mergeMetrics.TotalMergeTime) / m.successfulMerges) - } - - // 更新最大时间 - if mergeTime > m.mergeMetrics.MaxMergeTime { - m.mergeMetrics.MaxMergeTime = mergeTime - } -} - -// updateMergeStatistics 更新合并统计 -func (m *AdvancedDataMerger) updateMergeStatistics(existing, new, merged *CacheOperation) { - m.mutex.Lock() - defer m.mutex.Unlock() - - // 数据大小统计 - beforeSize := int64(existing.DataSize + new.DataSize) - afterSize := int64(merged.DataSize) - - m.mergeMetrics.DataSizeBefore += beforeSize - m.mergeMetrics.DataSizeAfter += afterSize - - // 计算压缩比例 - if m.mergeMetrics.DataSizeBefore > 0 { - m.mergeMetrics.CompressionRatio = float64(m.mergeMetrics.DataSizeAfter) / - float64(m.mergeMetrics.DataSizeBefore) - } - - // 按类型统计 - m.mergeMetrics.MergesByPlugin[merged.PluginName]++ - m.mergeMetrics.MergesByKeyword[merged.Keyword]++ - - // 结果整合统计 - originalCount := int64(len(existing.Data) + len(new.Data)) - mergedCount := int64(len(merged.Data)) - consolidated := originalCount - mergedCount - - if consolidated > 0 { - m.mergeMetrics.ResultsConsolidated += consolidated - m.mergeMetrics.StorageSaved += beforeSize - afterSize - } -} - -// GetMergeStats 获取合并统计 -func (m *AdvancedDataMerger) GetMergeStats() map[string]interface{} { - m.mutex.RLock() - defer m.mutex.RUnlock() - - successRate := float64(0) - if m.totalMerges > 0 { - successRate = float64(m.successfulMerges) / float64(m.totalMerges) - } - - return map[string]interface{}{ - "total_merges": m.totalMerges, - "successful_merges": m.successfulMerges, - "failed_merges": m.failedMerges, - "success_rate": successRate, - "merge_metrics": m.mergeMetrics, - "average_merge_time": m.mergeMetrics.AverageMergeTime, - "max_merge_time": m.mergeMetrics.MaxMergeTime, - "compression_ratio": m.mergeMetrics.CompressionRatio, - "duplicates_removed": m.mergeMetrics.DuplicatesRemoved, - "results_consolidated": m.mergeMetrics.ResultsConsolidated, - "storage_saved": m.mergeMetrics.StorageSaved, - } -} - -// 实现各种合并策略 - -// SameKeyMergeStrategy 相同键合并策略 -type SameKeyMergeStrategy struct{} - -func (s *SameKeyMergeStrategy) CanMerge(existing, new *CacheOperation) bool { - return existing.Key == new.Key -} - -func (s *SameKeyMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) { - // 委托给合并器的方法 - return nil, fmt.Errorf("应该使用合并器的方法") -} - -func (s *SameKeyMergeStrategy) GetPriority() int { - return 1 -} - -// SamePluginKeywordMergeStrategy 同插件同关键词合并策略 -type SamePluginKeywordMergeStrategy struct{} - -func (s *SamePluginKeywordMergeStrategy) CanMerge(existing, new *CacheOperation) bool { - return existing.PluginName == new.PluginName && existing.Keyword == new.Keyword -} - -func (s *SamePluginKeywordMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) { - return nil, fmt.Errorf("应该使用合并器的方法") -} - -func (s *SamePluginKeywordMergeStrategy) GetPriority() int { - return 2 -} - -// DeduplicationMergeStrategy 去重合并策略 -type DeduplicationMergeStrategy struct{} - -func (s *DeduplicationMergeStrategy) CanMerge(existing, new *CacheOperation) bool { - // 检查是否有重复结果 - return len(existing.Data) > 0 && len(new.Data) > 0 -} - -func (s *DeduplicationMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) { - return nil, fmt.Errorf("应该使用合并器的方法") -} - -func (s *DeduplicationMergeStrategy) GetPriority() int { - return 4 -} - -// ContentSimilarityMergeStrategy 内容相似性合并策略 -type ContentSimilarityMergeStrategy struct{} - -func (s *ContentSimilarityMergeStrategy) CanMerge(existing, new *CacheOperation) bool { - // 简单的相似性检测：关键词相似度 - return existing.Keyword == new.Keyword || - (len(existing.Keyword) > 3 && len(new.Keyword) > 3 && - existing.Keyword[:3] == new.Keyword[:3]) -} - -func (s *ContentSimilarityMergeStrategy) Merge(existing, new *CacheOperation) (*CacheOperation, error) { - return nil, fmt.Errorf("应该使用合并器的方法") -} - -func (s *ContentSimilarityMergeStrategy) GetPriority() int { - return 5 -} \ No newline at end of file diff --git a/util/cache/buffer_status_monitor.go b/util/cache/buffer_status_monitor.go deleted file mode 100644 index e7322b0..0000000 --- a/util/cache/buffer_status_monitor.go +++ /dev/null @@ -1,905 +0,0 @@ -package cache - -import ( - "fmt" - "runtime" - "sync" - "sync/atomic" - "time" - - "pansou/util/json" -) - -// BufferStatusMonitor 缓冲区状态监控器 -type BufferStatusMonitor struct { - // 监控配置 - monitorInterval time.Duration - alertThresholds *AlertThresholds - - // 监控状态 - isMonitoring int32 - shutdownChan chan struct{} - - // 健康检查 - healthChecker *HealthChecker - - // 报警系统 - alertManager *AlertManager - - // 性能指标 - performanceMetrics *PerformanceMetrics - - // 监控数据 - monitoringData *MonitoringData - dataMutex sync.RWMutex - - // 历史记录 - historyBuffer []MonitorSnapshot - historyMutex sync.Mutex - maxHistorySize int -} - -// AlertThresholds 报警阈值 -type AlertThresholds struct { - // 内存阈值 - MemoryUsageWarning int64 // 内存使用警告阈值（字节） - MemoryUsageCritical int64 // 内存使用严重阈值（字节） - - // 缓冲区阈值 - BufferCountWarning int // 缓冲区数量警告阈值 - BufferCountCritical int // 缓冲区数量严重阈值 - - // 操作阈值 - OperationQueueWarning int // 操作队列警告阈值 - OperationQueueCritical int // 操作队列严重阈值 - - // 时间阈值 - ProcessTimeWarning time.Duration // 处理时间警告阈值 - ProcessTimeCritical time.Duration // 处理时间严重阈值 - - // 成功率阈值 - SuccessRateWarning float64 // 成功率警告阈值 - SuccessRateCritical float64 // 成功率严重阈值 -} - -// HealthChecker 健康检查器 -type HealthChecker struct { - lastHealthCheck time.Time - healthCheckInterval time.Duration - healthStatus HealthStatus - healthHistory []HealthCheckResult - mutex sync.RWMutex -} - -// HealthStatus 健康状态 -type HealthStatus struct { - Overall string `json:"overall"` // healthy, warning, critical - LastCheck time.Time `json:"last_check"` - Components map[string]ComponentHealth `json:"components"` - Issues []HealthIssue `json:"issues,omitempty"` -} - -// ComponentHealth 组件健康状态 -type ComponentHealth struct { - Status string `json:"status"` - LastCheck time.Time `json:"last_check"` - Metrics map[string]interface{} `json:"metrics"` - Message string `json:"message,omitempty"` -} - -// HealthIssue 健康问题 -type HealthIssue struct { - Component string `json:"component"` - Severity string `json:"severity"` // warning, critical - Message string `json:"message"` - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` - Count int `json:"count"` -} - -// HealthCheckResult 健康检查结果 -type HealthCheckResult struct { - Timestamp time.Time `json:"timestamp"` - Status string `json:"status"` - CheckTime time.Duration `json:"check_time"` - Issues []HealthIssue `json:"issues"` -} - -// AlertManager 报警管理器 -type AlertManager struct { - alerts []Alert - alertHistory []Alert - mutex sync.RWMutex - maxAlertHistory int - - // 报警配置 - alertCooldown map[string]time.Time // 报警冷却时间 - cooldownPeriod time.Duration // 冷却期间 -} - -// Alert 报警 -type Alert struct { - ID string `json:"id"` - Level string `json:"level"` // info, warning, critical - Component string `json:"component"` - Message string `json:"message"` - Timestamp time.Time `json:"timestamp"` - Metadata map[string]interface{} `json:"metadata,omitempty"` - Resolved bool `json:"resolved"` - ResolvedAt *time.Time `json:"resolved_at,omitempty"` -} - -// PerformanceMetrics 性能指标 -type PerformanceMetrics struct { - // CPU指标 - CPUUsage float64 `json:"cpu_usage"` - CPUHistory []float64 `json:"cpu_history"` - - // 内存指标 - MemoryUsage int64 `json:"memory_usage"` - MemoryHistory []int64 `json:"memory_history"` - GCStats runtime.MemStats `json:"gc_stats"` - - // 吞吐量指标 - OperationsPerSecond float64 `json:"operations_per_second"` - ThroughputHistory []float64 `json:"throughput_history"` - - // 延迟指标 - AverageLatency time.Duration `json:"average_latency"` - P95Latency time.Duration `json:"p95_latency"` - P99Latency time.Duration `json:"p99_latency"` - LatencyHistory []time.Duration `json:"latency_history"` - - // 错误率指标 - ErrorRate float64 `json:"error_rate"` - ErrorHistory []float64 `json:"error_history"` - - // 资源利用率 - DiskIORate float64 `json:"disk_io_rate"` - NetworkIORate float64 `json:"network_io_rate"` - - // 更新时间 - LastUpdated time.Time `json:"last_updated"` -} - -// MonitoringData 监控数据 -type MonitoringData struct { - // 系统状态 - SystemHealth HealthStatus `json:"system_health"` - PerformanceMetrics PerformanceMetrics `json:"performance_metrics"` - - // 缓冲区状态 - BufferStates map[string]BufferState `json:"buffer_states"` - GlobalBufferStats *GlobalBufferStats `json:"global_buffer_stats"` - - // 实时统计 - RealTimeStats RealTimeStats `json:"real_time_stats"` - - // 趋势分析 - TrendAnalysis TrendAnalysis `json:"trend_analysis"` - - // 预测数据 - Predictions PredictionData `json:"predictions"` -} - -// BufferState 缓冲区状态 -type BufferState struct { - ID string `json:"id"` - Size int `json:"size"` - Capacity int `json:"capacity"` - UtilizationRate float64 `json:"utilization_rate"` - LastActivity time.Time `json:"last_activity"` - OperationsPerMin float64 `json:"operations_per_min"` - AverageDataSize int64 `json:"average_data_size"` - CompressionRatio float64 `json:"compression_ratio"` - Health string `json:"health"` -} - -// RealTimeStats 实时统计 -type RealTimeStats struct { - ActiveOperations int `json:"active_operations"` - QueuedOperations int `json:"queued_operations"` - ProcessingRate float64 `json:"processing_rate"` - ThroughputMBps float64 `json:"throughput_mbps"` - CacheHitRate float64 `json:"cache_hit_rate"` - CompressionRatio float64 `json:"compression_ratio"` - ErrorRate float64 `json:"error_rate"` - LastUpdated time.Time `json:"last_updated"` -} - -// TrendAnalysis 趋势分析 -type TrendAnalysis struct { - MemoryTrend string `json:"memory_trend"` // increasing, decreasing, stable - ThroughputTrend string `json:"throughput_trend"` - ErrorRateTrend string `json:"error_rate_trend"` - BufferUsageTrend string `json:"buffer_usage_trend"` - AnalysisTime time.Time `json:"analysis_time"` - Confidence float64 `json:"confidence"` -} - -// PredictionData 预测数据 -type PredictionData struct { - MemoryUsageIn1Hour int64 `json:"memory_usage_in_1hour"` - MemoryUsageIn24Hours int64 `json:"memory_usage_in_24hours"` - BufferOverflowRisk float64 `json:"buffer_overflow_risk"` - SystemLoadPrediction float64 `json:"system_load_prediction"` - RecommendedActions []string `json:"recommended_actions"` - ConfidenceLevel float64 `json:"confidence_level"` - PredictionTime time.Time `json:"prediction_time"` -} - -// MonitorSnapshot 监控快照 -type MonitorSnapshot struct { - Timestamp time.Time `json:"timestamp"` - SystemHealth HealthStatus `json:"system_health"` - BufferCount int `json:"buffer_count"` - TotalMemoryUsage int64 `json:"total_memory_usage"` - OperationsPerSecond float64 `json:"operations_per_second"` - ErrorRate float64 `json:"error_rate"` - CacheHitRate float64 `json:"cache_hit_rate"` -} - -// NewBufferStatusMonitor 创建缓冲区状态监控器 -func NewBufferStatusMonitor() *BufferStatusMonitor { - monitor := &BufferStatusMonitor{ - monitorInterval: 30 * time.Second, // 30秒监控间隔 - shutdownChan: make(chan struct{}), - maxHistorySize: 288, // 保存24小时历史（每30秒一个，24*60*2=2880，简化为288） - alertThresholds: &AlertThresholds{ - MemoryUsageWarning: 50 * 1024 * 1024, // 50MB - MemoryUsageCritical: 100 * 1024 * 1024, // 100MB - BufferCountWarning: 40, - BufferCountCritical: 50, - OperationQueueWarning: 500, - OperationQueueCritical: 1000, - ProcessTimeWarning: 5 * time.Second, - ProcessTimeCritical: 15 * time.Second, - SuccessRateWarning: 0.95, // 95% - SuccessRateCritical: 0.90, // 90% - }, - monitoringData: &MonitoringData{ - BufferStates: make(map[string]BufferState), - RealTimeStats: RealTimeStats{}, - TrendAnalysis: TrendAnalysis{}, - Predictions: PredictionData{}, - }, - } - - // 初始化组件 - monitor.healthChecker = &HealthChecker{ - healthCheckInterval: 1 * time.Minute, - healthStatus: HealthStatus{ - Overall: "healthy", - Components: make(map[string]ComponentHealth), - Issues: make([]HealthIssue, 0), - }, - healthHistory: make([]HealthCheckResult, 0), - } - - monitor.alertManager = &AlertManager{ - alerts: make([]Alert, 0), - alertHistory: make([]Alert, 0), - maxAlertHistory: 1000, - alertCooldown: make(map[string]time.Time), - cooldownPeriod: 5 * time.Minute, // 5分钟冷却期 - } - - monitor.performanceMetrics = &PerformanceMetrics{ - CPUHistory: make([]float64, 0), - MemoryHistory: make([]int64, 0), - ThroughputHistory: make([]float64, 0), - LatencyHistory: make([]time.Duration, 0), - ErrorHistory: make([]float64, 0), - } - - return monitor -} - -// Start 启动监控器 -func (b *BufferStatusMonitor) Start(globalManager *GlobalBufferManager) { - if !atomic.CompareAndSwapInt32(&b.isMonitoring, 0, 1) { - return // 已经在监控中 - } - - // 状态监控器启动（静默） - - go b.monitoringLoop(globalManager) - go b.healthCheckLoop() - go b.alertProcessingLoop() -} - -// monitoringLoop 监控循环 -func (b *BufferStatusMonitor) monitoringLoop(globalManager *GlobalBufferManager) { - ticker := time.NewTicker(b.monitorInterval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - b.collectMetrics(globalManager) - b.analyzeData() - b.checkAlerts() - b.updatePredictions() - b.saveSnapshot() - - case <-b.shutdownChan: - return - } - } -} - -// healthCheckLoop 健康检查循环 -func (b *BufferStatusMonitor) healthCheckLoop() { - ticker := time.NewTicker(b.healthChecker.healthCheckInterval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - b.performHealthCheck() - - case <-b.shutdownChan: - return - } - } -} - -// alertProcessingLoop 报警处理循环 -func (b *BufferStatusMonitor) alertProcessingLoop() { - ticker := time.NewTicker(1 * time.Minute) // 每分钟检查一次报警 - defer ticker.Stop() - - for { - select { - case <-ticker.C: - b.processAlerts() - - case <-b.shutdownChan: - return - } - } -} - -// collectMetrics 收集指标 -func (b *BufferStatusMonitor) collectMetrics(globalManager *GlobalBufferManager) { - b.dataMutex.Lock() - defer b.dataMutex.Unlock() - - // 收集全局缓冲区统计 - b.monitoringData.GlobalBufferStats = globalManager.GetStats() - - // 收集缓冲区状态 - bufferInfo := globalManager.GetBufferInfo() - for id, info := range bufferInfo { - if infoMap, ok := info.(map[string]interface{}); ok { - bufferState := BufferState{ - ID: id, - LastActivity: time.Now(), - Health: "healthy", - } - - // 提取缓冲区信息 - if size, ok := infoMap["total_operations"].(int64); ok { - bufferState.Size = int(size) - } - if dataSize, ok := infoMap["total_data_size"].(int64); ok { - bufferState.AverageDataSize = dataSize - } - if ratio, ok := infoMap["compress_ratio"].(float64); ok { - bufferState.CompressionRatio = ratio - } - - b.monitoringData.BufferStates[id] = bufferState - } - } - - // 收集性能指标 - b.collectPerformanceMetrics() - - // 更新实时统计 - b.updateRealTimeStats() -} - -// collectPerformanceMetrics 收集性能指标 -func (b *BufferStatusMonitor) collectPerformanceMetrics() { - // 收集内存统计 - runtime.ReadMemStats(&b.performanceMetrics.GCStats) - - currentMemory := int64(b.performanceMetrics.GCStats.Alloc) - b.performanceMetrics.MemoryUsage = currentMemory - - // 更新内存历史 - b.performanceMetrics.MemoryHistory = append(b.performanceMetrics.MemoryHistory, currentMemory) - if len(b.performanceMetrics.MemoryHistory) > 100 { // 保留最近100个数据点 - b.performanceMetrics.MemoryHistory = b.performanceMetrics.MemoryHistory[1:] - } - - // 简化的CPU使用率估算（基于GC统计） - gcCPUPercent := float64(b.performanceMetrics.GCStats.GCCPUFraction) * 100 - b.performanceMetrics.CPUUsage = gcCPUPercent - - // 更新CPU历史 - b.performanceMetrics.CPUHistory = append(b.performanceMetrics.CPUHistory, gcCPUPercent) - if len(b.performanceMetrics.CPUHistory) > 100 { - b.performanceMetrics.CPUHistory = b.performanceMetrics.CPUHistory[1:] - } - - b.performanceMetrics.LastUpdated = time.Now() -} - -// updateRealTimeStats 更新实时统计 -func (b *BufferStatusMonitor) updateRealTimeStats() { - stats := &b.monitoringData.RealTimeStats - - if b.monitoringData.GlobalBufferStats != nil { - globalStats := b.monitoringData.GlobalBufferStats - - // 活跃操作数 - stats.ActiveOperations = int(globalStats.ActiveBuffers) - - // 处理速率（操作/秒） - if globalStats.TotalOperationsBuffered > 0 { - stats.ProcessingRate = float64(globalStats.TotalOperationsBuffered) / - time.Since(globalStats.LastCleanupTime).Seconds() - } - - // 压缩比例 - stats.CompressionRatio = globalStats.AverageCompressionRatio - - // 缓存命中率 - stats.CacheHitRate = globalStats.HitRate - } - - // 内存使用（MB/s） - if b.performanceMetrics.MemoryUsage > 0 { - stats.ThroughputMBps = float64(b.performanceMetrics.MemoryUsage) / 1024 / 1024 - } - - stats.LastUpdated = time.Now() -} - -// analyzeData 分析数据 -func (b *BufferStatusMonitor) analyzeData() { - b.analyzeTrends() - b.detectAnomalies() -} - -// analyzeTrends 分析趋势 -func (b *BufferStatusMonitor) analyzeTrends() { - trends := &b.monitoringData.TrendAnalysis - - // 内存趋势分析 - if len(b.performanceMetrics.MemoryHistory) >= 3 { - recent := b.performanceMetrics.MemoryHistory[len(b.performanceMetrics.MemoryHistory)-3:] - if recent[2] > recent[1] && recent[1] > recent[0] { - trends.MemoryTrend = "increasing" - } else if recent[2] < recent[1] && recent[1] < recent[0] { - trends.MemoryTrend = "decreasing" - } else { - trends.MemoryTrend = "stable" - } - } - - // 缓冲区使用趋势 - bufferCount := len(b.monitoringData.BufferStates) - if bufferCount > b.alertThresholds.BufferCountWarning { - trends.BufferUsageTrend = "increasing" - } else { - trends.BufferUsageTrend = "stable" - } - - trends.AnalysisTime = time.Now() - trends.Confidence = 0.8 // 简化的置信度 -} - -// detectAnomalies 检测异常 -func (b *BufferStatusMonitor) detectAnomalies() { - // 内存异常检测 - if b.performanceMetrics.MemoryUsage > b.alertThresholds.MemoryUsageCritical { - b.triggerAlert("memory", "critical", - fmt.Sprintf("内存使用过高: %d bytes", b.performanceMetrics.MemoryUsage)) - } else if b.performanceMetrics.MemoryUsage > b.alertThresholds.MemoryUsageWarning { - b.triggerAlert("memory", "warning", - fmt.Sprintf("内存使用警告: %d bytes", b.performanceMetrics.MemoryUsage)) - } - - // 缓冲区数量异常检测 - bufferCount := len(b.monitoringData.BufferStates) - if bufferCount > b.alertThresholds.BufferCountCritical { - b.triggerAlert("buffer_count", "critical", - fmt.Sprintf("缓冲区数量过多: %d", bufferCount)) - } else if bufferCount > b.alertThresholds.BufferCountWarning { - b.triggerAlert("buffer_count", "warning", - fmt.Sprintf("缓冲区数量警告: %d", bufferCount)) - } -} - -// checkAlerts 检查报警 -func (b *BufferStatusMonitor) checkAlerts() { - // 检查系统健康状态 - if b.healthChecker.healthStatus.Overall == "critical" { - b.triggerAlert("system_health", "critical", "系统健康状态严重") - } else if b.healthChecker.healthStatus.Overall == "warning" { - b.triggerAlert("system_health", "warning", "系统健康状态警告") - } -} - -// triggerAlert 触发报警 -func (b *BufferStatusMonitor) triggerAlert(component, level, message string) { - alertKey := fmt.Sprintf("%s_%s", component, level) - - // 检查冷却期 - b.alertManager.mutex.Lock() - if lastAlert, exists := b.alertManager.alertCooldown[alertKey]; exists { - if time.Since(lastAlert) < b.alertManager.cooldownPeriod { - b.alertManager.mutex.Unlock() - return // 还在冷却期内 - } - } - - // 创建新报警 - alert := Alert{ - ID: fmt.Sprintf("%s_%d", alertKey, time.Now().Unix()), - Level: level, - Component: component, - Message: message, - Timestamp: time.Now(), - Metadata: make(map[string]interface{}), - Resolved: false, - } - - // 添加相关指标作为元数据 - alert.Metadata["memory_usage"] = b.performanceMetrics.MemoryUsage - alert.Metadata["buffer_count"] = len(b.monitoringData.BufferStates) - alert.Metadata["cpu_usage"] = b.performanceMetrics.CPUUsage - - b.alertManager.alerts = append(b.alertManager.alerts, alert) - b.alertManager.alertCooldown[alertKey] = time.Now() - - b.alertManager.mutex.Unlock() - - // 输出报警日志 - // fmt.Printf("🚨 [报警] %s - %s: %s\n", level, component, message) -} - -// updatePredictions 更新预测 -func (b *BufferStatusMonitor) updatePredictions() { - predictions := &b.monitoringData.Predictions - - // 简化的内存使用预测 - if len(b.performanceMetrics.MemoryHistory) >= 5 { - history := b.performanceMetrics.MemoryHistory - recent := history[len(history)-5:] - - // 简单线性预测 - growth := float64(recent[4]-recent[0]) / 4 - predictions.MemoryUsageIn1Hour = recent[4] + int64(growth*120) // 2小时数据点预测1小时 - predictions.MemoryUsageIn24Hours = recent[4] + int64(growth*2880) // 预测24小时 - } - - // 缓冲区溢出风险评估 - bufferCount := len(b.monitoringData.BufferStates) - if bufferCount > b.alertThresholds.BufferCountWarning { - predictions.BufferOverflowRisk = float64(bufferCount) / float64(b.alertThresholds.BufferCountCritical) - } else { - predictions.BufferOverflowRisk = 0.1 - } - - // 推荐行动 - predictions.RecommendedActions = b.generateRecommendations() - predictions.ConfidenceLevel = 0.7 - predictions.PredictionTime = time.Now() -} - -// generateRecommendations 生成推荐 -func (b *BufferStatusMonitor) generateRecommendations() []string { - recommendations := make([]string, 0) - - // 基于内存使用推荐 - if b.performanceMetrics.MemoryUsage > b.alertThresholds.MemoryUsageWarning { - recommendations = append(recommendations, "考虑增加内存或减少缓冲区大小") - } - - // 基于缓冲区数量推荐 - bufferCount := len(b.monitoringData.BufferStates) - if bufferCount > b.alertThresholds.BufferCountWarning { - recommendations = append(recommendations, "考虑调整缓冲区清理频率") - } - - // 基于趋势推荐 - if b.monitoringData.TrendAnalysis.MemoryTrend == "increasing" { - recommendations = append(recommendations, "内存使用呈增长趋势，建议监控和优化") - } - - if len(recommendations) == 0 { - recommendations = append(recommendations, "系统运行正常，继续监控") - } - - return recommendations -} - -// performHealthCheck 执行健康检查 -func (b *BufferStatusMonitor) performHealthCheck() { - startTime := time.Now() - - b.healthChecker.mutex.Lock() - defer b.healthChecker.mutex.Unlock() - - health := &b.healthChecker.healthStatus - health.LastCheck = time.Now() - health.Issues = make([]HealthIssue, 0) - - // 检查内存健康 - memoryHealth := b.checkMemoryHealth() - health.Components["memory"] = memoryHealth - - // 检查缓冲区健康 - bufferHealth := b.checkBufferHealth() - health.Components["buffers"] = bufferHealth - - // 检查性能健康 - performanceHealth := b.checkPerformanceHealth() - health.Components["performance"] = performanceHealth - - // 确定整体健康状态 - health.Overall = b.determineOverallHealth() - - // 记录健康检查结果 - checkResult := HealthCheckResult{ - Timestamp: time.Now(), - Status: health.Overall, - CheckTime: time.Since(startTime), - Issues: health.Issues, - } - - b.healthChecker.healthHistory = append(b.healthChecker.healthHistory, checkResult) - if len(b.healthChecker.healthHistory) > 100 { // 保留最近100次检查 - b.healthChecker.healthHistory = b.healthChecker.healthHistory[1:] - } - - b.healthChecker.lastHealthCheck = time.Now() -} - -// checkMemoryHealth 检查内存健康 -func (b *BufferStatusMonitor) checkMemoryHealth() ComponentHealth { - health := ComponentHealth{ - Status: "healthy", - LastCheck: time.Now(), - Metrics: make(map[string]interface{}), - } - - memUsage := b.performanceMetrics.MemoryUsage - health.Metrics["usage_bytes"] = memUsage - health.Metrics["usage_mb"] = memUsage / 1024 / 1024 - - if memUsage > b.alertThresholds.MemoryUsageCritical { - health.Status = "critical" - health.Message = "内存使用严重过高" - } else if memUsage > b.alertThresholds.MemoryUsageWarning { - health.Status = "warning" - health.Message = "内存使用偏高" - } else { - health.Message = "内存使用正常" - } - - return health -} - -// checkBufferHealth 检查缓冲区健康 -func (b *BufferStatusMonitor) checkBufferHealth() ComponentHealth { - health := ComponentHealth{ - Status: "healthy", - LastCheck: time.Now(), - Metrics: make(map[string]interface{}), - } - - bufferCount := len(b.monitoringData.BufferStates) - health.Metrics["buffer_count"] = bufferCount - health.Metrics["max_buffers"] = b.alertThresholds.BufferCountCritical - - if bufferCount > b.alertThresholds.BufferCountCritical { - health.Status = "critical" - health.Message = "缓冲区数量过多" - } else if bufferCount > b.alertThresholds.BufferCountWarning { - health.Status = "warning" - health.Message = "缓冲区数量偏高" - } else { - health.Message = "缓冲区状态正常" - } - - return health -} - -// checkPerformanceHealth 检查性能健康 -func (b *BufferStatusMonitor) checkPerformanceHealth() ComponentHealth { - health := ComponentHealth{ - Status: "healthy", - LastCheck: time.Now(), - Metrics: make(map[string]interface{}), - } - - cpuUsage := b.performanceMetrics.CPUUsage - health.Metrics["cpu_usage"] = cpuUsage - health.Metrics["gc_cpu_fraction"] = b.performanceMetrics.GCStats.GCCPUFraction - - if cpuUsage > 80 { - health.Status = "warning" - health.Message = "CPU使用率偏高" - } else { - health.Message = "性能状态正常" - } - - return health -} - -// determineOverallHealth 确定整体健康状态 -func (b *BufferStatusMonitor) determineOverallHealth() string { - hasCritical := false - hasWarning := false - - for _, component := range b.healthChecker.healthStatus.Components { - switch component.Status { - case "critical": - hasCritical = true - case "warning": - hasWarning = true - } - } - - if hasCritical { - return "critical" - } else if hasWarning { - return "warning" - } - - return "healthy" -} - -// processAlerts 处理报警 -func (b *BufferStatusMonitor) processAlerts() { - b.alertManager.mutex.Lock() - defer b.alertManager.mutex.Unlock() - - // 检查是否有报警需要自动解决 - for i := range b.alertManager.alerts { - alert := &b.alertManager.alerts[i] - if !alert.Resolved { - if b.shouldResolveAlert(alert) { - now := time.Now() - alert.Resolved = true - alert.ResolvedAt = &now - - fmt.Printf("✅ [报警解决] %s - %s: %s\n", - alert.Level, alert.Component, alert.Message) - } - } - } - - // 移动已解决的报警到历史记录 - activeAlerts := make([]Alert, 0) - for _, alert := range b.alertManager.alerts { - if !alert.Resolved { - activeAlerts = append(activeAlerts, alert) - } else { - b.alertManager.alertHistory = append(b.alertManager.alertHistory, alert) - } - } - - b.alertManager.alerts = activeAlerts - - // 限制历史记录大小 - if len(b.alertManager.alertHistory) > b.alertManager.maxAlertHistory { - excess := len(b.alertManager.alertHistory) - b.alertManager.maxAlertHistory - b.alertManager.alertHistory = b.alertManager.alertHistory[excess:] - } -} - -// shouldResolveAlert 检查是否应该解决报警 -func (b *BufferStatusMonitor) shouldResolveAlert(alert *Alert) bool { - switch alert.Component { - case "memory": - return b.performanceMetrics.MemoryUsage < b.alertThresholds.MemoryUsageWarning - case "buffer_count": - return len(b.monitoringData.BufferStates) < b.alertThresholds.BufferCountWarning - case "system_health": - return b.healthChecker.healthStatus.Overall == "healthy" - } - - return false -} - -// saveSnapshot 保存监控快照 -func (b *BufferStatusMonitor) saveSnapshot() { - b.historyMutex.Lock() - defer b.historyMutex.Unlock() - - snapshot := MonitorSnapshot{ - Timestamp: time.Now(), - SystemHealth: b.healthChecker.healthStatus, - BufferCount: len(b.monitoringData.BufferStates), - TotalMemoryUsage: b.performanceMetrics.MemoryUsage, - OperationsPerSecond: b.monitoringData.RealTimeStats.ProcessingRate, - ErrorRate: b.monitoringData.RealTimeStats.ErrorRate, - CacheHitRate: b.monitoringData.RealTimeStats.CacheHitRate, - } - - b.historyBuffer = append(b.historyBuffer, snapshot) - - // 限制历史记录大小 - if len(b.historyBuffer) > b.maxHistorySize { - b.historyBuffer = b.historyBuffer[1:] - } -} - -// Stop 停止监控器 -func (b *BufferStatusMonitor) Stop() { - if !atomic.CompareAndSwapInt32(&b.isMonitoring, 1, 0) { - return - } - - close(b.shutdownChan) - fmt.Printf("🔍 [缓冲区状态监控器] 已停止监控\n") -} - -// GetMonitoringData 获取监控数据 -func (b *BufferStatusMonitor) GetMonitoringData() *MonitoringData { - b.dataMutex.RLock() - defer b.dataMutex.RUnlock() - - // 深拷贝监控数据 - dataCopy := *b.monitoringData - return &dataCopy -} - -// GetHealthStatus 获取健康状态 -func (b *BufferStatusMonitor) GetHealthStatus() HealthStatus { - b.healthChecker.mutex.RLock() - defer b.healthChecker.mutex.RUnlock() - - return b.healthChecker.healthStatus -} - -// GetActiveAlerts 获取活跃报警 -func (b *BufferStatusMonitor) GetActiveAlerts() []Alert { - b.alertManager.mutex.RLock() - defer b.alertManager.mutex.RUnlock() - - alerts := make([]Alert, len(b.alertManager.alerts)) - copy(alerts, b.alertManager.alerts) - return alerts -} - -// GetMonitorHistory 获取监控历史 -func (b *BufferStatusMonitor) GetMonitorHistory(limit int) []MonitorSnapshot { - b.historyMutex.Lock() - defer b.historyMutex.Unlock() - - if limit <= 0 || limit > len(b.historyBuffer) { - limit = len(b.historyBuffer) - } - - history := make([]MonitorSnapshot, limit) - startIndex := len(b.historyBuffer) - limit - copy(history, b.historyBuffer[startIndex:]) - - return history -} - -// ExportMonitoringReport 导出监控报告 -func (b *BufferStatusMonitor) ExportMonitoringReport() (string, error) { - report := map[string]interface{}{ - "timestamp": time.Now(), - "monitoring_data": b.GetMonitoringData(), - "health_status": b.GetHealthStatus(), - "active_alerts": b.GetActiveAlerts(), - "performance_metrics": b.performanceMetrics, - "recent_history": b.GetMonitorHistory(50), // 最近50个快照 - } - - jsonData, err := json.MarshalIndent(report, "", " ") - if err != nil { - return "", fmt.Errorf("导出监控报告失败: %v", err) - } - - return string(jsonData), nil -} \ No newline at end of file diff --git a/util/cache/delayed_batch_write_manager.go b/util/cache/delayed_batch_write_manager.go index 5f40fdb..834494e 100644 --- a/util/cache/delayed_batch_write_manager.go +++ b/util/cache/delayed_batch_write_manager.go @@ -41,25 +41,25 @@ type CacheOperation struct { // CacheWriteConfig 缓存写入配置 type CacheWriteConfig struct { - // 🎯 核心策略 + // 核心策略 Strategy CacheWriteStrategy `env:"CACHE_WRITE_STRATEGY" default:"hybrid"` - // ⏱️ 批量写入参数（自动计算，但可手动覆盖） + // 批量写入参数（自动计算，但可手动覆盖） MaxBatchInterval time.Duration `env:"BATCH_MAX_INTERVAL"` // 0表示自动计算 MaxBatchSize int `env:"BATCH_MAX_SIZE"` // 0表示自动计算 MaxBatchDataSize int `env:"BATCH_MAX_DATA_SIZE"` // 0表示自动计算 - // 🎛️ 行为参数 + // 行为参数 HighPriorityRatio float64 `env:"HIGH_PRIORITY_RATIO" default:"0.3"` EnableCompression bool // 默认启用操作合并 - // 📊 内部计算参数（运行时动态调整） + // 内部计算参数（运行时动态调整） idleThresholdCPU float64 // CPU空闲阈值 idleThresholdDisk float64 // 磁盘空闲阈值 forceFlushInterval time.Duration // 强制刷新间隔 autoTuneInterval time.Duration // 调优检查间隔 - // 🔧 约束边界（硬编码） + // 约束边界（硬编码） minBatchInterval time.Duration // 最小30秒 maxBatchInterval time.Duration // 最大10分钟 minBatchSize int // 最小10个 @@ -68,16 +68,16 @@ type CacheWriteConfig struct { // Initialize 初始化配置 func (c *CacheWriteConfig) Initialize() error { - // 🔧 设置硬编码约束边界 + // 设置硬编码约束边界 c.minBatchInterval = 30 * time.Second c.maxBatchInterval = 600 * time.Second // 10分钟 c.minBatchSize = 10 c.maxBatchSize = 1000 - // 🎯 加载环境变量 + // 加载环境变量 c.loadFromEnvironment() - // 🤖 自动计算最优参数（除非手动设置） + // 自动计算最优参数（除非手动设置） if c.MaxBatchInterval == 0 { c.MaxBatchInterval = c.calculateOptimalBatchInterval() } @@ -88,13 +88,13 @@ func (c *CacheWriteConfig) Initialize() error { c.MaxBatchDataSize = c.calculateOptimalDataSize() } - // 🔧 内部参数自动设置 + // 内部参数自动设置 c.forceFlushInterval = c.MaxBatchInterval * 5 // 5倍批量间隔 c.autoTuneInterval = 300 * time.Second // 5分钟调优间隔 c.idleThresholdCPU = 0.3 // CPU空闲阈值 c.idleThresholdDisk = 0.5 // 磁盘空闲阈值 - // ✅ 参数验证和约束 + // 参数验证和约束 return c.validateAndConstraint() } @@ -134,7 +134,7 @@ func (c *CacheWriteConfig) loadFromEnvironment() { // calculateOptimalBatchInterval 计算最优批量间隔 func (c *CacheWriteConfig) calculateOptimalBatchInterval() time.Duration { - // 🎯 基于系统性能动态计算 + // 基于系统性能动态计算 var memStats runtime.MemStats runtime.ReadMemStats(&memStats) @@ -164,7 +164,7 @@ func (c *CacheWriteConfig) calculateOptimalBatchInterval() time.Duration { // calculateOptimalBatchSize 计算最优批量大小 func (c *CacheWriteConfig) calculateOptimalBatchSize() int { - // 🎯 基于CPU核心数和内存动态计算 + // 基于CPU核心数和内存动态计算 numCPU := runtime.NumCPU() var memStats runtime.MemStats @@ -194,7 +194,7 @@ func (c *CacheWriteConfig) calculateOptimalBatchSize() int { // calculateOptimalDataSize 计算最优数据大小 func (c *CacheWriteConfig) calculateOptimalDataSize() int { - // 🎯 基于可用内存计算 + // 基于可用内存计算 var memStats runtime.MemStats runtime.ReadMemStats(&memStats) availableMemoryGB := float64(memStats.Sys) / 1024 / 1024 / 1024 @@ -214,7 +214,7 @@ func (c *CacheWriteConfig) calculateOptimalDataSize() int { // validateAndConstraint 验证和约束配置 func (c *CacheWriteConfig) validateAndConstraint() error { - // 🔧 验证配置合理性 + // 验证配置合理性 if c.MaxBatchInterval < c.minBatchInterval { return fmt.Errorf("批量间隔配置错误: MaxBatchInterval(%v) < MinBatchInterval(%v)", c.MaxBatchInterval, c.minBatchInterval) @@ -230,7 +230,7 @@ func (c *CacheWriteConfig) validateAndConstraint() error { c.HighPriorityRatio) } - // 🎯 应用最终约束 + // 应用最终约束 if c.MaxBatchInterval > c.maxBatchInterval { c.MaxBatchInterval = c.maxBatchInterval } @@ -256,7 +256,7 @@ type DelayedBatchWriteManager struct { queueBuffer []*CacheOperation queueMutex sync.Mutex - // 🚀 全局缓冲区管理器 + // 全局缓冲区管理器 globalBufferManager *GlobalBufferManager // 统计信息 @@ -320,7 +320,7 @@ func NewDelayedBatchWriteManager() (*DelayedBatchWriteManager, error) { return nil, fmt.Errorf("配置初始化失败: %v", err) } - // 🚀 创建全局缓冲区管理器 + // 创建全局缓冲区管理器 globalBufferManager := NewGlobalBufferManager(BufferHybrid) manager := &DelayedBatchWriteManager{ @@ -349,7 +349,7 @@ func (m *DelayedBatchWriteManager) Initialize() error { m.initMutex.Lock() defer m.initMutex.Unlock() - // 🚀 初始化全局缓冲区管理器 + // 初始化全局缓冲区管理器 if err := m.globalBufferManager.Initialize(); err != nil { return fmt.Errorf("全局缓冲区管理器初始化失败: %v", err) } @@ -364,7 +364,7 @@ func (m *DelayedBatchWriteManager) Initialize() error { // 启动自动调优goroutine go m.autoTuningProcessor() - // 🔍 启动全局缓冲区监控 + // 启动全局缓冲区监控 go m.globalBufferMonitor() fmt.Printf("缓存写入策略: %s\n", m.strategy) @@ -383,7 +383,7 @@ func (m *DelayedBatchWriteManager) HandleCacheOperation(op *CacheOperation) erro return err } - // 🔥 关键：无论什么策略，都立即更新内存缓存 + // 关键：无论什么策略，都立即更新内存缓存 if err := m.updateMemoryCache(op); err != nil { return fmt.Errorf("内存缓存更新失败: %v", err) } @@ -393,20 +393,20 @@ func (m *DelayedBatchWriteManager) HandleCacheOperation(op *CacheOperation) erro return m.immediateWriteToDisk(op) } - // 🚀 使用全局缓冲区管理器进行智能缓冲 + // 使用全局缓冲区管理器进行智能缓冲 return m.handleWithGlobalBuffer(op) } // handleWithGlobalBuffer 使用全局缓冲区处理操作 func (m *DelayedBatchWriteManager) handleWithGlobalBuffer(op *CacheOperation) error { - // 🎯 尝试添加到全局缓冲区 + // 尝试添加到全局缓冲区 buffer, shouldFlush, err := m.globalBufferManager.AddOperation(op) if err != nil { // 全局缓冲区失败，降级到本地队列 return m.enqueueForBatchWrite(op) } - // 🚀 如果需要刷新缓冲区 + // 如果需要刷新缓冲区 if shouldFlush { return m.flushGlobalBuffer(buffer.ID) } @@ -425,7 +425,7 @@ func (m *DelayedBatchWriteManager) flushGlobalBuffer(bufferID string) error { return nil } - // 🎯 按优先级排序操作 + // 按优先级排序操作 sort.Slice(operations, func(i, j int) bool { if operations[i].Priority != operations[j].Priority { return operations[i].Priority < operations[j].Priority @@ -433,14 +433,14 @@ func (m *DelayedBatchWriteManager) flushGlobalBuffer(bufferID string) error { return operations[i].Timestamp.Before(operations[j].Timestamp) }) - // 📊 统计信息更新 + // 统计信息更新 atomic.AddInt64(&m.stats.BatchWrites, 1) atomic.AddInt64(&m.stats.TotalWrites, 1) m.stats.LastFlushTime = time.Now() m.stats.LastFlushTrigger = "全局缓冲区触发" m.stats.LastBatchSize = len(operations) - // 🚀 批量写入磁盘 + // 批量写入磁盘 err = m.batchWriteToDisk(operations) if err != nil { atomic.AddInt64(&m.stats.FailedWrites, 1) @@ -462,7 +462,7 @@ func (m *DelayedBatchWriteManager) globalBufferMonitor() { for { select { case <-ticker.C: - // 🔍 检查是否有过期的缓冲区需要刷新 + // 检查是否有过期的缓冲区需要刷新 m.checkAndFlushExpiredBuffers() case <-m.shutdownChan: @@ -473,26 +473,26 @@ func (m *DelayedBatchWriteManager) globalBufferMonitor() { // checkAndFlushExpiredBuffers 检查并刷新过期缓冲区 func (m *DelayedBatchWriteManager) checkAndFlushExpiredBuffers() { - // 🔧 修复：使用原子操作获取需要刷新的缓冲区列表 + // 使用原子操作获取需要刷新的缓冲区列表 expiredBuffers := m.globalBufferManager.GetExpiredBuffersForFlush() flushedCount := 0 for _, bufferID := range expiredBuffers { if err := m.flushGlobalBuffer(bufferID); err != nil { - // 🎯 改进：区分错误类型，缓冲区不存在是正常情况 + // 区分错误类型，缓冲区不存在是正常情况 if isBufferNotExistError(err) { // 静默处理：缓冲区已被其他线程清理，这是正常的 continue } // 只有真正的错误才打印警告 - fmt.Printf("⚠️ [全局缓冲区] 刷新缓冲区失败 %s: %v\n", bufferID, err) + fmt.Printf("[全局缓冲区] 刷新缓冲区失败 %s: %v\n", bufferID, err) } else { flushedCount++ } } if flushedCount > 0 { - fmt.Printf("🔄 [全局缓冲区] 刷新完成，处理 %d 个过期缓冲区\n", flushedCount) + fmt.Printf("[全局缓冲区] 刷新完成，处理 %d 个过期缓冲区\n", flushedCount) } } @@ -505,18 +505,16 @@ func isBufferNotExistError(err error) bool { // updateMemoryCache 更新内存缓存（立即执行） func (m *DelayedBatchWriteManager) updateMemoryCache(op *CacheOperation) error { - // 🔥 关键修复：如果有主缓存更新函数，立即更新内存层 + // 如果有主缓存更新函数，立即更新内存层 if m.mainCacheUpdater != nil { // 序列化数据 - data, err := m.serializer.Serialize(op.Data) + _, err := m.serializer.Serialize(op.Data) if err != nil { return fmt.Errorf("内存缓存数据序列化失败: %v", err) } // 这里只更新内存，不写磁盘（磁盘由批量写入处理） // 注意：mainCacheUpdater实际上是SetBothLevels，会同时更新内存和磁盘 - // 为了避免重复写磁盘，我们暂时保持原逻辑 - _ = data // 暂不使用，避免编译警告 } return nil } @@ -644,21 +642,21 @@ func (m *DelayedBatchWriteManager) Shutdown(timeout time.Duration) error { go func() { var lastErr error - // 🚀 第一步：强制刷新全局缓冲区（优先级最高） + // 第一步：强制刷新全局缓冲区（优先级最高） if err := m.flushAllGlobalBuffers(); err != nil { - fmt.Printf("❌ [数据保护] 全局缓冲区刷新失败: %v\n", err) + fmt.Printf("[数据保护] 全局缓冲区刷新失败: %v\n", err) lastErr = err } - // 🔧 第二步：刷新本地队列 + // 第二步：刷新本地队列 if err := m.flushAllPendingData(); err != nil { - fmt.Printf("❌ [数据保护] 本地队列刷新失败: %v\n", err) + fmt.Printf("[数据保护] 本地队列刷新失败: %v\n", err) lastErr = err } - // 🔄 第三步：关闭全局缓冲区管理器 + // 第三步：关闭全局缓冲区管理器 if err := m.globalBufferManager.Shutdown(); err != nil { - fmt.Printf("❌ [数据保护] 全局缓冲区管理器关闭失败: %v\n", err) + fmt.Printf("[数据保护] 全局缓冲区管理器关闭失败: %v\n", err) lastErr = err } @@ -686,7 +684,7 @@ func (m *DelayedBatchWriteManager) flushAllGlobalBuffers() error { for bufferID, operations := range allBuffers { if len(operations) > 0 { if err := m.batchWriteToDisk(operations); err != nil { - fmt.Printf("❌ [全局缓冲区] 缓冲区 %s 刷新失败: %v\n", bufferID, err) + fmt.Printf("[全局缓冲区] 缓冲区 %s 刷新失败: %v\n", bufferID, err) lastErr = fmt.Errorf("刷新全局缓冲区 %s 失败: %v", bufferID, err) continue } @@ -799,7 +797,7 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error { return nil } - // 🔧 操作合并：如果启用压缩，使用合并后的操作 + // 操作合并：如果启用压缩，使用合并后的操作 var operations []*CacheOperation if m.config.EnableCompression { operations = m.getCompressedOperations() @@ -812,7 +810,7 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error { return nil } - // 🎯 按优先级排序：确保重要数据优先写入 + // 按优先级排序：确保重要数据优先写入 sort.Slice(operations, func(i, j int) bool { if operations[i].Priority != operations[j].Priority { return operations[i].Priority < operations[j].Priority // 数字越小优先级越高 @@ -820,13 +818,13 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error { return operations[i].Timestamp.Before(operations[j].Timestamp) }) - // 📊 统计信息更新 + // 统计信息更新 atomic.AddInt64(&m.stats.BatchWrites, 1) m.stats.LastFlushTime = time.Now() m.stats.LastFlushTrigger = trigger m.stats.LastBatchSize = len(operations) - // 🚀 批量写入磁盘 + // 批量写入磁盘 err := m.batchWriteToDisk(operations) if err != nil { atomic.AddInt64(&m.stats.FailedWrites, 1) @@ -841,7 +839,7 @@ func (m *DelayedBatchWriteManager) executeBatchWrite(trigger string) error { m.mapMutex.Unlock() } - // 📈 成功统计 + // 成功统计 atomic.AddInt64(&m.stats.SuccessfulWrites, 1) atomic.AddInt64(&m.stats.TotalWrites, 1) m.stats.TotalOperationsWritten += len(operations) @@ -895,10 +893,10 @@ func (m *DelayedBatchWriteManager) emergencyFlush() error { // autoTuneParameters 自适应参数调优 func (m *DelayedBatchWriteManager) autoTuneParameters() { - // 🤖 完全自动调优，无需配置开关 + // 完全自动调优，无需配置开关 stats := m.collectRecentStats() - // 🎯 调优批量间隔：基于系统负载动态调整 + // 调优批量间隔：基于系统负载动态调整 avgSystemLoad := stats.SystemLoadAverage switch { case avgSystemLoad > 0.8: // 高负载：延长间隔，减少干扰 @@ -907,7 +905,7 @@ func (m *DelayedBatchWriteManager) autoTuneParameters() { m.config.MaxBatchInterval = m.maxDuration(m.config.MaxBatchInterval*8/10, m.config.minBatchInterval) } - // 🎯 调优批量大小：基于写入频率动态调整 + // 调优批量大小：基于写入频率动态调整 queueSize := int(atomic.LoadInt32(&m.stats.CurrentQueueSize)) switch { case queueSize > 200: // 高频：增大批量，提高效率 @@ -954,20 +952,13 @@ func (m *DelayedBatchWriteManager) GetStats() map[string]interface{} { stats.SystemLoadAverage = float64(stats.TotalWrites) / float64(stats.TotalOperations) } - // 🚀 获取全局缓冲区统计 + // 获取全局缓冲区统计 globalBufferStats := m.globalBufferManager.GetStats() - // 🔍 获取监控数据（如果监控器存在） - var monitoringData *MonitoringData - if m.globalBufferManager.statusMonitor != nil { - monitoringData = m.globalBufferManager.statusMonitor.GetMonitoringData() - } - - // 🎯 合并所有统计信息 + // 合并所有统计信息 combinedStats := map[string]interface{}{ "write_manager": &stats, "global_buffer": globalBufferStats, - "monitoring": monitoringData, "buffer_info": m.globalBufferManager.GetBufferInfo(), } diff --git a/util/cache/disk_cache.go b/util/cache/disk_cache.go index 7ecabb2..ff838e9 100644 --- a/util/cache/disk_cache.go +++ b/util/cache/disk_cache.go @@ -135,7 +135,7 @@ func (c *DiskCache) Set(key string, data []byte, ttl time.Duration) error { filename := c.getFilename(key) filePath := filepath.Join(c.path, filename) - // 🔥 确保目录存在（防止外部删除缓存目录） + // 确保目录存在（防止外部删除缓存目录） if err := os.MkdirAll(c.path, 0755); err != nil { return fmt.Errorf("创建缓存目录失败: %v", err) } diff --git a/util/cache/enhanced_two_level_cache.go b/util/cache/enhanced_two_level_cache.go index e4d6c2e..fec4275 100644 --- a/util/cache/enhanced_two_level_cache.go +++ b/util/cache/enhanced_two_level_cache.go @@ -34,7 +34,7 @@ func NewEnhancedTwoLevelCache() (*EnhancedTwoLevelCache, error) { // 创建序列化器 serializer := NewGobSerializer() - // 🔥 设置内存缓存的磁盘缓存引用，用于LRU淘汰时的备份 + // 设置内存缓存的磁盘缓存引用，用于LRU淘汰时的备份 memCache.SetDiskCacheReference(diskCache) return &EnhancedTwoLevelCache{ @@ -65,7 +65,7 @@ func (c *EnhancedTwoLevelCache) Set(key string, data []byte, ttl time.Duration) func (c *EnhancedTwoLevelCache) SetMemoryOnly(key string, data []byte, ttl time.Duration) error { now := time.Now() - // 🔥 只更新内存缓存，不触发磁盘写入 + // 只更新内存缓存，不触发磁盘写入 c.memory.SetWithTimestamp(key, data, ttl, now) return nil @@ -78,7 +78,7 @@ func (c *EnhancedTwoLevelCache) SetBothLevels(key string, data []byte, ttl time. // 同步更新内存缓存 c.memory.SetWithTimestamp(key, data, ttl, now) - // 🔥 修复：同步更新磁盘缓存，确保数据立即写入 + // 同步更新磁盘缓存，确保数据立即写入 return c.disk.Set(key, data, ttl) } @@ -155,7 +155,7 @@ func (c *EnhancedTwoLevelCache) FlushMemoryToDisk() error { for key, item := range allItems { // 同步写入到磁盘缓存 if err := c.disk.Set(key, item.Data, item.TTL); err != nil { - fmt.Printf("❌ [内存同步] 同步失败: %s -> %v\n", key, err) + fmt.Printf("[内存同步] 同步失败: %s -> %v\n", key, err) lastErr = err continue } diff --git a/util/cache/global_buffer_manager.go b/util/cache/global_buffer_manager.go index 3381718..3b86fb2 100644 --- a/util/cache/global_buffer_manager.go +++ b/util/cache/global_buffer_manager.go @@ -71,14 +71,7 @@ type GlobalBufferManager struct { buffers map[string]*GlobalBuffer // 缓冲区映射 buffersMutex sync.RWMutex // 缓冲区锁 - // 搜索模式分析 - patternAnalyzer *SearchPatternAnalyzer - - // 数据合并器 - dataMerger *AdvancedDataMerger - - // 状态监控 - statusMonitor *BufferStatusMonitor + // 已移除：搜索模式分析、数据合并器、状态监控 // 统计信息 stats *GlobalBufferStats @@ -132,10 +125,7 @@ func NewGlobalBufferManager(strategy GlobalBufferStrategy) *GlobalBufferManager }, } - // 初始化组件 - manager.patternAnalyzer = NewSearchPatternAnalyzer() - manager.dataMerger = NewAdvancedDataMerger() - manager.statusMonitor = NewBufferStatusMonitor() + // 初始化组件（移除未使用的监控与合并器） return manager } @@ -150,8 +140,7 @@ func (g *GlobalBufferManager) Initialize() error { g.cleanupTicker = time.NewTicker(5 * time.Minute) // 每5分钟清理一次 go g.cleanupRoutine() - // 启动状态监控 - go g.statusMonitor.Start(g) + // 移除状态监控启动（监控已删除） // 初始化完成（静默） return nil @@ -163,13 +152,13 @@ func (g *GlobalBufferManager) AddOperation(op *CacheOperation) (*GlobalBuffer, b return nil, false, err } - // 🎯 根据策略确定缓冲区ID + // 根据策略确定缓冲区ID bufferID := g.determineBufferID(op) g.buffersMutex.Lock() defer g.buffersMutex.Unlock() - // 🔧 获取或创建缓冲区 + // 获取或创建缓冲区 buffer, exists := g.buffers[bufferID] if !exists { buffer = g.createNewBuffer(bufferID, op) @@ -178,10 +167,10 @@ func (g *GlobalBufferManager) AddOperation(op *CacheOperation) (*GlobalBuffer, b atomic.AddInt64(&g.stats.ActiveBuffers, 1) } - // 🚀 添加操作到缓冲区 + // 添加操作到缓冲区 shouldFlush := g.addOperationToBuffer(buffer, op) - // 📊 更新统计 + // 更新统计 atomic.AddInt64(&g.stats.TotalOperationsBuffered, 1) return buffer, shouldFlush, nil @@ -197,8 +186,8 @@ func (g *GlobalBufferManager) determineBufferID(op *CacheOperation) string { return fmt.Sprintf("plugin_%s", op.PluginName) case BufferByPattern: - pattern := g.patternAnalyzer.AnalyzePattern(op) - return fmt.Sprintf("pattern_%s", pattern.KeywordPattern) + // 已移除模式分析器，退化为按关键词分组 + return fmt.Sprintf("keyword_%s", op.Keyword) case BufferHybrid: // 混合策略优化：插件+时间窗口（去掉关键词避免高并发爆炸） @@ -236,33 +225,26 @@ func (g *GlobalBufferManager) addOperationToBuffer(buffer *GlobalBuffer, op *Cac buffer.mutex.Lock() defer buffer.mutex.Unlock() - // 🔧 数据合并优化 - merged := g.dataMerger.TryMergeOperation(buffer, op) - if merged { - atomic.AddInt64(&g.stats.TotalOperationsMerged, 1) - atomic.AddInt64(&g.stats.TotalDataMerged, int64(op.DataSize)) - } else { - // 添加新操作 - buffer.Operations = append(buffer.Operations, op) - buffer.TotalOperations++ - buffer.TotalDataSize += int64(op.DataSize) - - // 按关键词分组 - if buffer.KeywordGroups[op.Keyword] == nil { - buffer.KeywordGroups[op.Keyword] = make([]*CacheOperation, 0) - } - buffer.KeywordGroups[op.Keyword] = append(buffer.KeywordGroups[op.Keyword], op) - - // 按插件分组 - if buffer.PluginGroups[op.PluginName] == nil { - buffer.PluginGroups[op.PluginName] = make([]*CacheOperation, 0) - } - buffer.PluginGroups[op.PluginName] = append(buffer.PluginGroups[op.PluginName], op) + // 直接追加（已移除数据合并器） + buffer.Operations = append(buffer.Operations, op) + buffer.TotalOperations++ + buffer.TotalDataSize += int64(op.DataSize) + + // 按关键词分组 + if buffer.KeywordGroups[op.Keyword] == nil { + buffer.KeywordGroups[op.Keyword] = make([]*CacheOperation, 0) } + buffer.KeywordGroups[op.Keyword] = append(buffer.KeywordGroups[op.Keyword], op) + + // 按插件分组 + if buffer.PluginGroups[op.PluginName] == nil { + buffer.PluginGroups[op.PluginName] = make([]*CacheOperation, 0) + } + buffer.PluginGroups[op.PluginName] = append(buffer.PluginGroups[op.PluginName], op) buffer.LastUpdatedAt = time.Now() - // 🎯 检查是否应该刷新 + // 检查是否应该刷新 return g.shouldFlushBuffer(buffer) } @@ -503,10 +485,10 @@ func (g *GlobalBufferManager) GetExpiredBuffersForFlush() []string { defer g.buffersMutex.RUnlock() now := time.Now() - expiredBuffers := make([]string, 0, 10) // 🚀 预分配容量，减少内存重分配 + expiredBuffers := make([]string, 0, 10) // 预分配容量，减少内存重分配 for id, buffer := range g.buffers { - // 🎯 快速预检查：先检查时间，减少锁竞争 + // 快速预检查：先检查时间，减少锁竞争 if now.Sub(buffer.LastUpdatedAt) <= 4*time.Minute { continue // 跳过未过期的缓冲区 } diff --git a/util/cache/metric_collector.go b/util/cache/metric_collector.go deleted file mode 100644 index e282f66..0000000 --- a/util/cache/metric_collector.go +++ /dev/null @@ -1,208 +0,0 @@ -package cache - -import ( - "runtime" - "sync/atomic" - "time" -) - -// NewMetricCollector 创建指标收集器 -func NewMetricCollector() *MetricCollector { - return &MetricCollector{ - systemMetrics: &SystemMetrics{}, - applicationMetrics: &ApplicationMetrics{}, - cacheMetrics: &CacheMetrics{}, - metricsHistory: make([]MetricSnapshot, 0), - maxHistorySize: 1000, // 保留1000个历史快照 - collectionChan: make(chan struct{}), - } -} - -// Start 启动指标收集 -func (m *MetricCollector) Start(interval time.Duration) error { - if !atomic.CompareAndSwapInt32(&m.isCollecting, 0, 1) { - return nil // 已经在收集中 - } - - go m.collectionLoop(interval) - return nil -} - -// Stop 停止指标收集 -func (m *MetricCollector) Stop() { - if atomic.CompareAndSwapInt32(&m.isCollecting, 1, 0) { - close(m.collectionChan) - } -} - -// collectionLoop 收集循环 -func (m *MetricCollector) collectionLoop(interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - m.collectMetrics() - - case <-m.collectionChan: - return - } - } -} - -// collectMetrics 收集指标 -func (m *MetricCollector) collectMetrics() { - now := time.Now() - - // 收集系统指标 - systemMetrics := m.collectSystemMetrics(now) - - // 收集应用指标 - applicationMetrics := m.collectApplicationMetrics(now) - - // 收集缓存指标 - cacheMetrics := m.collectCacheMetrics(now) - - // 创建快照 - snapshot := MetricSnapshot{ - Timestamp: now, - System: *systemMetrics, - Application: *applicationMetrics, - Cache: *cacheMetrics, - } - - // 计算综合指标 - snapshot.OverallPerformance = m.calculateOverallPerformance(&snapshot) - snapshot.Efficiency = m.calculateEfficiency(&snapshot) - snapshot.Stability = m.calculateStability(&snapshot) - - // 保存快照 - m.saveSnapshot(snapshot) -} - -// collectSystemMetrics 收集系统指标 -func (m *MetricCollector) collectSystemMetrics(timestamp time.Time) *SystemMetrics { - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - - return &SystemMetrics{ - Timestamp: timestamp, - CPUUsage: float64(memStats.GCCPUFraction), - MemoryUsage: int64(memStats.Alloc), - MemoryTotal: int64(memStats.Sys), - DiskIORate: 0, // 简化实现 - NetworkIORate: 0, // 简化实现 - GoroutineCount: runtime.NumGoroutine(), - GCPauseDuration: time.Duration(memStats.PauseTotalNs), - HeapSize: int64(memStats.HeapSys), - AllocRate: float64(memStats.Mallocs - memStats.Frees), - } -} - -// collectApplicationMetrics 收集应用指标 -func (m *MetricCollector) collectApplicationMetrics(timestamp time.Time) *ApplicationMetrics { - // 简化实现，实际应用中应该从应用监控系统获取 - return &ApplicationMetrics{ - Timestamp: timestamp, - RequestRate: 100.0, // 模拟值 - ResponseTime: 50 * time.Millisecond, - ErrorRate: 0.01, - ThroughputMBps: 10.5, - ConcurrentUsers: 50, - QueueDepth: 5, - ProcessingRate: 95.5, - } -} - -// collectCacheMetrics 收集缓存指标 -func (m *MetricCollector) collectCacheMetrics(timestamp time.Time) *CacheMetrics { - // 简化实现，实际应用中应该从缓存系统获取 - return &CacheMetrics{ - Timestamp: timestamp, - HitRate: 0.85, - WriteRate: 20.0, - ReadRate: 80.0, - EvictionRate: 2.0, - CompressionRatio: 0.6, - StorageUsage: 1024 * 1024 * 100, // 100MB - BufferUtilization: 0.75, - BatchEfficiency: 0.9, - } -} - -// calculateOverallPerformance 计算整体性能 -func (m *MetricCollector) calculateOverallPerformance(snapshot *MetricSnapshot) float64 { - // 性能评分算法 - cpuScore := (1.0 - snapshot.System.CPUUsage) * 30 - memoryScore := (1.0 - float64(snapshot.System.MemoryUsage)/float64(snapshot.System.MemoryTotal)) * 25 - responseScore := (1.0 - float64(snapshot.Application.ResponseTime)/float64(time.Second)) * 25 - cacheScore := snapshot.Cache.HitRate * 20 - - return cpuScore + memoryScore + responseScore + cacheScore -} - -// calculateEfficiency 计算效率 -func (m *MetricCollector) calculateEfficiency(snapshot *MetricSnapshot) float64 { - // 效率评分算法 - cacheEfficiency := snapshot.Cache.HitRate * 0.4 - batchEfficiency := snapshot.Cache.BatchEfficiency * 0.3 - compressionEfficiency := snapshot.Cache.CompressionRatio * 0.3 - - return cacheEfficiency + batchEfficiency + compressionEfficiency -} - -// calculateStability 计算稳定性 -func (m *MetricCollector) calculateStability(snapshot *MetricSnapshot) float64 { - // 稳定性评分算法（基于变化率） - errorRateStability := (1.0 - snapshot.Application.ErrorRate) * 0.5 - responseTimeStability := 1.0 - (float64(snapshot.Application.ResponseTime) / float64(time.Second)) - if responseTimeStability < 0 { - responseTimeStability = 0 - } - responseTimeStability *= 0.5 - - return errorRateStability + responseTimeStability -} - -// saveSnapshot 保存快照 -func (m *MetricCollector) saveSnapshot(snapshot MetricSnapshot) { - m.historyMutex.Lock() - defer m.historyMutex.Unlock() - - m.metricsHistory = append(m.metricsHistory, snapshot) - - // 限制历史记录大小 - if len(m.metricsHistory) > m.maxHistorySize { - m.metricsHistory = m.metricsHistory[1:] - } -} - -// GetLatestMetrics 获取最新指标 -func (m *MetricCollector) GetLatestMetrics() *MetricSnapshot { - m.historyMutex.RLock() - defer m.historyMutex.RUnlock() - - if len(m.metricsHistory) == 0 { - return nil - } - - latest := m.metricsHistory[len(m.metricsHistory)-1] - return &latest -} - -// GetMetricsHistory 获取指标历史 -func (m *MetricCollector) GetMetricsHistory(limit int) []MetricSnapshot { - m.historyMutex.RLock() - defer m.historyMutex.RUnlock() - - if limit <= 0 || limit > len(m.metricsHistory) { - limit = len(m.metricsHistory) - } - - history := make([]MetricSnapshot, limit) - startIndex := len(m.metricsHistory) - limit - copy(history, m.metricsHistory[startIndex:]) - - return history -} \ No newline at end of file diff --git a/util/cache/performance_analyzer.go b/util/cache/performance_analyzer.go deleted file mode 100644 index f83b84a..0000000 --- a/util/cache/performance_analyzer.go +++ /dev/null @@ -1,479 +0,0 @@ -package cache - -import ( - "fmt" - "math" - "sort" - "sync" - "time" -) - -// TrendAnalyzer 趋势分析器 -type TrendAnalyzer struct { - trends map[string]*TrendData - mutex sync.RWMutex -} - -// TrendData 趋势数据 -type TrendData struct { - Values []float64 - Timestamps []time.Time - Slope float64 - RSquared float64 - LastUpdate time.Time -} - -// AnomalyDetector 异常检测器 -type AnomalyDetector struct { - baselines map[string]*Baseline - mutex sync.RWMutex -} - -// Baseline 基线数据 -type Baseline struct { - Mean float64 - StdDev float64 - Min float64 - Max float64 - SampleSize int - LastUpdate time.Time -} - -// CorrelationAnalyzer 相关性分析器 -type CorrelationAnalyzer struct { - correlationMatrix map[string]map[string]float64 - mutex sync.RWMutex -} - -// NewPerformanceAnalyzer 创建性能分析器 -func NewPerformanceAnalyzer() *PerformanceAnalyzer { - return &PerformanceAnalyzer{ - trendAnalyzer: &TrendAnalyzer{ - trends: make(map[string]*TrendData), - }, - anomalyDetector: &AnomalyDetector{ - baselines: make(map[string]*Baseline), - }, - correlationAnalyzer: &CorrelationAnalyzer{ - correlationMatrix: make(map[string]map[string]float64), - }, - currentTrends: make(map[string]Trend), - detectedAnomalies: make([]Anomaly, 0), - correlations: make(map[string]float64), - } -} - -// AnalyzeTrends 分析趋势 -func (p *PerformanceAnalyzer) AnalyzeTrends(history []MetricSnapshot) { - if len(history) < 3 { - return // 数据点太少，无法分析趋势 - } - - p.mutex.Lock() - defer p.mutex.Unlock() - - // 分析各个指标的趋势 - metrics := map[string][]float64{ - "cpu_usage": make([]float64, len(history)), - "memory_usage": make([]float64, len(history)), - "response_time": make([]float64, len(history)), - "cache_hit_rate": make([]float64, len(history)), - "overall_performance": make([]float64, len(history)), - } - - timestamps := make([]time.Time, len(history)) - - // 提取时间序列数据 - for i, snapshot := range history { - metrics["cpu_usage"][i] = snapshot.System.CPUUsage - metrics["memory_usage"][i] = float64(snapshot.System.MemoryUsage) / 1024 / 1024 // MB - metrics["response_time"][i] = float64(snapshot.Application.ResponseTime) / float64(time.Millisecond) - metrics["cache_hit_rate"][i] = snapshot.Cache.HitRate - metrics["overall_performance"][i] = snapshot.OverallPerformance - timestamps[i] = snapshot.Timestamp - } - - // 分析每个指标的趋势 - for metricName, values := range metrics { - trend := p.calculateTrend(metricName, values, timestamps) - p.currentTrends[metricName] = trend - } -} - -// calculateTrend 计算趋势 -func (p *PerformanceAnalyzer) calculateTrend(metricName string, values []float64, timestamps []time.Time) Trend { - if len(values) < 2 { - return Trend{ - Metric: metricName, - Direction: "stable", - Slope: 0, - Confidence: 0, - Duration: 0, - Prediction: values[len(values)-1], - } - } - - // 线性回归计算趋势 - slope, intercept, rSquared := p.linearRegression(values, timestamps) - - // 确定趋势方向 - direction := "stable" - if math.Abs(slope) > 0.01 { // 阈值 - if slope > 0 { - direction = "increasing" - } else { - direction = "decreasing" - } - } - - // 计算置信度 - confidence := math.Min(rSquared, 1.0) - - // 预测未来值 - futureTime := timestamps[len(timestamps)-1].Add(5 * time.Minute) - prediction := intercept + slope*float64(futureTime.Unix()) - - return Trend{ - Metric: metricName, - Direction: direction, - Slope: slope, - Confidence: confidence, - Duration: timestamps[len(timestamps)-1].Sub(timestamps[0]), - Prediction: prediction, - } -} - -// linearRegression 线性回归 -func (p *PerformanceAnalyzer) linearRegression(y []float64, timestamps []time.Time) (slope, intercept, rSquared float64) { - n := float64(len(y)) - if n < 2 { - return 0, y[0], 0 - } - - // 转换时间戳为数值 - x := make([]float64, len(timestamps)) - for i, t := range timestamps { - x[i] = float64(t.Unix()) - } - - // 计算均值 - var sumX, sumY float64 - for i := 0; i < len(x); i++ { - sumX += x[i] - sumY += y[i] - } - meanX := sumX / n - meanY := sumY / n - - // 计算斜率和截距 - var numerator, denominator float64 - for i := 0; i < len(x); i++ { - numerator += (x[i] - meanX) * (y[i] - meanY) - denominator += (x[i] - meanX) * (x[i] - meanX) - } - - if denominator == 0 { - return 0, meanY, 0 - } - - slope = numerator / denominator - intercept = meanY - slope*meanX - - // 计算R² - var ssRes, ssTot float64 - for i := 0; i < len(y); i++ { - predicted := intercept + slope*x[i] - ssRes += (y[i] - predicted) * (y[i] - predicted) - ssTot += (y[i] - meanY) * (y[i] - meanY) - } - - if ssTot == 0 { - rSquared = 1.0 - } else { - rSquared = 1.0 - ssRes/ssTot - } - - return slope, intercept, math.Max(0, rSquared) -} - -// DetectAnomalies 检测异常 -func (p *PerformanceAnalyzer) DetectAnomalies(currentMetrics *MetricSnapshot) { - if currentMetrics == nil { - return - } - - p.mutex.Lock() - defer p.mutex.Unlock() - - // 清空之前的异常 - p.detectedAnomalies = make([]Anomaly, 0) - - // 检测各个指标的异常 - metrics := map[string]float64{ - "cpu_usage": currentMetrics.System.CPUUsage, - "memory_usage": float64(currentMetrics.System.MemoryUsage) / 1024 / 1024, - "response_time": float64(currentMetrics.Application.ResponseTime) / float64(time.Millisecond), - "error_rate": currentMetrics.Application.ErrorRate, - "cache_hit_rate": currentMetrics.Cache.HitRate, - "overall_performance": currentMetrics.OverallPerformance, - } - - for metricName, value := range metrics { - anomaly := p.detectMetricAnomaly(metricName, value, currentMetrics.Timestamp) - if anomaly != nil { - p.detectedAnomalies = append(p.detectedAnomalies, *anomaly) - } - } -} - -// detectMetricAnomaly 检测单个指标异常 -func (p *PerformanceAnalyzer) detectMetricAnomaly(metricName string, value float64, timestamp time.Time) *Anomaly { - baseline := p.anomalyDetector.baselines[metricName] - if baseline == nil { - // 创建新基线 - p.anomalyDetector.baselines[metricName] = &Baseline{ - Mean: value, - StdDev: 0, - Min: value, - Max: value, - SampleSize: 1, - LastUpdate: timestamp, - } - return nil - } - - // 更新基线 - p.updateBaseline(baseline, value, timestamp) - - // 使用3-sigma规则检测异常 - if baseline.StdDev > 0 { - zScore := math.Abs(value-baseline.Mean) / baseline.StdDev - - var severity string - var impact float64 - - if zScore > 3.0 { - severity = "high" - impact = 0.8 - } else if zScore > 2.0 { - severity = "medium" - impact = 0.5 - } else if zScore > 1.5 { - severity = "low" - impact = 0.2 - } else { - return nil // 无异常 - } - - // 确定异常描述 - description := fmt.Sprintf("%s异常: 当前值%.2f, 期望范围[%.2f, %.2f]", - metricName, value, - baseline.Mean-2*baseline.StdDev, - baseline.Mean+2*baseline.StdDev) - - return &Anomaly{ - Metric: metricName, - Timestamp: timestamp, - Severity: severity, - Value: value, - ExpectedRange: [2]float64{baseline.Mean - 2*baseline.StdDev, baseline.Mean + 2*baseline.StdDev}, - Description: description, - Impact: impact, - } - } - - return nil -} - -// updateBaseline 更新基线 -func (p *PerformanceAnalyzer) updateBaseline(baseline *Baseline, newValue float64, timestamp time.Time) { - // 增量更新均值和标准差 - oldMean := baseline.Mean - baseline.SampleSize++ - baseline.Mean += (newValue - baseline.Mean) / float64(baseline.SampleSize) - - // 更新方差（Welford算法） - if baseline.SampleSize > 1 { - variance := (float64(baseline.SampleSize-2)*baseline.StdDev*baseline.StdDev + - (newValue-oldMean)*(newValue-baseline.Mean)) / float64(baseline.SampleSize-1) - baseline.StdDev = math.Sqrt(math.Max(0, variance)) - } - - // 更新最值 - if newValue < baseline.Min { - baseline.Min = newValue - } - if newValue > baseline.Max { - baseline.Max = newValue - } - - baseline.LastUpdate = timestamp -} - -// AnalyzeCorrelations 分析相关性 -func (p *PerformanceAnalyzer) AnalyzeCorrelations(history []MetricSnapshot) { - if len(history) < 3 { - return - } - - p.mutex.Lock() - defer p.mutex.Unlock() - - // 提取指标数据 - metrics := map[string][]float64{ - "cpu_usage": make([]float64, len(history)), - "memory_usage": make([]float64, len(history)), - "response_time": make([]float64, len(history)), - "cache_hit_rate": make([]float64, len(history)), - "overall_performance": make([]float64, len(history)), - } - - for i, snapshot := range history { - metrics["cpu_usage"][i] = snapshot.System.CPUUsage - metrics["memory_usage"][i] = float64(snapshot.System.MemoryUsage) / 1024 / 1024 - metrics["response_time"][i] = float64(snapshot.Application.ResponseTime) / float64(time.Millisecond) - metrics["cache_hit_rate"][i] = snapshot.Cache.HitRate - metrics["overall_performance"][i] = snapshot.OverallPerformance - } - - // 计算相关性矩阵 - metricNames := make([]string, 0, len(metrics)) - for name := range metrics { - metricNames = append(metricNames, name) - } - sort.Strings(metricNames) - - for i, metric1 := range metricNames { - if p.correlationAnalyzer.correlationMatrix[metric1] == nil { - p.correlationAnalyzer.correlationMatrix[metric1] = make(map[string]float64) - } - - for j, metric2 := range metricNames { - if i <= j { - correlation := p.calculateCorrelation(metrics[metric1], metrics[metric2]) - p.correlationAnalyzer.correlationMatrix[metric1][metric2] = correlation - p.correlationAnalyzer.correlationMatrix[metric2][metric1] = correlation - - // 保存重要相关性 - if math.Abs(correlation) > 0.5 && metric1 != metric2 { - p.correlations[fmt.Sprintf("%s_%s", metric1, metric2)] = correlation - } - } - } - } -} - -// calculateCorrelation 计算皮尔逊相关系数 -func (p *PerformanceAnalyzer) calculateCorrelation(x, y []float64) float64 { - if len(x) != len(y) || len(x) < 2 { - return 0 - } - - n := float64(len(x)) - - // 计算均值 - var sumX, sumY float64 - for i := 0; i < len(x); i++ { - sumX += x[i] - sumY += y[i] - } - meanX := sumX / n - meanY := sumY / n - - // 计算协方差和方差 - var covariance, varianceX, varianceY float64 - for i := 0; i < len(x); i++ { - dx := x[i] - meanX - dy := y[i] - meanY - covariance += dx * dy - varianceX += dx * dx - varianceY += dy * dy - } - - // 计算相关系数 - if varianceX == 0 || varianceY == 0 { - return 0 - } - - correlation := covariance / math.Sqrt(varianceX*varianceY) - return correlation -} - -// AnalyzeIssues 分析性能问题 -func (p *PerformanceAnalyzer) AnalyzeIssues(currentMetrics *MetricSnapshot) []string { - if currentMetrics == nil { - return nil - } - - issues := make([]string, 0) - - // CPU使用率过高 - if currentMetrics.System.CPUUsage > 0.8 { - issues = append(issues, "high_cpu_usage") - } - - // 内存使用率过高 - memoryUsageRatio := float64(currentMetrics.System.MemoryUsage) / float64(currentMetrics.System.MemoryTotal) - if memoryUsageRatio > 0.85 { - issues = append(issues, "high_memory_usage") - } - - // 响应时间过长 - if currentMetrics.Application.ResponseTime > 1*time.Second { - issues = append(issues, "high_response_time") - } - - // 错误率过高 - if currentMetrics.Application.ErrorRate > 0.05 { - issues = append(issues, "high_error_rate") - } - - // 缓存命中率过低 - if currentMetrics.Cache.HitRate < 0.7 { - issues = append(issues, "low_cache_hit_rate") - } - - // 整体性能过低 - if currentMetrics.OverallPerformance < 60 { - issues = append(issues, "low_overall_performance") - } - - return issues -} - -// GetCurrentTrends 获取当前趋势 -func (p *PerformanceAnalyzer) GetCurrentTrends() map[string]Trend { - p.mutex.RLock() - defer p.mutex.RUnlock() - - trends := make(map[string]Trend) - for k, v := range p.currentTrends { - trends[k] = v - } - - return trends -} - -// GetDetectedAnomalies 获取检测到的异常 -func (p *PerformanceAnalyzer) GetDetectedAnomalies() []Anomaly { - p.mutex.RLock() - defer p.mutex.RUnlock() - - anomalies := make([]Anomaly, len(p.detectedAnomalies)) - copy(anomalies, p.detectedAnomalies) - - return anomalies -} - -// GetCorrelations 获取相关性 -func (p *PerformanceAnalyzer) GetCorrelations() map[string]float64 { - p.mutex.RLock() - defer p.mutex.RUnlock() - - correlations := make(map[string]float64) - for k, v := range p.correlations { - correlations[k] = v - } - - return correlations -} \ No newline at end of file diff --git a/util/cache/predictive_model.go b/util/cache/predictive_model.go deleted file mode 100644 index c983224..0000000 --- a/util/cache/predictive_model.go +++ /dev/null @@ -1,396 +0,0 @@ -package cache - -import ( - "fmt" - "math" - "time" -) - -// NewPredictiveModel 创建预测模型 -func NewPredictiveModel() *PredictiveModel { - return &PredictiveModel{ - modelType: "linear_regression", - coefficients: make([]float64, 0), - seasonalFactors: make([]float64, 0), - trainingData: make([]DataPoint, 0), - testData: make([]DataPoint, 0), - predictions: make(map[string]Prediction), - } -} - -// Train 训练模型 -func (p *PredictiveModel) Train(dataset *LearningDataset) error { - p.mutex.Lock() - defer p.mutex.Unlock() - - if dataset == nil || len(dataset.Features) == 0 { - return fmt.Errorf("训练数据集为空") - } - - // 准备训练数据 - p.prepareTrainingData(dataset) - - // 根据模型类型进行训练 - switch p.modelType { - case "linear_regression": - return p.trainLinearRegression(dataset) - case "exponential_smoothing": - return p.trainExponentialSmoothing(dataset) - default: - return fmt.Errorf("不支持的模型类型: %s", p.modelType) - } -} - -// prepareTrainingData 准备训练数据 -func (p *PredictiveModel) prepareTrainingData(dataset *LearningDataset) { - dataset.mutex.RLock() - defer dataset.mutex.RUnlock() - - totalSamples := len(dataset.Features) - if totalSamples == 0 { - return - } - - // 数据分割 - trainSize := int(float64(totalSamples) * 0.8) // 80%用于训练 - - p.trainingData = make([]DataPoint, trainSize) - p.testData = make([]DataPoint, totalSamples-trainSize) - - // 填充训练数据 - for i := 0; i < trainSize; i++ { - p.trainingData[i] = DataPoint{ - Timestamp: time.Now().Add(-time.Duration(totalSamples-i) * time.Minute), - Values: make(map[string]float64), - } - - // 转换特征为命名值 - if len(dataset.Features[i]) >= 5 { - p.trainingData[i].Values["cpu_usage"] = dataset.Features[i][0] - p.trainingData[i].Values["memory_usage"] = dataset.Features[i][1] - p.trainingData[i].Values["response_time"] = dataset.Features[i][2] - p.trainingData[i].Values["cache_hit_rate"] = dataset.Features[i][3] - p.trainingData[i].Values["compression_ratio"] = dataset.Features[i][4] - } - } - - // 填充测试数据 - for i := 0; i < len(p.testData); i++ { - testIndex := trainSize + i - p.testData[i] = DataPoint{ - Timestamp: time.Now().Add(-time.Duration(totalSamples-testIndex) * time.Minute), - Values: make(map[string]float64), - } - - if len(dataset.Features[testIndex]) >= 5 { - p.testData[i].Values["cpu_usage"] = dataset.Features[testIndex][0] - p.testData[i].Values["memory_usage"] = dataset.Features[testIndex][1] - p.testData[i].Values["response_time"] = dataset.Features[testIndex][2] - p.testData[i].Values["cache_hit_rate"] = dataset.Features[testIndex][3] - p.testData[i].Values["compression_ratio"] = dataset.Features[testIndex][4] - } - } -} - -// trainLinearRegression 训练线性回归模型 -func (p *PredictiveModel) trainLinearRegression(dataset *LearningDataset) error { - dataset.mutex.RLock() - defer dataset.mutex.RUnlock() - - if len(dataset.Features) == 0 || len(dataset.Labels) != len(dataset.Features) { - return fmt.Errorf("训练数据不匹配") - } - - featuresCount := len(dataset.Features[0]) - samplesCount := len(dataset.Features) - - // 初始化系数（包括偏置项） - p.coefficients = make([]float64, featuresCount+1) - - // 使用梯度下降训练 - learningRate := 0.01 - iterations := 1000 - - for iter := 0; iter < iterations; iter++ { - // 计算预测值和误差 - totalLoss := 0.0 - gradients := make([]float64, len(p.coefficients)) - - for i := 0; i < samplesCount; i++ { - // 计算预测值 - predicted := p.coefficients[0] // 偏置项 - for j := 0; j < featuresCount; j++ { - predicted += p.coefficients[j+1] * dataset.Features[i][j] - } - - // 计算误差 - error := predicted - dataset.Labels[i] - totalLoss += error * error - - // 计算梯度 - gradients[0] += error // 偏置项梯度 - for j := 0; j < featuresCount; j++ { - gradients[j+1] += error * dataset.Features[i][j] - } - } - - // 更新参数 - for j := 0; j < len(p.coefficients); j++ { - p.coefficients[j] -= learningRate * gradients[j] / float64(samplesCount) - } - - // 计算平均损失 - avgLoss := totalLoss / float64(samplesCount) - - // 早停条件 - if avgLoss < 0.001 { - break - } - } - - return nil -} - -// trainExponentialSmoothing 训练指数平滑模型 -func (p *PredictiveModel) trainExponentialSmoothing(dataset *LearningDataset) error { - dataset.mutex.RLock() - defer dataset.mutex.RUnlock() - - if len(dataset.Labels) < 2 { - return fmt.Errorf("指数平滑需要至少2个数据点") - } - - // 简单指数平滑参数 - alpha := 0.3 // 平滑参数 - - // 初始化 - p.coefficients = make([]float64, 2) - p.coefficients[0] = dataset.Labels[0] // 初始水平 - p.coefficients[1] = alpha // 平滑参数 - - // 计算趋势组件 - if len(dataset.Labels) > 1 { - p.trendComponent = dataset.Labels[1] - dataset.Labels[0] - } - - return nil -} - -// Predict 进行预测 -func (p *PredictiveModel) Predict(features []float64, horizon time.Duration) (*Prediction, error) { - p.mutex.RLock() - defer p.mutex.RUnlock() - - if len(p.coefficients) == 0 { - return nil, fmt.Errorf("模型尚未训练") - } - - var predictedValue float64 - var confidence float64 - - switch p.modelType { - case "linear_regression": - if len(features) != len(p.coefficients)-1 { - return nil, fmt.Errorf("特征维度不匹配") - } - - // 线性回归预测 - predictedValue = p.coefficients[0] // 偏置项 - for i, feature := range features { - predictedValue += p.coefficients[i+1] * feature - } - - // 置信度基于训练数据的拟合程度 - confidence = math.Max(0.5, p.rmse) // 简化的置信度计算 - - case "exponential_smoothing": - // 指数平滑预测 - predictedValue = p.coefficients[0] + p.trendComponent*float64(horizon/time.Minute) - confidence = 0.7 // 固定置信度 - - default: - return nil, fmt.Errorf("不支持的模型类型: %s", p.modelType) - } - - return &Prediction{ - Metric: "overall_performance", - FutureValue: predictedValue, - Confidence: confidence, - TimeHorizon: horizon, - PredictedAt: time.Now(), - }, nil -} - -// Validate 验证模型性能 -func (p *PredictiveModel) Validate() float64 { - p.mutex.Lock() - defer p.mutex.Unlock() - - if len(p.testData) == 0 { - return 0 - } - - correctPredictions := 0 - totalPredictions := len(p.testData) - - for _, testPoint := range p.testData { - // 提取特征 - features := []float64{ - testPoint.Values["cpu_usage"], - testPoint.Values["memory_usage"], - testPoint.Values["response_time"], - testPoint.Values["cache_hit_rate"], - testPoint.Values["compression_ratio"], - } - - // 进行预测 - prediction, err := p.Predict(features, 5*time.Minute) - if err != nil { - continue - } - - // 计算实际性能分数（简化） - actualPerformance := p.calculatePerformanceScore(testPoint.Values) - - // 判断预测是否准确（容忍误差10%） - errorRate := math.Abs(prediction.FutureValue-actualPerformance) / actualPerformance - if errorRate < 0.1 { - correctPredictions++ - } - } - - accuracy := float64(correctPredictions) / float64(totalPredictions) - p.accuracy = accuracy - - // 计算RMSE - p.calculateRMSE() - - return accuracy -} - -// calculatePerformanceScore 计算性能分数 -func (p *PredictiveModel) calculatePerformanceScore(values map[string]float64) float64 { - // 简化的性能分数计算 - cpuScore := (1.0 - values["cpu_usage"]) * 30 - memoryScore := (1.0 - values["memory_usage"]/1000) * 25 // 假设内存以MB为单位 - responseScore := (1.0 - values["response_time"]/1000) * 25 // 假设响应时间以ms为单位 - cacheScore := values["cache_hit_rate"] * 20 - - return math.Max(0, cpuScore+memoryScore+responseScore+cacheScore) -} - -// calculateRMSE 计算均方根误差 -func (p *PredictiveModel) calculateRMSE() { - if len(p.testData) == 0 { - return - } - - var sumSquaredErrors float64 - validPredictions := 0 - - for _, testPoint := range p.testData { - features := []float64{ - testPoint.Values["cpu_usage"], - testPoint.Values["memory_usage"], - testPoint.Values["response_time"], - testPoint.Values["cache_hit_rate"], - testPoint.Values["compression_ratio"], - } - - prediction, err := p.Predict(features, 5*time.Minute) - if err != nil { - continue - } - - actualPerformance := p.calculatePerformanceScore(testPoint.Values) - error := prediction.FutureValue - actualPerformance - sumSquaredErrors += error * error - validPredictions++ - } - - if validPredictions > 0 { - p.rmse = math.Sqrt(sumSquaredErrors / float64(validPredictions)) - } -} - -// PredictMultiple 预测多个指标 -func (p *PredictiveModel) PredictMultiple(currentMetrics *MetricSnapshot, horizons []time.Duration) map[string]Prediction { - if currentMetrics == nil { - return nil - } - - features := []float64{ - currentMetrics.System.CPUUsage, - float64(currentMetrics.System.MemoryUsage) / 1024 / 1024, - float64(currentMetrics.Application.ResponseTime) / float64(time.Millisecond), - currentMetrics.Cache.HitRate, - currentMetrics.Cache.CompressionRatio, - } - - predictions := make(map[string]Prediction) - - for _, horizon := range horizons { - predictionKey := fmt.Sprintf("performance_%s", horizon.String()) - - prediction, err := p.Predict(features, horizon) - if err == nil { - predictions[predictionKey] = *prediction - } - } - - return predictions -} - -// GetAccuracy 获取模型准确率 -func (p *PredictiveModel) GetAccuracy() float64 { - p.mutex.RLock() - defer p.mutex.RUnlock() - - return p.accuracy -} - -// GetPredictions 获取所有预测结果 -func (p *PredictiveModel) GetPredictions() map[string]Prediction { - p.mutex.RLock() - defer p.mutex.RUnlock() - - predictions := make(map[string]Prediction) - for k, v := range p.predictions { - predictions[k] = v - } - - return predictions -} - -// UpdatePredictions 更新预测结果 -func (p *PredictiveModel) UpdatePredictions(currentMetrics *MetricSnapshot) { - if currentMetrics == nil { - return - } - - // 预测未来1小时的性能 - horizons := []time.Duration{ - 5 * time.Minute, - 15 * time.Minute, - 30 * time.Minute, - 1 * time.Hour, - } - - newPredictions := p.PredictMultiple(currentMetrics, horizons) - - p.mutex.Lock() - defer p.mutex.Unlock() - - // 更新预测结果 - for k, v := range newPredictions { - p.predictions[k] = v - } - - // 清理过期预测 - now := time.Now() - for k, v := range p.predictions { - if now.Sub(v.PredictedAt) > 2*time.Hour { - delete(p.predictions, k) - } - } -} \ No newline at end of file diff --git a/util/cache/search_pattern_analyzer.go b/util/cache/search_pattern_analyzer.go deleted file mode 100644 index e0362bd..0000000 --- a/util/cache/search_pattern_analyzer.go +++ /dev/null @@ -1,431 +0,0 @@ -package cache - -import ( - "crypto/md5" - "fmt" - "regexp" - "sort" - "strings" - "sync" - "time" -) - -// SearchPatternAnalyzer 搜索模式分析器 -type SearchPatternAnalyzer struct { - // 模式缓存 - patternCache map[string]*SearchPattern - cacheMutex sync.RWMutex - - // 分析规则 - keywordRules []*KeywordRule - - // 统计信息 - analysisCount int64 - cacheHitCount int64 - - // 配置 - maxCacheSize int - cacheExpiry time.Duration -} - -// KeywordRule 关键词规则 -type KeywordRule struct { - Name string - Pattern *regexp.Regexp - Priority int - Description string -} - -// NewSearchPatternAnalyzer 创建搜索模式分析器 -func NewSearchPatternAnalyzer() *SearchPatternAnalyzer { - analyzer := &SearchPatternAnalyzer{ - patternCache: make(map[string]*SearchPattern), - maxCacheSize: 1000, // 最大缓存1000个模式 - cacheExpiry: 1 * time.Hour, // 1小时过期 - } - - // 初始化关键词规则 - analyzer.initializeKeywordRules() - - return analyzer -} - -// initializeKeywordRules 初始化关键词规则 -func (s *SearchPatternAnalyzer) initializeKeywordRules() { - s.keywordRules = []*KeywordRule{ - { - Name: "电影资源", - Pattern: regexp.MustCompile(`(?i)(电影|movie|film|影片|HD|4K|蓝光|BluRay)`), - Priority: 1, - Description: "电影相关搜索", - }, - { - Name: "电视剧资源", - Pattern: regexp.MustCompile(`(?i)(电视剧|TV|series|连续剧|美剧|韩剧|日剧)`), - Priority: 1, - Description: "电视剧相关搜索", - }, - { - Name: "动漫资源", - Pattern: regexp.MustCompile(`(?i)(动漫|anime|动画|漫画|manga)`), - Priority: 1, - Description: "动漫相关搜索", - }, - { - Name: "音乐资源", - Pattern: regexp.MustCompile(`(?i)(音乐|music|歌曲|专辑|album|MP3|FLAC)`), - Priority: 2, - Description: "音乐相关搜索", - }, - { - Name: "游戏资源", - Pattern: regexp.MustCompile(`(?i)(游戏|game|单机|网游|手游|steam)`), - Priority: 2, - Description: "游戏相关搜索", - }, - { - Name: "软件资源", - Pattern: regexp.MustCompile(`(?i)(软件|software|app|应用|工具|破解)`), - Priority: 2, - Description: "软件相关搜索", - }, - { - Name: "学习资源", - Pattern: regexp.MustCompile(`(?i)(教程|tutorial|课程|学习|教学|资料)`), - Priority: 3, - Description: "学习资源搜索", - }, - { - Name: "文档资源", - Pattern: regexp.MustCompile(`(?i)(文档|doc|pdf|txt|电子书|ebook)`), - Priority: 3, - Description: "文档资源搜索", - }, - { - Name: "通用搜索", - Pattern: regexp.MustCompile(`.*`), // 匹配所有 - Priority: 4, - Description: "通用搜索模式", - }, - } -} - -// AnalyzePattern 分析搜索模式 -func (s *SearchPatternAnalyzer) AnalyzePattern(op *CacheOperation) *SearchPattern { - s.analysisCount++ - - // 🔧 生成缓存键 - cacheKey := s.generateCacheKey(op) - - // 🚀 检查缓存 - s.cacheMutex.RLock() - if cached, exists := s.patternCache[cacheKey]; exists { - // 检查是否过期 - if time.Since(cached.LastAccessTime) < s.cacheExpiry { - cached.LastAccessTime = time.Now() - cached.Frequency++ - s.cacheMutex.RUnlock() - s.cacheHitCount++ - return cached - } - } - s.cacheMutex.RUnlock() - - // 🎯 分析新模式 - pattern := s.analyzeNewPattern(op) - - // 🗄️ 缓存结果 - s.cachePattern(cacheKey, pattern) - - return pattern -} - -// generateCacheKey 生成缓存键 -func (s *SearchPatternAnalyzer) generateCacheKey(op *CacheOperation) string { - // 使用关键词和插件名生成缓存键 - source := fmt.Sprintf("%s_%s", - s.normalizeKeyword(op.Keyword), - op.PluginName) - - // MD5哈希以节省内存 - hash := md5.Sum([]byte(source)) - return fmt.Sprintf("%x", hash) -} - -// normalizeKeyword 标准化关键词 -func (s *SearchPatternAnalyzer) normalizeKeyword(keyword string) string { - // 转换为小写 - normalized := strings.ToLower(keyword) - - // 移除特殊字符和多余空格 - normalized = regexp.MustCompile(`[^\w\s\u4e00-\u9fff]`).ReplaceAllString(normalized, " ") - normalized = regexp.MustCompile(`\s+`).ReplaceAllString(normalized, " ") - normalized = strings.TrimSpace(normalized) - - return normalized -} - -// analyzeNewPattern 分析新模式 -func (s *SearchPatternAnalyzer) analyzeNewPattern(op *CacheOperation) *SearchPattern { - pattern := &SearchPattern{ - KeywordPattern: s.classifyKeyword(op.Keyword), - PluginSet: []string{op.PluginName}, - TimeWindow: s.determineTimeWindow(op), - Frequency: 1, - LastAccessTime: time.Now(), - Metadata: make(map[string]interface{}), - } - - // 🔍 关键词分析 - s.analyzeKeywordCharacteristics(pattern, op.Keyword) - - // 🔍 插件分析 - s.analyzePluginCharacteristics(pattern, op.PluginName) - - // 🔍 时间模式分析 - s.analyzeTimePattern(pattern, op.Timestamp) - - return pattern -} - -// classifyKeyword 分类关键词 -func (s *SearchPatternAnalyzer) classifyKeyword(keyword string) string { - // 按优先级检查规则 - for _, rule := range s.keywordRules { - if rule.Pattern.MatchString(keyword) { - return rule.Name - } - } - - return "通用搜索" -} - -// analyzeKeywordCharacteristics 分析关键词特征 -func (s *SearchPatternAnalyzer) analyzeKeywordCharacteristics(pattern *SearchPattern, keyword string) { - metadata := pattern.Metadata - - // 分析关键词长度 - metadata["keyword_length"] = len(keyword) - - // 分析关键词复杂度（包含的词数） - words := strings.Fields(keyword) - metadata["word_count"] = len(words) - - // 分析是否包含特殊字符 - hasSpecialChars := regexp.MustCompile(`[^\w\s\u4e00-\u9fff]`).MatchString(keyword) - metadata["has_special_chars"] = hasSpecialChars - - // 分析是否包含数字 - hasNumbers := regexp.MustCompile(`\d`).MatchString(keyword) - metadata["has_numbers"] = hasNumbers - - // 分析语言类型 - hasChinese := regexp.MustCompile(`[\u4e00-\u9fff]`).MatchString(keyword) - hasEnglish := regexp.MustCompile(`[a-zA-Z]`).MatchString(keyword) - - if hasChinese && hasEnglish { - metadata["language"] = "mixed" - } else if hasChinese { - metadata["language"] = "chinese" - } else if hasEnglish { - metadata["language"] = "english" - } else { - metadata["language"] = "other" - } - - // 预测搜索频率（基于关键词特征） - complexity := len(words) - if hasSpecialChars { - complexity++ - } - if hasNumbers { - complexity++ - } - - // 复杂度越低，搜索频率可能越高 - predictedFrequency := "medium" - if complexity <= 2 { - predictedFrequency = "high" - } else if complexity >= 5 { - predictedFrequency = "low" - } - - metadata["predicted_frequency"] = predictedFrequency -} - -// analyzePluginCharacteristics 分析插件特征 -func (s *SearchPatternAnalyzer) analyzePluginCharacteristics(pattern *SearchPattern, pluginName string) { - metadata := pattern.Metadata - - // 插件类型分析（基于名称推断） - pluginType := "general" - if strings.Contains(strings.ToLower(pluginName), "4k") { - pluginType = "high_quality" - } else if strings.Contains(strings.ToLower(pluginName), "pan") { - pluginType = "cloud_storage" - } else if strings.Contains(strings.ToLower(pluginName), "search") { - pluginType = "search_engine" - } - - metadata["plugin_type"] = pluginType - metadata["plugin_name"] = pluginName -} - -// analyzeTimePattern 分析时间模式 -func (s *SearchPatternAnalyzer) analyzeTimePattern(pattern *SearchPattern, timestamp time.Time) { - metadata := pattern.Metadata - - // 时间段分析 - hour := timestamp.Hour() - var timePeriod string - switch { - case hour >= 6 && hour < 12: - timePeriod = "morning" - case hour >= 12 && hour < 18: - timePeriod = "afternoon" - case hour >= 18 && hour < 22: - timePeriod = "evening" - default: - timePeriod = "night" - } - - metadata["time_period"] = timePeriod - - // 工作日/周末分析 - weekday := timestamp.Weekday() - isWeekend := weekday == time.Saturday || weekday == time.Sunday - metadata["is_weekend"] = isWeekend - - // 预测最佳缓存时间（基于时间模式） - if isWeekend || timePeriod == "evening" { - pattern.TimeWindow = 30 * time.Minute // 高峰期，较长缓存 - } else { - pattern.TimeWindow = 15 * time.Minute // 非高峰期，较短缓存 - } -} - -// determineTimeWindow 确定时间窗口 -func (s *SearchPatternAnalyzer) determineTimeWindow(op *CacheOperation) time.Duration { - // 基本时间窗口：15分钟 - baseWindow := 15 * time.Minute - - // 根据优先级调整 - switch op.Priority { - case 1: // 高优先级插件 - return baseWindow * 2 // 30分钟 - case 2: // 中高优先级插件 - return baseWindow * 3 / 2 // 22.5分钟 - case 3: // 中等优先级插件 - return baseWindow // 15分钟 - case 4: // 低优先级插件 - return baseWindow / 2 // 7.5分钟 - default: - return baseWindow - } -} - -// cachePattern 缓存模式 -func (s *SearchPatternAnalyzer) cachePattern(cacheKey string, pattern *SearchPattern) { - s.cacheMutex.Lock() - defer s.cacheMutex.Unlock() - - // 检查缓存大小，必要时清理 - if len(s.patternCache) >= s.maxCacheSize { - s.cleanupCache() - } - - s.patternCache[cacheKey] = pattern -} - -// cleanupCache 清理缓存 -func (s *SearchPatternAnalyzer) cleanupCache() { - now := time.Now() - - // 收集需要删除的键 - toDelete := make([]string, 0) - for key, pattern := range s.patternCache { - if now.Sub(pattern.LastAccessTime) > s.cacheExpiry { - toDelete = append(toDelete, key) - } - } - - // 如果过期删除不够，按使用频率删除 - if len(toDelete) < len(s.patternCache)/4 { // 删除不到25% - // 按频率排序，删除使用频率最低的 - type patternFreq struct { - key string - frequency int - lastAccess time.Time - } - - patterns := make([]patternFreq, 0, len(s.patternCache)) - for key, pattern := range s.patternCache { - patterns = append(patterns, patternFreq{ - key: key, - frequency: pattern.Frequency, - lastAccess: pattern.LastAccessTime, - }) - } - - // 按频率排序（频率低的在前） - sort.Slice(patterns, func(i, j int) bool { - if patterns[i].frequency == patterns[j].frequency { - return patterns[i].lastAccess.Before(patterns[j].lastAccess) - } - return patterns[i].frequency < patterns[j].frequency - }) - - // 删除前25% - deleteCount := len(patterns) / 4 - for i := 0; i < deleteCount; i++ { - toDelete = append(toDelete, patterns[i].key) - } - } - - // 执行删除 - for _, key := range toDelete { - delete(s.patternCache, key) - } -} - -// GetCacheStats 获取缓存统计 -func (s *SearchPatternAnalyzer) GetCacheStats() map[string]interface{} { - s.cacheMutex.RLock() - defer s.cacheMutex.RUnlock() - - hitRate := float64(0) - if s.analysisCount > 0 { - hitRate = float64(s.cacheHitCount) / float64(s.analysisCount) - } - - return map[string]interface{}{ - "cache_size": len(s.patternCache), - "max_cache_size": s.maxCacheSize, - "analysis_count": s.analysisCount, - "cache_hit_count": s.cacheHitCount, - "hit_rate": hitRate, - "cache_expiry": s.cacheExpiry, - } -} - -// GetPopularPatterns 获取热门模式 -func (s *SearchPatternAnalyzer) GetPopularPatterns(limit int) []*SearchPattern { - s.cacheMutex.RLock() - defer s.cacheMutex.RUnlock() - - patterns := make([]*SearchPattern, 0, len(s.patternCache)) - for _, pattern := range s.patternCache { - patterns = append(patterns, pattern) - } - - // 按频率排序 - sort.Slice(patterns, func(i, j int) bool { - return patterns[i].Frequency > patterns[j].Frequency - }) - - if limit > 0 && limit < len(patterns) { - patterns = patterns[:limit] - } - - return patterns -} \ No newline at end of file diff --git a/util/cache/sharded_memory_cache.go b/util/cache/sharded_memory_cache.go index 44172ad..4c577fc 100644 --- a/util/cache/sharded_memory_cache.go +++ b/util/cache/sharded_memory_cache.go @@ -8,7 +8,7 @@ import ( "time" ) -// 🔥 全局清理任务相关变量（单例模式） +// 全局清理任务相关变量（单例模式） var ( globalCleanupTicker *time.Ticker globalCleanupOnce sync.Once @@ -16,7 +16,7 @@ var ( cacheRegistryMutex sync.RWMutex ) -// 🔥 清理目标接口 +// 清理目标接口 type cleanupTarget interface { CleanExpired() } @@ -45,8 +45,8 @@ type ShardedMemoryCache struct { maxSize int64 itemsPerShard int sizePerShard int64 - diskCache *ShardedDiskCache // 🔥 新增：磁盘缓存引用 - diskCacheMutex sync.RWMutex // 🔥 新增：磁盘缓存引用的保护锁 + diskCache *ShardedDiskCache // 磁盘缓存引用 + diskCacheMutex sync.RWMutex // 磁盘缓存引用的保护锁 } // 创建新的分片内存缓存 @@ -237,7 +237,7 @@ func (c *ShardedMemoryCache) evictFromShard(shard *memoryCacheShard) { go func(key string, data []byte, expiry time.Time) { ttl := time.Until(expiry) if ttl > 0 { - diskCache.Set(key, data, ttl) // 🔥 保持相同TTL + diskCache.Set(key, data, ttl) // 保持相同TTL } }(oldestKey, oldestItem.data, oldestItem.expiry) } @@ -302,7 +302,7 @@ func (c *ShardedMemoryCache) Clear() { wg.Wait() } -// 🔥 启动全局清理任务（单例模式） +// 启动全局清理任务（单例模式） func startGlobalCleanupTask() { globalCleanupOnce.Do(func() { globalCleanupTicker = time.NewTicker(5 * time.Minute) @@ -322,7 +322,7 @@ func startGlobalCleanupTask() { }) } -// 🔥 注册缓存到全局清理任务 +// 注册缓存到全局清理任务 func registerForCleanup(cache cleanupTarget) { cacheRegistryMutex.Lock() defer cacheRegistryMutex.Unlock() diff --git a/util/cache/tuning_strategy.go b/util/cache/tuning_strategy.go deleted file mode 100644 index 7bea8c2..0000000 --- a/util/cache/tuning_strategy.go +++ /dev/null @@ -1,463 +0,0 @@ -package cache - -import ( - "sort" - "time" -) - -// NewTuningStrategy 创建调优策略 -func NewTuningStrategy() *TuningStrategy { - strategy := &TuningStrategy{ - strategyType: "balanced", - rules: make([]*TuningRule, 0), - parameterAdjustments: make(map[string]ParameterAdjustment), - executionHistory: make([]*StrategyExecution, 0), - } - - // 初始化调优规则 - strategy.initializeRules() - - return strategy -} - -// NewLearningDataset 创建学习数据集 -func NewLearningDataset() *LearningDataset { - return &LearningDataset{ - Features: make([][]float64, 0), - Labels: make([]float64, 0), - Weights: make([]float64, 0), - FeatureStats: make([]FeatureStatistics, 0), - TrainingSplit: 0.8, - ValidationSplit: 0.1, - TestSplit: 0.1, - } -} - -// initializeRules 初始化调优规则 -func (t *TuningStrategy) initializeRules() { - t.rules = []*TuningRule{ - { - Name: "高CPU使用率调优", - Priority: 1, - Enabled: true, - Condition: func(metrics *MetricSnapshot) bool { - return metrics.System.CPUUsage > 0.8 - }, - Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - return t.createCPUOptimizationDecision(engine) - }, - }, - { - Name: "高内存使用调优", - Priority: 2, - Enabled: true, - Condition: func(metrics *MetricSnapshot) bool { - memoryRatio := float64(metrics.System.MemoryUsage) / float64(metrics.System.MemoryTotal) - return memoryRatio > 0.85 - }, - Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - return t.createMemoryOptimizationDecision(engine) - }, - }, - { - Name: "响应时间过长调优", - Priority: 3, - Enabled: true, - Condition: func(metrics *MetricSnapshot) bool { - return metrics.Application.ResponseTime > 500*time.Millisecond - }, - Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - return t.createResponseTimeOptimizationDecision(engine) - }, - }, - { - Name: "缓存命中率低调优", - Priority: 4, - Enabled: true, - Condition: func(metrics *MetricSnapshot) bool { - return metrics.Cache.HitRate < 0.7 - }, - Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - return t.createCacheOptimizationDecision(engine) - }, - }, - { - Name: "整体性能低调优", - Priority: 5, - Enabled: true, - Condition: func(metrics *MetricSnapshot) bool { - return metrics.OverallPerformance < 60 - }, - Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - return t.createOverallPerformanceDecision(engine) - }, - }, - { - Name: "预防性调优", - Priority: 10, - Enabled: true, - Condition: func(metrics *MetricSnapshot) bool { - // 基于趋势的预防性调优 - return false // 暂时禁用，需要趋势数据 - }, - Action: func(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - return t.createPreventiveDecision(engine) - }, - }, - } -} - -// GenerateDecision 生成调优决策 -func (t *TuningStrategy) GenerateDecision(metrics *MetricSnapshot, issues []string) *TuningDecision { - if metrics == nil { - return nil - } - - t.mutex.Lock() - defer t.mutex.Unlock() - - // 按优先级排序规则 - sort.Slice(t.rules, func(i, j int) bool { - return t.rules[i].Priority < t.rules[j].Priority - }) - - // 检查规则并生成决策 - for _, rule := range t.rules { - if !rule.Enabled { - continue - } - - // 检查冷却时间（防止频繁调优） - if time.Since(rule.LastTriggered) < 5*time.Minute { - continue - } - - // 检查条件 - if rule.Condition(metrics) { - decision, err := rule.Action(nil) // 简化实现，不传递engine - if err != nil { - continue - } - - // 更新规则状态 - rule.LastTriggered = time.Now() - rule.TriggerCount++ - - // 设置决策基本信息 - decision.Timestamp = time.Now() - decision.Trigger = rule.Name - - return decision - } - } - - return nil -} - -// createCPUOptimizationDecision 创建CPU优化决策 -func (t *TuningStrategy) createCPUOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - adjustments := []ParameterAdjustment{ - { - ParameterName: "batch_interval", - CurrentValue: "60s", - ProposedValue: "90s", - AdjustmentRatio: 0.5, // 增加50% - Reason: "减少CPU负载", - ExpectedImpact: "降低CPU使用率", - Risk: "medium", - }, - { - ParameterName: "batch_size", - CurrentValue: 100, - ProposedValue: 150, - AdjustmentRatio: 0.5, - Reason: "减少处理频率", - ExpectedImpact: "降低CPU负载", - Risk: "low", - }, - } - - return &TuningDecision{ - Adjustments: adjustments, - Confidence: 0.8, - ExpectedImprovement: 0.15, - Risk: 0.3, - AutoExecute: true, - }, nil -} - -// createMemoryOptimizationDecision 创建内存优化决策 -func (t *TuningStrategy) createMemoryOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - adjustments := []ParameterAdjustment{ - { - ParameterName: "max_buffer_size", - CurrentValue: 1000, - ProposedValue: 700, - AdjustmentRatio: -0.3, // 减少30% - Reason: "减少内存占用", - ExpectedImpact: "降低内存使用率", - Risk: "medium", - }, - { - ParameterName: "cache_cleanup_frequency", - CurrentValue: "5m", - ProposedValue: "3m", - AdjustmentRatio: -0.4, - Reason: "更频繁清理缓存", - ExpectedImpact: "释放内存空间", - Risk: "low", - }, - } - - return &TuningDecision{ - Adjustments: adjustments, - Confidence: 0.85, - ExpectedImprovement: 0.2, - Risk: 0.25, - AutoExecute: true, - }, nil -} - -// createResponseTimeOptimizationDecision 创建响应时间优化决策 -func (t *TuningStrategy) createResponseTimeOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - adjustments := []ParameterAdjustment{ - { - ParameterName: "batch_interval", - CurrentValue: "60s", - ProposedValue: "30s", - AdjustmentRatio: -0.5, // 减少50% - Reason: "更快的数据写入", - ExpectedImpact: "降低响应时间", - Risk: "medium", - }, - { - ParameterName: "concurrent_workers", - CurrentValue: 4, - ProposedValue: 6, - AdjustmentRatio: 0.5, - Reason: "增加并发处理", - ExpectedImpact: "提高处理速度", - Risk: "high", - }, - } - - return &TuningDecision{ - Adjustments: adjustments, - Confidence: 0.75, - ExpectedImprovement: 0.25, - Risk: 0.4, - AutoExecute: false, // 高风险，需要手动确认 - }, nil -} - -// createCacheOptimizationDecision 创建缓存优化决策 -func (t *TuningStrategy) createCacheOptimizationDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - adjustments := []ParameterAdjustment{ - { - ParameterName: "cache_ttl", - CurrentValue: "1h", - ProposedValue: "2h", - AdjustmentRatio: 1.0, // 增加100% - Reason: "延长缓存生存时间", - ExpectedImpact: "提高缓存命中率", - Risk: "low", - }, - { - ParameterName: "cache_size_limit", - CurrentValue: 1000, - ProposedValue: 1500, - AdjustmentRatio: 0.5, - Reason: "增加缓存容量", - ExpectedImpact: "减少缓存驱逐", - Risk: "medium", - }, - } - - return &TuningDecision{ - Adjustments: adjustments, - Confidence: 0.9, - ExpectedImprovement: 0.3, - Risk: 0.2, - AutoExecute: true, - }, nil -} - -// createOverallPerformanceDecision 创建整体性能优化决策 -func (t *TuningStrategy) createOverallPerformanceDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - adjustments := []ParameterAdjustment{ - { - ParameterName: "global_optimization", - CurrentValue: false, - ProposedValue: true, - AdjustmentRatio: 1.0, - Reason: "启用全局优化", - ExpectedImpact: "整体性能提升", - Risk: "medium", - }, - { - ParameterName: "compression_level", - CurrentValue: "standard", - ProposedValue: "high", - AdjustmentRatio: 0.3, - Reason: "提高压缩效率", - ExpectedImpact: "减少存储开销", - Risk: "low", - }, - } - - return &TuningDecision{ - Adjustments: adjustments, - Confidence: 0.7, - ExpectedImprovement: 0.2, - Risk: 0.35, - AutoExecute: true, - }, nil -} - -// createPreventiveDecision 创建预防性调优决策 -func (t *TuningStrategy) createPreventiveDecision(engine *AdaptiveTuningEngine) (*TuningDecision, error) { - // 基于趋势预测的预防性调优 - adjustments := []ParameterAdjustment{ - { - ParameterName: "preventive_scaling", - CurrentValue: 1.0, - ProposedValue: 1.1, - AdjustmentRatio: 0.1, - Reason: "预防性资源扩展", - ExpectedImpact: "避免性能下降", - Risk: "low", - }, - } - - return &TuningDecision{ - Adjustments: adjustments, - Confidence: 0.6, - ExpectedImprovement: 0.1, - Risk: 0.15, - AutoExecute: true, - }, nil -} - -// ExecuteDecision 执行决策 -func (t *TuningStrategy) ExecuteDecision(decision *TuningDecision) *StrategyExecution { - execution := &StrategyExecution{ - Timestamp: time.Now(), - Decision: decision, - Executed: false, - } - - // 简化的执行逻辑 - execution.Executed = true - execution.Result = &ExecutionResult{ - Success: true, - PerformanceBefore: 70.0, // 模拟值 - PerformanceAfter: 85.0, // 模拟值 - Improvement: 0.15, - SideEffects: []string{}, - } - - // 记录执行历史 - t.mutex.Lock() - t.executionHistory = append(t.executionHistory, execution) - - // 限制历史记录大小 - if len(t.executionHistory) > 100 { - t.executionHistory = t.executionHistory[1:] - } - t.mutex.Unlock() - - return execution -} - -// GetExecutionHistory 获取执行历史 -func (t *TuningStrategy) GetExecutionHistory(limit int) []*StrategyExecution { - t.mutex.RLock() - defer t.mutex.RUnlock() - - if limit <= 0 || limit > len(t.executionHistory) { - limit = len(t.executionHistory) - } - - history := make([]*StrategyExecution, limit) - startIndex := len(t.executionHistory) - limit - copy(history, t.executionHistory[startIndex:]) - - return history -} - -// UpdateStrategy 更新策略 -func (t *TuningStrategy) UpdateStrategy(strategyType string) { - t.mutex.Lock() - defer t.mutex.Unlock() - - t.strategyType = strategyType - - // 根据策略类型调整规则优先级和启用状态 - switch strategyType { - case "conservative": - // 保守策略：只启用低风险规则 - for _, rule := range t.rules { - rule.Enabled = rule.Priority <= 5 - } - - case "aggressive": - // 激进策略：启用所有规则 - for _, rule := range t.rules { - rule.Enabled = true - } - - case "balanced": - // 平衡策略：默认设置 - for _, rule := range t.rules { - rule.Enabled = true - } - } -} - -// GetStrategyStats 获取策略统计 -func (t *TuningStrategy) GetStrategyStats() map[string]interface{} { - t.mutex.RLock() - defer t.mutex.RUnlock() - - stats := map[string]interface{}{ - "strategy_type": t.strategyType, - "total_executions": len(t.executionHistory), - "enabled_rules": 0, - "rule_statistics": make(map[string]interface{}), - } - - enabledRules := 0 - ruleStats := make(map[string]interface{}) - - for _, rule := range t.rules { - if rule.Enabled { - enabledRules++ - } - - ruleStats[rule.Name] = map[string]interface{}{ - "enabled": rule.Enabled, - "priority": rule.Priority, - "trigger_count": rule.TriggerCount, - "last_triggered": rule.LastTriggered, - } - } - - stats["enabled_rules"] = enabledRules - stats["rule_statistics"] = ruleStats - - // 计算成功率 - successfulExecutions := 0 - for _, execution := range t.executionHistory { - if execution.Result != nil && execution.Result.Success { - successfulExecutions++ - } - } - - if len(t.executionHistory) > 0 { - stats["success_rate"] = float64(successfulExecutions) / float64(len(t.executionHistory)) - } else { - stats["success_rate"] = 0.0 - } - - return stats -} \ No newline at end of file diff --git a/util/parser_util.go b/util/parser_util.go index a557542..306507b 100644 --- a/util/parser_util.go +++ b/util/parser_util.go @@ -135,18 +135,6 @@ func ParseSearchResults(html string, channel string) ([]model.SearchResult, stri var results []model.SearchResult var nextPageParam string - // 查找分页链接 - 使用next而不是prev来获取下一页 - // doc.Find("link[rel='next']").Each(func(i int, s *goquery.Selection) { - // href, exists := s.Attr("href") - // if exists { - // // 从href中提取before参数 - // parts := strings.Split(href, "before=") - // if len(parts) > 1 { - // nextPageParam = strings.Split(parts[1], "&")[0] - // } - // } - // }) - // 查找消息块 doc.Find(".tgme_widget_message_wrap").Each(func(i int, s *goquery.Selection) { messageDiv := s.Find(".tgme_widget_message")