Files
urldb/utils/scheduler.go

1065 lines
30 KiB
Go
Raw Normal View History

2025-07-11 17:45:16 +08:00
package utils
import (
2025-07-18 00:34:27 +08:00
"fmt"
"math/rand"
2025-07-11 17:45:16 +08:00
"strings"
2025-07-12 21:23:23 +08:00
"sync"
2025-07-11 17:45:16 +08:00
"time"
2025-07-17 14:08:52 +08:00
2025-07-18 09:42:07 +08:00
panutils "github.com/ctwj/urldb/common"
commonutils "github.com/ctwj/urldb/common/utils"
"github.com/ctwj/urldb/db/entity"
"github.com/ctwj/urldb/db/repo"
2025-07-24 01:05:46 +08:00
"gorm.io/gorm"
2025-07-11 17:45:16 +08:00
)
// Scheduler 定时任务管理器
type Scheduler struct {
2025-07-25 22:24:08 +08:00
doubanService *DoubanService
hotDramaRepo repo.HotDramaRepository
readyResourceRepo repo.ReadyResourceRepository
resourceRepo repo.ResourceRepository
systemConfigRepo repo.SystemConfigRepository
panRepo repo.PanRepository
cksRepo repo.CksRepository
// 新增
tagRepo repo.TagRepository
categoryRepo repo.CategoryRepository
2025-07-12 21:23:23 +08:00
stopChan chan bool
isRunning bool
readyResourceRunning bool
2025-07-18 00:34:27 +08:00
autoTransferRunning bool
2025-07-12 21:23:23 +08:00
processingMutex sync.Mutex // 防止ready_resource任务重叠执行
hotDramaMutex sync.Mutex // 防止热播剧任务重叠执行
2025-07-18 00:34:27 +08:00
autoTransferMutex sync.Mutex // 防止自动转存任务重叠执行
2025-07-15 12:50:24 +08:00
// 平台映射缓存
panCache map[string]*uint // serviceType -> panID
panCacheOnce sync.Once
2025-07-11 17:45:16 +08:00
}
// NewScheduler 创建新的定时任务管理器
2025-07-25 22:24:08 +08:00
func NewScheduler(hotDramaRepo repo.HotDramaRepository, readyResourceRepo repo.ReadyResourceRepository, resourceRepo repo.ResourceRepository, systemConfigRepo repo.SystemConfigRepository, panRepo repo.PanRepository, cksRepo repo.CksRepository, tagRepo repo.TagRepository, categoryRepo repo.CategoryRepository) *Scheduler {
2025-07-11 17:45:16 +08:00
return &Scheduler{
2025-07-12 21:23:23 +08:00
doubanService: NewDoubanService(),
hotDramaRepo: hotDramaRepo,
readyResourceRepo: readyResourceRepo,
resourceRepo: resourceRepo,
systemConfigRepo: systemConfigRepo,
2025-07-15 12:50:24 +08:00
panRepo: panRepo,
2025-07-18 00:34:27 +08:00
cksRepo: cksRepo,
2025-07-25 22:24:08 +08:00
tagRepo: tagRepo,
categoryRepo: categoryRepo,
2025-07-12 21:23:23 +08:00
stopChan: make(chan bool),
isRunning: false,
readyResourceRunning: false,
2025-07-18 00:34:27 +08:00
autoTransferRunning: false,
2025-07-12 21:23:23 +08:00
processingMutex: sync.Mutex{},
hotDramaMutex: sync.Mutex{},
2025-07-18 00:34:27 +08:00
autoTransferMutex: sync.Mutex{},
2025-07-15 12:50:24 +08:00
panCache: make(map[string]*uint),
2025-07-11 17:45:16 +08:00
}
}
// StartHotDramaScheduler 启动热播剧定时任务
func (s *Scheduler) StartHotDramaScheduler() {
if s.isRunning {
2025-07-17 17:42:52 +08:00
Info("热播剧定时任务已在运行中")
2025-07-11 17:45:16 +08:00
return
}
s.isRunning = true
2025-07-17 17:42:52 +08:00
Info("启动热播剧定时任务")
2025-07-11 17:45:16 +08:00
go func() {
2025-07-15 18:44:09 +08:00
ticker := time.NewTicker(12 * time.Hour) // 每12小时执行一次
2025-07-11 17:45:16 +08:00
defer ticker.Stop()
// 立即执行一次
s.fetchHotDramaData()
for {
select {
case <-ticker.C:
2025-07-12 21:23:23 +08:00
// 使用TryLock防止任务重叠执行
if s.hotDramaMutex.TryLock() {
go func() {
defer s.hotDramaMutex.Unlock()
s.fetchHotDramaData()
}()
} else {
2025-07-17 17:42:52 +08:00
Info("上一次热播剧任务还在执行中,跳过本次执行")
2025-07-12 21:23:23 +08:00
}
2025-07-11 17:45:16 +08:00
case <-s.stopChan:
2025-07-17 17:42:52 +08:00
Info("停止热播剧定时任务")
2025-07-11 17:45:16 +08:00
return
}
}
}()
}
// StopHotDramaScheduler 停止热播剧定时任务
func (s *Scheduler) StopHotDramaScheduler() {
if !s.isRunning {
2025-07-17 17:42:52 +08:00
Info("热播剧定时任务未在运行")
2025-07-11 17:45:16 +08:00
return
}
s.stopChan <- true
s.isRunning = false
2025-07-17 17:42:52 +08:00
Info("已发送停止信号给热播剧定时任务")
2025-07-11 17:45:16 +08:00
}
// fetchHotDramaData 获取热播剧数据
func (s *Scheduler) fetchHotDramaData() {
2025-07-17 17:42:52 +08:00
Info("开始获取热播剧数据...")
2025-07-11 17:45:16 +08:00
2025-07-15 18:44:09 +08:00
// 直接处理电影和电视剧数据不再需要FetchHotDramaNames
s.processHotDramaNames([]string{})
2025-07-11 17:45:16 +08:00
}
// processHotDramaNames 处理热播剧名字
func (s *Scheduler) processHotDramaNames(dramaNames []string) {
2025-07-17 17:42:52 +08:00
Info("开始处理热播剧数据,共 %d 个", len(dramaNames))
2025-07-11 17:45:16 +08:00
2025-07-15 18:44:09 +08:00
// 收集所有数据
var allDramas []*entity.HotDrama
// 获取电影数据
movieDramas := s.processMovieData()
allDramas = append(allDramas, movieDramas...)
// 获取电视剧数据
tvDramas := s.processTvData()
allDramas = append(allDramas, tvDramas...)
// 清空数据库
2025-07-17 17:42:52 +08:00
Info("准备清空数据库,当前共有 %d 条数据", len(allDramas))
2025-07-15 18:44:09 +08:00
if err := s.hotDramaRepo.DeleteAll(); err != nil {
2025-07-17 17:42:52 +08:00
Error("清空数据库失败: %v", err)
2025-07-15 18:44:09 +08:00
return
}
2025-07-17 17:42:52 +08:00
Info("数据库清空完成")
2025-07-15 18:44:09 +08:00
// 批量插入所有数据
if len(allDramas) > 0 {
2025-07-17 17:42:52 +08:00
Info("开始批量插入 %d 条数据", len(allDramas))
2025-07-15 18:44:09 +08:00
if err := s.hotDramaRepo.BatchCreate(allDramas); err != nil {
2025-07-17 17:42:52 +08:00
Error("批量插入数据失败: %v", err)
2025-07-15 18:44:09 +08:00
} else {
2025-07-17 17:42:52 +08:00
Info("成功批量插入 %d 条数据", len(allDramas))
2025-07-15 18:44:09 +08:00
}
} else {
2025-07-17 17:42:52 +08:00
Info("没有数据需要插入")
2025-07-15 18:44:09 +08:00
}
2025-07-11 17:45:16 +08:00
2025-07-17 17:42:52 +08:00
Info("热播剧数据处理完成")
2025-07-11 17:45:16 +08:00
}
// processMovieData 处理电影数据
2025-07-15 18:44:09 +08:00
func (s *Scheduler) processMovieData() []*entity.HotDrama {
2025-07-17 17:42:52 +08:00
Info("开始处理电影数据...")
2025-07-11 17:45:16 +08:00
2025-07-15 18:44:09 +08:00
var movieDramas []*entity.HotDrama
// 使用GetTypePage方法获取电影数据
movieResult, err := s.doubanService.GetTypePage("热门", "全部")
2025-07-11 17:45:16 +08:00
if err != nil {
2025-07-17 17:42:52 +08:00
Error("获取电影榜单失败: %v", err)
2025-07-15 18:44:09 +08:00
return movieDramas
2025-07-11 17:45:16 +08:00
}
if movieResult.Success && movieResult.Data != nil {
2025-07-17 17:42:52 +08:00
Info("电影获取到 %d 个数据", len(movieResult.Data.Items))
2025-07-15 18:44:09 +08:00
2025-07-11 17:45:16 +08:00
for _, item := range movieResult.Data.Items {
drama := &entity.HotDrama{
2025-07-15 16:34:21 +08:00
Title: item.Title,
CardSubtitle: item.CardSubtitle,
EpisodesInfo: item.EpisodesInfo,
IsNew: item.IsNew,
Rating: item.Rating.Value,
RatingCount: item.Rating.Count,
Year: item.Year,
Region: item.Region,
Genres: strings.Join(item.Genres, ", "),
Directors: strings.Join(item.Directors, ", "),
Actors: strings.Join(item.Actors, ", "),
PosterURL: item.Pic.Normal,
Category: "电影",
SubType: "热门",
Source: "douban",
DoubanID: item.ID,
DoubanURI: item.URI,
2025-07-11 17:45:16 +08:00
}
2025-07-15 18:44:09 +08:00
movieDramas = append(movieDramas, drama)
2025-07-17 17:42:52 +08:00
Info("收集电影: %s (评分: %.1f, 年份: %s, 地区: %s)",
2025-07-15 18:44:09 +08:00
item.Title, item.Rating.Value, item.Year, item.Region)
2025-07-11 17:45:16 +08:00
}
2025-07-15 18:44:09 +08:00
} else {
2025-07-17 17:42:52 +08:00
Warn("电影获取数据失败或为空")
2025-07-11 17:45:16 +08:00
}
2025-07-15 18:44:09 +08:00
2025-07-17 17:42:52 +08:00
Info("电影数据处理完成,共收集 %d 条数据", len(movieDramas))
2025-07-15 18:44:09 +08:00
return movieDramas
2025-07-11 17:45:16 +08:00
}
// processTvData 处理电视剧数据
2025-07-15 18:44:09 +08:00
func (s *Scheduler) processTvData() []*entity.HotDrama {
2025-07-17 17:42:52 +08:00
Info("开始处理电视剧数据...")
2025-07-11 17:45:16 +08:00
2025-07-15 18:44:09 +08:00
var tvDramas []*entity.HotDrama
2025-07-11 17:45:16 +08:00
2025-07-15 18:44:09 +08:00
// 获取所有tv类型
tvTypes := s.doubanService.GetAllTvTypes()
2025-07-17 17:42:52 +08:00
Info("获取到 %d 个tv类型: %v", len(tvTypes), tvTypes)
2025-07-15 18:44:09 +08:00
// 遍历每个type分别请求数据
for _, tvType := range tvTypes {
2025-07-17 17:42:52 +08:00
Info("正在处理tv类型: %s", tvType)
2025-07-11 17:45:16 +08:00
2025-07-15 18:44:09 +08:00
// 使用GetTypePage方法请求数据
tvResult, err := s.doubanService.GetTypePage("tv", tvType)
if err != nil {
2025-07-17 17:42:52 +08:00
Error("获取tv类型 %s 数据失败: %v", tvType, err)
2025-07-15 18:44:09 +08:00
continue
}
if tvResult.Success && tvResult.Data != nil {
2025-07-17 17:42:52 +08:00
Info("tv类型 %s 获取到 %d 个数据", tvType, len(tvResult.Data.Items))
2025-07-15 18:44:09 +08:00
for _, item := range tvResult.Data.Items {
drama := &entity.HotDrama{
Title: item.Title,
CardSubtitle: item.CardSubtitle,
EpisodesInfo: item.EpisodesInfo,
IsNew: item.IsNew,
Rating: item.Rating.Value,
RatingCount: item.Rating.Count,
Year: item.Year,
Region: item.Region,
Genres: strings.Join(item.Genres, ", "),
Directors: strings.Join(item.Directors, ", "),
Actors: strings.Join(item.Actors, ", "),
PosterURL: item.Pic.Normal,
Category: "电视剧",
SubType: tvType, // 使用具体的tv类型
Source: "douban",
DoubanID: item.ID,
DoubanURI: item.URI,
}
tvDramas = append(tvDramas, drama)
2025-07-17 17:42:52 +08:00
Info("收集tv类型 %s: %s (评分: %.1f, 年份: %s, 地区: %s)",
2025-07-15 18:44:09 +08:00
tvType, item.Title, item.Rating.Value, item.Year, item.Region)
2025-07-11 17:45:16 +08:00
}
2025-07-15 18:44:09 +08:00
} else {
2025-07-17 17:42:52 +08:00
Warn("tv类型 %s 获取数据失败或为空", tvType)
2025-07-11 17:45:16 +08:00
}
2025-07-15 18:44:09 +08:00
// 每个type请求间隔1秒避免请求过于频繁
time.Sleep(1 * time.Second)
2025-07-11 17:45:16 +08:00
}
2025-07-15 18:44:09 +08:00
2025-07-17 17:42:52 +08:00
Info("电视剧数据处理完成,共收集 %d 条数据", len(tvDramas))
2025-07-15 18:44:09 +08:00
return tvDramas
2025-07-11 17:45:16 +08:00
}
// IsRunning 检查定时任务是否在运行
func (s *Scheduler) IsRunning() bool {
return s.isRunning
}
// GetHotDramaNames 手动获取热播剧名字(用于测试或手动调用)
func (s *Scheduler) GetHotDramaNames() ([]string, error) {
2025-07-15 18:44:09 +08:00
// 由于删除了FetchHotDramaNames方法返回空数组
return []string{}, nil
2025-07-11 17:45:16 +08:00
}
2025-07-12 21:23:23 +08:00
// StartReadyResourceScheduler 启动待处理资源自动处理任务
func (s *Scheduler) StartReadyResourceScheduler() {
if s.readyResourceRunning {
2025-07-17 17:42:52 +08:00
Info("待处理资源自动处理任务已在运行中")
2025-07-12 21:23:23 +08:00
return
}
s.readyResourceRunning = true
2025-07-17 17:42:52 +08:00
Info("启动待处理资源自动处理任务")
2025-07-12 21:23:23 +08:00
go func() {
2025-07-15 12:50:24 +08:00
// 获取系统配置中的间隔时间
2025-07-29 17:04:49 +08:00
interval := 3 * time.Minute // 默认3分钟
if autoProcessInterval, err := s.systemConfigRepo.GetConfigInt(entity.ConfigKeyAutoProcessInterval); err == nil && autoProcessInterval > 0 {
interval = time.Duration(autoProcessInterval) * time.Minute
2025-07-15 12:50:24 +08:00
}
ticker := time.NewTicker(interval)
2025-07-12 21:23:23 +08:00
defer ticker.Stop()
2025-07-17 17:42:52 +08:00
Info("待处理资源自动处理任务已启动,间隔时间: %v", interval)
2025-07-15 12:50:24 +08:00
2025-07-12 21:23:23 +08:00
// 立即执行一次
s.processReadyResources()
for {
select {
case <-ticker.C:
// 使用TryLock防止任务重叠执行
if s.processingMutex.TryLock() {
go func() {
defer s.processingMutex.Unlock()
s.processReadyResources()
}()
} else {
2025-07-17 17:42:52 +08:00
Info("上一次待处理资源任务还在执行中,跳过本次执行")
2025-07-12 21:23:23 +08:00
}
case <-s.stopChan:
2025-07-17 17:42:52 +08:00
Info("停止待处理资源自动处理任务")
2025-07-12 21:23:23 +08:00
return
}
}
}()
}
// StopReadyResourceScheduler 停止待处理资源自动处理任务
func (s *Scheduler) StopReadyResourceScheduler() {
if !s.readyResourceRunning {
2025-07-17 17:42:52 +08:00
Info("待处理资源自动处理任务未在运行")
2025-07-12 21:23:23 +08:00
return
}
s.stopChan <- true
s.readyResourceRunning = false
2025-07-17 17:42:52 +08:00
Info("已发送停止信号给待处理资源自动处理任务")
2025-07-12 21:23:23 +08:00
}
// processReadyResources 处理待处理资源
func (s *Scheduler) processReadyResources() {
2025-07-17 17:42:52 +08:00
Info("开始处理待处理资源...")
2025-07-12 21:23:23 +08:00
// 检查系统配置,确认是否启用自动处理
2025-07-29 17:04:49 +08:00
autoProcess, err := s.systemConfigRepo.GetConfigBool(entity.ConfigKeyAutoProcessReadyResources)
2025-07-12 21:23:23 +08:00
if err != nil {
2025-07-17 17:42:52 +08:00
Error("获取系统配置失败: %v", err)
2025-07-12 21:23:23 +08:00
return
}
2025-07-29 17:04:49 +08:00
if !autoProcess {
2025-07-17 17:42:52 +08:00
Info("自动处理待处理资源功能已禁用")
2025-07-12 21:23:23 +08:00
return
}
// 获取所有没有错误的待处理资源
2025-07-12 21:23:23 +08:00
readyResources, err := s.readyResourceRepo.FindAll()
// readyResources, err := s.readyResourceRepo.FindWithoutErrors()
2025-07-12 21:23:23 +08:00
if err != nil {
2025-07-17 17:42:52 +08:00
Error("获取待处理资源失败: %v", err)
2025-07-12 21:23:23 +08:00
return
}
if len(readyResources) == 0 {
2025-07-17 17:42:52 +08:00
Info("没有待处理的资源")
2025-07-12 21:23:23 +08:00
return
}
2025-07-17 17:42:52 +08:00
Info("找到 %d 个待处理资源,开始处理...", len(readyResources))
2025-07-12 21:23:23 +08:00
processedCount := 0
2025-07-13 09:04:50 +08:00
factory := panutils.GetInstance() // 使用单例模式
2025-07-12 21:23:23 +08:00
for _, readyResource := range readyResources {
//readyResource.URL 是 查重
exits, err := s.resourceRepo.FindExists(readyResource.URL)
if err != nil {
2025-07-17 17:42:52 +08:00
Error("查重失败: %v", err)
2025-07-12 21:23:23 +08:00
continue
}
if exits {
2025-07-17 17:42:52 +08:00
Info("资源已存在: %s", readyResource.URL)
2025-07-12 21:23:23 +08:00
s.readyResourceRepo.Delete(readyResource.ID)
continue
}
2025-07-13 08:02:55 +08:00
if err := s.convertReadyResourceToResource(readyResource, factory); err != nil {
2025-07-17 17:42:52 +08:00
Error("处理资源失败 (ID: %d): %v", readyResource.ID, err)
// 保存完整的错误信息
readyResource.ErrorMsg = err.Error()
if updateErr := s.readyResourceRepo.Update(&readyResource); updateErr != nil {
Error("更新错误信息失败 (ID: %d): %v", readyResource.ID, updateErr)
} else {
Info("已保存错误信息到资源 (ID: %d): %s", readyResource.ID, err.Error())
}
} else {
// 处理成功删除readyResource
s.readyResourceRepo.Delete(readyResource.ID)
processedCount++
Info("成功处理资源: %s", readyResource.URL)
2025-07-12 21:23:23 +08:00
}
}
2025-07-17 17:42:52 +08:00
Info("待处理资源处理完成,共处理 %d 个资源", processedCount)
2025-07-12 21:23:23 +08:00
}
// convertReadyResourceToResource 将待处理资源转换为正式资源
2025-07-13 08:02:55 +08:00
func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyResource, factory *panutils.PanFactory) error {
2025-07-17 17:42:52 +08:00
Debug("开始处理资源: %s", readyResource.URL)
2025-07-13 08:02:55 +08:00
2025-07-13 09:04:50 +08:00
// 提取分享ID和服务类型
shareID, serviceType := panutils.ExtractShareId(readyResource.URL)
if serviceType == panutils.NotFound {
2025-07-17 17:42:52 +08:00
Warn("不支持的链接地址: %s", readyResource.URL)
return NewUnsupportedLinkError(readyResource.URL)
2025-07-13 09:04:50 +08:00
}
2025-07-17 17:42:52 +08:00
Debug("检测到服务类型: %s, 分享ID: %s", serviceType.String(), shareID)
2025-07-13 09:04:50 +08:00
2025-07-25 22:24:08 +08:00
resource := &entity.Resource{
Title: derefString(readyResource.Title),
Description: readyResource.Description,
URL: readyResource.URL,
Cover: readyResource.Img,
IsValid: true,
IsPublic: true,
Key: readyResource.Key,
PanID: s.getPanIDByServiceType(serviceType),
}
2025-07-13 09:57:36 +08:00
// 不是夸克,直接保存,
if serviceType != panutils.Quark {
// 检测是否有效
checkResult, err := commonutils.CheckURL(readyResource.URL)
if err != nil {
Error("链接检查失败: %v", err)
return NewLinkCheckError(readyResource.URL, err.Error())
}
2025-07-13 09:57:36 +08:00
if !checkResult.Status {
2025-07-17 17:42:52 +08:00
Warn("链接无效: %s", readyResource.URL)
return NewInvalidLinkError(readyResource.URL, "链接状态检查失败")
2025-07-13 09:57:36 +08:00
}
2025-07-25 22:24:08 +08:00
} else {
// 获取夸克网盘账号的 cookie
panID := s.getPanIDByServiceType(serviceType)
if panID == nil {
Error("未找到对应的平台ID")
return NewPlatformNotFoundError(serviceType.String())
}
accounts, err := s.cksRepo.FindByPanID(*panID)
if err != nil {
Error("获取夸克网盘账号失败: %v", err)
return NewServiceCreationError(readyResource.URL, fmt.Sprintf("获取网盘账号失败: %v", err))
}
if len(accounts) == 0 {
Error("没有可用的夸克网盘账号")
return NewNoAccountError(serviceType.String())
}
// 选择第一个有效的账号
var selectedAccount *entity.Cks
for _, account := range accounts {
if account.IsValid {
selectedAccount = &account
break
}
}
if selectedAccount == nil {
Error("没有有效的夸克网盘账号")
return NewNoValidAccountError(serviceType.String())
}
Debug("使用夸克网盘账号: %d, Cookie: %s", selectedAccount.ID, selectedAccount.Ck[:20]+"...")
2025-07-25 22:24:08 +08:00
// 准备配置
config := &panutils.PanConfig{
URL: readyResource.URL,
Code: "", // 可以从readyResource中获取
IsType: 1, // 转存并分享后的资源信息 0 转存后分享, 1 只获取基本信息
ExpiredType: 1, // 永久分享
AdFid: "",
Stoken: "",
Cookie: selectedAccount.Ck, // 添加 cookie
2025-07-25 22:24:08 +08:00
}
// 通过工厂获取对应的网盘服务单例
panService, err := factory.CreatePanService(readyResource.URL, config)
if err != nil {
Error("获取网盘服务失败: %v", err)
return NewServiceCreationError(readyResource.URL, err.Error())
2025-07-25 22:24:08 +08:00
}
// 统一处理:尝试转存获取标题
result, err := panService.Transfer(shareID)
if err != nil {
Error("网盘信息获取失败: %v", err)
return NewTransferFailedError(readyResource.URL, err.Error())
2025-07-25 22:24:08 +08:00
}
if !result.Success {
Error("网盘信息获取失败: %s", result.Message)
return NewTransferFailedError(readyResource.URL, result.Message)
2025-07-25 22:24:08 +08:00
}
2025-07-13 09:57:36 +08:00
// 如果获取到了标题,更新资源标题
// if result.Title != "" {
// resource.Title = result.Title
// }
2025-07-13 08:02:55 +08:00
}
2025-07-25 22:24:08 +08:00
// 处理标签
tagIDs, err := s.handleTags(readyResource.Tags)
if err != nil {
2025-07-25 22:24:08 +08:00
Error("处理标签失败: %v", err)
return NewTagProcessingError(err.Error())
2025-07-13 08:02:55 +08:00
}
// 如果没有标签tagIDs 可能为 nil这是正常的
if tagIDs == nil {
tagIDs = []uint{} // 初始化为空数组
}
2025-07-25 22:24:08 +08:00
// 处理分类
categoryID, err := s.resolveCategory(readyResource.Category, tagIDs)
2025-07-13 09:04:50 +08:00
if err != nil {
2025-07-25 22:24:08 +08:00
Error("处理分类失败: %v", err)
return NewCategoryProcessingError(err.Error())
2025-07-13 09:04:50 +08:00
}
2025-07-25 22:24:08 +08:00
if categoryID != nil {
resource.CategoryID = categoryID
2025-07-13 09:04:50 +08:00
}
2025-07-25 22:24:08 +08:00
// 保存资源
err = s.resourceRepo.Create(resource)
if err != nil {
Error("资源保存失败: %v", err)
return NewResourceSaveError(readyResource.URL, err.Error())
2025-07-25 22:24:08 +08:00
}
// 插入 resource_tags 关联
if len(tagIDs) > 0 {
for _, tagID := range tagIDs {
err := s.resourceRepo.CreateResourceTag(resource.ID, tagID)
if err != nil {
Error("插入资源标签关联失败: %v", err)
// 这里不返回错误,因为资源已经保存成功,标签关联失败不影响主要功能
}
2025-07-13 08:02:55 +08:00
}
} else {
Debug("没有标签,跳过插入资源标签关联")
2025-07-12 21:23:23 +08:00
}
return nil
}
2025-07-26 00:32:02 +08:00
// // getOrCreateCategory 获取或创建分类
// func (s *Scheduler) getOrCreateCategory(categoryName string) (uint, error) {
// // 这里需要实现分类的查找和创建逻辑
// // 由于没有CategoryRepository的注入这里先返回0
// // 你可以根据需要添加CategoryRepository的依赖
// return 0, nil
// }
2025-07-12 21:23:23 +08:00
2025-07-15 12:50:24 +08:00
// initPanCache 初始化平台映射缓存
func (s *Scheduler) initPanCache() {
s.panCacheOnce.Do(func() {
// 获取所有平台数据
pans, err := s.panRepo.FindAll()
if err != nil {
2025-07-17 17:42:52 +08:00
Error("初始化平台缓存失败: %v", err)
2025-07-15 12:50:24 +08:00
return
}
// 建立 ServiceType 到 PanID 的映射
serviceTypeToPanName := map[string]string{
"quark": "quark",
"alipan": "aliyun", // 阿里云盘在数据库中的名称是 aliyun
"baidu": "baidu",
"uc": "uc",
"unknown": "other",
}
// 创建平台名称到ID的映射
panNameToID := make(map[string]*uint)
for _, pan := range pans {
panID := pan.ID
panNameToID[pan.Name] = &panID
}
// 建立 ServiceType 到 PanID 的映射
for serviceType, panName := range serviceTypeToPanName {
if panID, exists := panNameToID[panName]; exists {
s.panCache[serviceType] = panID
2025-07-17 17:42:52 +08:00
Debug("平台映射缓存: %s -> %s (ID: %d)", serviceType, panName, *panID)
2025-07-15 12:50:24 +08:00
} else {
2025-07-17 17:42:52 +08:00
Warn("警告: 未找到平台 %s 对应的数据库记录", panName)
2025-07-15 12:50:24 +08:00
}
}
// 确保有默认的 other 平台
if otherID, exists := panNameToID["other"]; exists {
s.panCache["unknown"] = otherID
}
2025-07-17 17:42:52 +08:00
Info("平台映射缓存初始化完成,共 %d 个映射", len(s.panCache))
2025-07-15 12:50:24 +08:00
})
}
// getPanIDByServiceType 根据服务类型获取平台ID
func (s *Scheduler) getPanIDByServiceType(serviceType panutils.ServiceType) *uint {
s.initPanCache()
serviceTypeStr := serviceType.String()
if panID, exists := s.panCache[serviceTypeStr]; exists {
return panID
}
// 如果找不到,返回 other 平台的ID
if otherID, exists := s.panCache["other"]; exists {
2025-07-17 17:42:52 +08:00
Warn("未找到服务类型 %s 的映射,使用默认平台 other", serviceTypeStr)
2025-07-15 12:50:24 +08:00
return otherID
}
2025-07-17 17:42:52 +08:00
Warn("未找到服务类型 %s 的映射且没有默认平台返回nil", serviceTypeStr)
2025-07-15 12:50:24 +08:00
return nil
}
2025-07-12 21:23:23 +08:00
// IsReadyResourceRunning 检查待处理资源自动处理任务是否在运行
func (s *Scheduler) IsReadyResourceRunning() bool {
return s.readyResourceRunning
}
2025-07-18 00:34:27 +08:00
// StartAutoTransferScheduler 启动自动转存定时任务
func (s *Scheduler) StartAutoTransferScheduler() {
if s.autoTransferRunning {
Info("自动转存定时任务已在运行中")
return
}
s.autoTransferRunning = true
Info("启动自动转存定时任务")
go func() {
// 获取系统配置中的间隔时间
interval := 5 * time.Minute // 默认5分钟
2025-07-29 17:04:49 +08:00
if autoProcessInterval, err := s.systemConfigRepo.GetConfigInt(entity.ConfigKeyAutoProcessInterval); err == nil && autoProcessInterval > 0 {
interval = time.Duration(autoProcessInterval) * time.Minute
2025-07-18 00:34:27 +08:00
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
Info("自动转存定时任务已启动,间隔时间: %v", interval)
// 立即执行一次
s.processAutoTransfer()
for {
select {
case <-ticker.C:
// 使用TryLock防止任务重叠执行
if s.autoTransferMutex.TryLock() {
go func() {
defer s.autoTransferMutex.Unlock()
s.processAutoTransfer()
}()
} else {
Info("上一次自动转存任务还在执行中,跳过本次执行")
}
case <-s.stopChan:
Info("停止自动转存定时任务")
return
}
}
}()
}
// StopAutoTransferScheduler 停止自动转存定时任务
func (s *Scheduler) StopAutoTransferScheduler() {
if !s.autoTransferRunning {
Info("自动转存定时任务未在运行")
return
}
s.stopChan <- true
s.autoTransferRunning = false
Info("已发送停止信号给自动转存定时任务")
}
// IsAutoTransferRunning 检查自动转存定时任务是否在运行
func (s *Scheduler) IsAutoTransferRunning() bool {
return s.autoTransferRunning
}
// processAutoTransfer 处理自动转存
func (s *Scheduler) processAutoTransfer() {
Info("开始处理自动转存...")
// 检查系统配置,确认是否启用自动转存
2025-07-29 17:04:49 +08:00
autoTransferEnabled, err := s.systemConfigRepo.GetConfigBool(entity.ConfigKeyAutoTransferEnabled)
2025-07-18 00:34:27 +08:00
if err != nil {
Error("获取系统配置失败: %v", err)
return
}
2025-07-29 17:04:49 +08:00
if !autoTransferEnabled {
2025-07-18 00:34:27 +08:00
Info("自动转存功能已禁用")
return
}
2025-07-24 01:05:46 +08:00
// 获取quark平台ID
panRepoImpl, ok := s.panRepo.(interface{ GetDB() *gorm.DB })
if !ok {
Error("panRepo不支持GetDB方法")
return
}
var quarkPan entity.Pan
err = panRepoImpl.GetDB().Where("name = ?", "quark").First(&quarkPan).Error
if err != nil {
Error("未找到quark平台: %v", err)
return
}
quarkPanID := quarkPan.ID
// 获取所有账号
2025-07-18 00:34:27 +08:00
accounts, err := s.cksRepo.FindAll()
if err != nil {
Error("获取网盘账号失败: %v", err)
return
}
2025-07-29 17:04:49 +08:00
// 获取最小存储空间配置
autoTransferMinSpace, err := s.systemConfigRepo.GetConfigInt(entity.ConfigKeyAutoTransferMinSpace)
if err != nil {
Error("获取最小存储空间配置失败: %v", err)
return
}
2025-07-24 01:05:46 +08:00
// 过滤只保留已激活、quark平台、剩余空间足够的账号
2025-07-29 17:04:49 +08:00
minSpaceBytes := int64(autoTransferMinSpace) * 1024 * 1024 * 1024
2025-07-24 01:05:46 +08:00
var validAccounts []entity.Cks
for _, acc := range accounts {
if acc.IsValid && acc.PanID == quarkPanID && acc.LeftSpace >= minSpaceBytes {
validAccounts = append(validAccounts, acc)
}
}
if len(validAccounts) == 0 {
Info("没有可用的quark网盘账号")
2025-07-18 00:34:27 +08:00
return
}
2025-07-24 01:05:46 +08:00
Info("找到 %d 个可用quark网盘账号开始自动转存处理...", len(validAccounts))
2025-07-18 00:34:27 +08:00
// 获取需要转存的资源
2025-07-29 17:04:49 +08:00
resources, err := s.getResourcesForTransfer(quarkPanID)
2025-07-18 00:34:27 +08:00
if err != nil {
Error("获取需要转存的资源失败: %v", err)
return
}
if len(resources) == 0 {
Info("没有需要转存的资源")
return
}
Info("找到 %d 个需要转存的资源", len(resources))
2025-07-24 01:05:46 +08:00
// 并发自动转存
resourceCh := make(chan *entity.Resource, len(resources))
for _, res := range resources {
resourceCh <- res
}
close(resourceCh)
var wg sync.WaitGroup
for _, account := range validAccounts {
wg.Add(1)
go func(acc entity.Cks) {
defer wg.Done()
factory := panutils.GetInstance() // 使用单例模式
for res := range resourceCh {
2025-07-29 17:04:49 +08:00
if err := s.transferResource(res, []entity.Cks{acc}, factory); err != nil {
2025-07-24 01:05:46 +08:00
Error("转存资源失败 (ID: %d): %v", res.ID, err)
} else {
Info("成功转存资源: %s", res.Title)
rand.Seed(GetCurrentTime().UnixNano())
sleepSec := rand.Intn(3) + 1 // 1,2,3
time.Sleep(time.Duration(sleepSec) * time.Second)
2025-07-24 01:05:46 +08:00
}
}
}(account)
2025-07-18 00:34:27 +08:00
}
2025-07-24 01:05:46 +08:00
wg.Wait()
Info("自动转存处理完成,账号数: %d资源数: %d", len(validAccounts), len(resources))
2025-07-18 00:34:27 +08:00
}
// getResourcesForTransfer 获取需要转存的资源
2025-07-29 17:04:49 +08:00
func (s *Scheduler) getResourcesForTransfer(quarkPanID uint) ([]*entity.Resource, error) {
// 获取自动转存限制天数配置
autoTransferLimitDays, err := s.systemConfigRepo.GetConfigInt(entity.ConfigKeyAutoTransferLimitDays)
if err != nil {
Error("获取自动转存限制天数配置失败: %v", err)
return nil, err
}
days := autoTransferLimitDays
2025-07-24 01:05:46 +08:00
var sinceTime time.Time
if days > 0 {
sinceTime = GetCurrentTime().AddDate(0, 0, -days)
2025-07-24 01:05:46 +08:00
} else {
sinceTime = time.Time{}
}
2025-07-18 00:34:27 +08:00
2025-07-24 01:05:46 +08:00
repoImpl, ok := s.resourceRepo.(*repo.ResourceRepositoryImpl)
if !ok {
return nil, fmt.Errorf("resourceRepo不是ResourceRepositoryImpl类型")
}
return repoImpl.GetResourcesForTransfer(quarkPanID, sinceTime)
2025-07-18 00:34:27 +08:00
}
2025-07-24 01:05:46 +08:00
var resourceUpdateMutex sync.Mutex // 全局互斥锁,保证多协程安全
2025-07-18 00:34:27 +08:00
2025-07-24 01:05:46 +08:00
// transferResource 转存单个资源
2025-07-29 17:04:49 +08:00
func (s *Scheduler) transferResource(resource *entity.Resource, accounts []entity.Cks, factory *panutils.PanFactory) error {
2025-07-24 01:05:46 +08:00
if len(accounts) == 0 {
return fmt.Errorf("没有可用的网盘账号")
}
account := accounts[0]
2025-07-18 00:34:27 +08:00
2025-07-24 01:05:46 +08:00
service, err := factory.CreatePanService(resource.URL, &panutils.PanConfig{
URL: resource.URL,
ExpiredType: 0,
IsType: 0,
Cookie: account.Ck,
})
if err != nil {
return fmt.Errorf("创建网盘服务失败: %v", err)
2025-07-18 00:34:27 +08:00
}
2025-07-29 17:04:49 +08:00
// 获取最小存储空间配置
autoTransferMinSpace, err := s.systemConfigRepo.GetConfigInt(entity.ConfigKeyAutoTransferMinSpace)
if err != nil {
Error("获取最小存储空间配置失败: %v", err)
return err
}
// 检查账号剩余空间
minSpaceBytes := int64(autoTransferMinSpace) * 1024 * 1024 * 1024
if account.LeftSpace < minSpaceBytes {
return fmt.Errorf("账号剩余空间不足,需要 %d GB当前剩余 %d GB", autoTransferMinSpace, account.LeftSpace/1024/1024/1024)
}
// 提取分享ID
2025-07-24 01:05:46 +08:00
shareID, _ := commonutils.ExtractShareIdString(resource.URL)
2025-07-29 17:04:49 +08:00
// 转存资源
2025-07-24 01:05:46 +08:00
result, err := service.Transfer(shareID)
if err != nil {
resourceUpdateMutex.Lock()
defer resourceUpdateMutex.Unlock()
s.resourceRepo.Update(&entity.Resource{
ID: resource.ID,
ErrorMsg: err.Error(),
})
return fmt.Errorf("转存失败: %v", err)
}
if result == nil || !result.Success {
errMsg := "转存失败"
if result != nil && result.Message != "" {
errMsg = result.Message
}
resourceUpdateMutex.Lock()
defer resourceUpdateMutex.Unlock()
s.resourceRepo.Update(&entity.Resource{
ID: resource.ID,
ErrorMsg: errMsg,
})
return fmt.Errorf("转存失败: %s", errMsg)
}
// 提取转存链接、fid等
var saveURL, fid string
if data, ok := result.Data.(map[string]interface{}); ok {
if v, ok := data["shareUrl"]; ok {
saveURL, _ = v.(string)
}
if v, ok := data["fid"]; ok {
fid, _ = v.(string)
}
}
if saveURL == "" {
saveURL = result.ShareURL
}
2025-07-18 00:34:27 +08:00
2025-07-24 01:05:46 +08:00
resourceUpdateMutex.Lock()
defer resourceUpdateMutex.Unlock()
err = s.resourceRepo.Update(&entity.Resource{
ID: resource.ID,
SaveURL: saveURL,
CkID: &account.ID,
Fid: fid,
ErrorMsg: "",
})
if err != nil {
return fmt.Errorf("保存转存结果失败: %v", err)
}
2025-07-18 00:34:27 +08:00
return nil
}
2025-07-29 17:04:49 +08:00
// selectBestAccount 选择最佳账号
func (s *Scheduler) selectBestAccount(accounts []entity.Cks) *entity.Cks {
if len(accounts) == 0 {
return nil
}
2025-07-18 00:34:27 +08:00
2025-07-29 17:04:49 +08:00
// 获取最小存储空间配置
autoTransferMinSpace, err := s.systemConfigRepo.GetConfigInt(entity.ConfigKeyAutoTransferMinSpace)
if err != nil {
Error("获取最小存储空间配置失败: %v", err)
return &accounts[0] // 返回第一个账号
}
2025-07-18 00:34:27 +08:00
2025-07-29 17:04:49 +08:00
minSpaceBytes := int64(autoTransferMinSpace) * 1024 * 1024 * 1024
2025-07-18 00:34:27 +08:00
2025-07-29 17:04:49 +08:00
var bestAccount *entity.Cks
var bestScore int64 = -1
2025-07-18 00:34:27 +08:00
2025-07-29 17:04:49 +08:00
for i := range accounts {
account := &accounts[i]
2025-07-18 00:34:27 +08:00
if account.LeftSpace < minSpaceBytes {
2025-07-29 17:04:49 +08:00
continue // 跳过空间不足的账号
2025-07-18 00:34:27 +08:00
}
2025-07-29 17:04:49 +08:00
score := s.calculateAccountScore(account)
if score > bestScore {
bestScore = score
bestAccount = account
2025-07-18 00:34:27 +08:00
}
}
return bestAccount
}
// calculateAccountScore 计算账号评分
func (s *Scheduler) calculateAccountScore(account *entity.Cks) int64 {
// TODO: 实现账号评分算法
// 1. VIP账号加分
// 2. 剩余空间大的账号加分
// 3. 使用率低的账号加分
// 4. 可以根据历史使用情况调整评分
score := int64(0)
// VIP账号加分
if account.VipStatus {
score += 1000
}
// 剩余空间加分每GB加1分
score += account.LeftSpace / (1024 * 1024 * 1024)
// 使用率加分(使用率越低分数越高)
if account.Space > 0 {
usageRate := float64(account.UsedSpace) / float64(account.Space)
score += int64((1 - usageRate) * 500) // 使用率越低,加分越多
}
return score
}
2025-07-25 22:24:08 +08:00
// 分割标签,支持中英文逗号
func splitTags(tagStr string) []string {
tagStr = strings.ReplaceAll(tagStr, "", ",")
return strings.Split(tagStr, ",")
}
// 处理标签返回所有标签ID
func (s *Scheduler) handleTags(tagStr string) ([]uint, error) {
if tagStr == "" {
Debug("标签字符串为空,返回空数组")
return []uint{}, nil // 返回空数组而不是 nil
2025-07-25 22:24:08 +08:00
}
Debug("开始处理标签字符串: %s", tagStr)
2025-07-25 22:24:08 +08:00
tagNames := splitTags(tagStr)
Debug("分割后的标签名称: %v", tagNames)
2025-07-25 22:24:08 +08:00
var tagIDs []uint
for _, name := range tagNames {
name = strings.TrimSpace(name)
if name == "" {
Debug("跳过空标签名称")
2025-07-25 22:24:08 +08:00
continue
}
Debug("查找标签: %s", name)
2025-07-25 22:24:08 +08:00
tag, err := s.tagRepo.FindByName(name)
if err != nil {
Debug("标签 %s 不存在,创建新标签", name)
2025-07-25 22:24:08 +08:00
// 不存在则新建
tag = &entity.Tag{Name: name}
err = s.tagRepo.Create(tag)
if err != nil {
Error("创建标签 %s 失败: %v", name, err)
return nil, fmt.Errorf("创建标签 %s 失败: %v", name, err)
2025-07-25 22:24:08 +08:00
}
Debug("成功创建标签: %s (ID: %d)", name, tag.ID)
} else {
Debug("找到已存在的标签: %s (ID: %d)", name, tag.ID)
2025-07-25 22:24:08 +08:00
}
tagIDs = append(tagIDs, tag.ID)
}
Debug("处理完成标签ID列表: %v", tagIDs)
2025-07-25 22:24:08 +08:00
return tagIDs, nil
}
// 分类处理逻辑
func (s *Scheduler) resolveCategory(categoryName string, tagIDs []uint) (*uint, error) {
Debug("开始处理分类,分类名称: %s, 标签ID列表: %v", categoryName, tagIDs)
2025-07-25 22:24:08 +08:00
if categoryName != "" {
Debug("查找分类: %s", categoryName)
2025-07-25 22:24:08 +08:00
cat, err := s.categoryRepo.FindByName(categoryName)
if err != nil {
Debug("分类 %s 不存在: %v", categoryName, err)
} else if cat != nil {
Debug("找到分类: %s (ID: %d)", categoryName, cat.ID)
2025-07-25 22:24:08 +08:00
return &cat.ID, nil
}
}
2025-07-25 22:24:08 +08:00
// 没有分类,尝试用标签反查
if len(tagIDs) == 0 {
Debug("没有标签,无法通过标签反查分类")
return nil, nil
}
Debug("尝试通过标签反查分类")
2025-07-25 22:24:08 +08:00
for _, tagID := range tagIDs {
Debug("查找标签ID: %d", tagID)
2025-07-25 22:24:08 +08:00
tag, err := s.tagRepo.GetByID(tagID)
if err != nil {
Debug("查找标签ID %d 失败: %v", tagID, err)
continue
}
if tag != nil && tag.CategoryID != nil {
Debug("通过标签 %s (ID: %d) 找到分类ID: %d", tag.Name, tagID, *tag.CategoryID)
2025-07-25 22:24:08 +08:00
return tag.CategoryID, nil
}
}
Debug("未找到分类返回nil")
2025-07-25 22:24:08 +08:00
return nil, nil
}
// 工具函数解引用string指针
func derefString(s *string) string {
if s == nil {
return ""
}
return *s
}