Files
urldb/task/transfer_processor.go

474 lines
14 KiB
Go
Raw Normal View History

2025-08-09 23:47:30 +08:00
package task
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
2025-08-10 13:52:41 +08:00
pan "github.com/ctwj/urldb/common"
2025-08-09 23:47:30 +08:00
"github.com/ctwj/urldb/db/entity"
"github.com/ctwj/urldb/db/repo"
"github.com/ctwj/urldb/utils"
)
// TransferProcessor 转存任务处理器
type TransferProcessor struct {
repoMgr *repo.RepositoryManager
}
// NewTransferProcessor 创建转存任务处理器
func NewTransferProcessor(repoMgr *repo.RepositoryManager) *TransferProcessor {
return &TransferProcessor{
repoMgr: repoMgr,
}
}
// GetTaskType 获取任务类型
func (tp *TransferProcessor) GetTaskType() string {
return "transfer"
}
// TransferInput 转存任务输入数据结构
type TransferInput struct {
Title string `json:"title"`
URL string `json:"url"`
CategoryID uint `json:"category_id"`
2025-08-12 00:27:10 +08:00
PanID uint `json:"pan_id"`
2025-08-09 23:47:30 +08:00
Tags []uint `json:"tags"`
}
// TransferOutput 转存任务输出数据结构
type TransferOutput struct {
ResourceID uint `json:"resource_id,omitempty"`
SaveURL string `json:"save_url,omitempty"`
Error string `json:"error,omitempty"`
Success bool `json:"success"`
Time string `json:"time"`
}
// Process 处理转存任务项
func (tp *TransferProcessor) Process(ctx context.Context, taskID uint, item *entity.TaskItem) error {
2025-10-28 11:07:00 +08:00
startTime := utils.GetCurrentTime()
utils.InfoWithFields(map[string]interface{}{
"task_item_id": item.ID,
"task_id": taskID,
}, "开始处理转存任务项: %d", item.ID)
2025-08-09 23:47:30 +08:00
// 解析输入数据
2025-10-28 11:07:00 +08:00
parseStart := utils.GetCurrentTime()
2025-08-09 23:47:30 +08:00
var input TransferInput
if err := json.Unmarshal([]byte(item.InputData), &input); err != nil {
2025-10-28 11:07:00 +08:00
parseDuration := time.Since(parseStart)
utils.ErrorWithFields(map[string]interface{}{
"error": err.Error(),
"duration_ms": parseDuration.Milliseconds(),
}, "解析输入数据失败: %v耗时: %v", err, parseDuration)
2025-08-09 23:47:30 +08:00
return fmt.Errorf("解析输入数据失败: %v", err)
}
2025-10-28 11:07:00 +08:00
parseDuration := time.Since(parseStart)
utils.DebugWithFields(map[string]interface{}{
"duration_ms": parseDuration.Milliseconds(),
}, "解析输入数据完成,耗时: %v", parseDuration)
2025-08-09 23:47:30 +08:00
// 验证输入数据
2025-10-28 11:07:00 +08:00
validateStart := utils.GetCurrentTime()
2025-08-09 23:47:30 +08:00
if err := tp.validateInput(&input); err != nil {
2025-10-28 11:07:00 +08:00
validateDuration := time.Since(validateStart)
utils.Error("输入数据验证失败: %v耗时: %v", err, validateDuration)
2025-08-09 23:47:30 +08:00
return fmt.Errorf("输入数据验证失败: %v", err)
}
2025-10-28 11:07:00 +08:00
validateDuration := time.Since(validateStart)
utils.DebugWithFields(map[string]interface{}{
"duration_ms": validateDuration.Milliseconds(),
}, "输入数据验证完成,耗时: %v", validateDuration)
2025-08-09 23:47:30 +08:00
2025-08-12 00:27:10 +08:00
// 获取任务配置中的账号信息
2025-10-28 11:07:00 +08:00
configStart := utils.GetCurrentTime()
2025-08-12 00:27:10 +08:00
var selectedAccounts []uint
task, err := tp.repoMgr.TaskRepository.GetByID(taskID)
if err == nil && task.Config != "" {
var taskConfig map[string]interface{}
if err := json.Unmarshal([]byte(task.Config), &taskConfig); err == nil {
if accounts, ok := taskConfig["selected_accounts"].([]interface{}); ok {
for _, acc := range accounts {
if accID, ok := acc.(float64); ok {
selectedAccounts = append(selectedAccounts, uint(accID))
}
}
}
}
}
2025-10-28 11:07:00 +08:00
configDuration := time.Since(configStart)
utils.Debug("获取任务配置完成,耗时: %v", configDuration)
2025-08-12 00:27:10 +08:00
2025-09-05 01:28:24 +08:00
if len(selectedAccounts) == 0 {
utils.Error("失败: %v", "没有指定转存账号")
}
2025-08-09 23:47:30 +08:00
// 检查资源是否已存在
2025-10-28 11:07:00 +08:00
checkStart := utils.GetCurrentTime()
2025-08-09 23:47:30 +08:00
exists, existingResource, err := tp.checkResourceExists(input.URL)
2025-10-28 11:07:00 +08:00
checkDuration := time.Since(checkStart)
2025-08-09 23:47:30 +08:00
if err != nil {
2025-10-28 11:07:00 +08:00
utils.Error("检查资源是否存在失败: %v耗时: %v", err, checkDuration)
} else {
utils.Debug("检查资源是否存在完成,耗时: %v", checkDuration)
2025-08-09 23:47:30 +08:00
}
if exists {
2025-08-10 13:52:41 +08:00
// 检查已存在的资源是否有有效的转存链接
if existingResource.SaveURL == "" {
// 资源存在但没有转存链接,需要重新转存
utils.Info("资源已存在但无转存链接,重新转存: %s", input.Title)
} else {
// 资源已存在且有转存链接,跳过转存
output := TransferOutput{
ResourceID: existingResource.ID,
SaveURL: existingResource.SaveURL,
Success: true,
2025-08-11 01:34:07 +08:00
Time: utils.GetCurrentTimeString(),
2025-08-10 13:52:41 +08:00
}
outputJSON, _ := json.Marshal(output)
item.OutputData = string(outputJSON)
2025-10-28 11:07:00 +08:00
elapsedTime := time.Since(startTime)
utils.Info("资源已存在且有转存链接,跳过转存: %s总耗时: %v", input.Title, elapsedTime)
2025-08-10 13:52:41 +08:00
return nil
}
}
2025-09-05 01:28:24 +08:00
// 查询出 账号列表
2025-10-28 11:07:00 +08:00
cksStart := utils.GetCurrentTime()
2025-09-05 01:28:24 +08:00
cks, err := tp.repoMgr.CksRepository.FindByIds(selectedAccounts)
2025-10-28 11:07:00 +08:00
cksDuration := time.Since(cksStart)
2025-09-05 01:28:24 +08:00
if err != nil {
2025-10-28 11:07:00 +08:00
utils.Error("读取账号失败: %v耗时: %v", err, cksDuration)
} else {
utils.Debug("读取账号完成,账号数量: %d耗时: %v", len(cks), cksDuration)
2025-09-05 01:28:24 +08:00
}
2025-08-10 13:52:41 +08:00
// 执行转存操作
2025-10-28 11:07:00 +08:00
transferStart := utils.GetCurrentTime()
2025-09-05 01:28:24 +08:00
resourceID, saveURL, err := tp.performTransfer(ctx, &input, cks)
2025-10-28 11:07:00 +08:00
transferDuration := time.Since(transferStart)
2025-08-10 13:52:41 +08:00
if err != nil {
// 转存失败,更新输出数据
2025-08-09 23:47:30 +08:00
output := TransferOutput{
2025-08-10 13:52:41 +08:00
Error: err.Error(),
Success: false,
2025-08-11 01:34:07 +08:00
Time: utils.GetCurrentTimeString(),
2025-08-09 23:47:30 +08:00
}
outputJSON, _ := json.Marshal(output)
item.OutputData = string(outputJSON)
2025-10-28 11:07:00 +08:00
elapsedTime := time.Since(startTime)
utils.ErrorWithFields(map[string]interface{}{
"task_item_id": item.ID,
"error": err.Error(),
"duration_ms": transferDuration.Milliseconds(),
"total_ms": elapsedTime.Milliseconds(),
}, "转存任务项处理失败: %d, 错误: %v转存耗时: %v总耗时: %v", item.ID, err, transferDuration, elapsedTime)
2025-08-10 13:52:41 +08:00
return fmt.Errorf("转存失败: %v", err)
2025-08-09 23:47:30 +08:00
}
2025-08-10 13:52:41 +08:00
// 验证转存结果
if saveURL == "" {
output := TransferOutput{
Error: "转存成功但未获取到分享链接",
Success: false,
2025-08-11 01:34:07 +08:00
Time: utils.GetCurrentTimeString(),
2025-08-10 13:52:41 +08:00
}
outputJSON, _ := json.Marshal(output)
item.OutputData = string(outputJSON)
2025-10-28 11:07:00 +08:00
elapsedTime := time.Since(startTime)
utils.Error("转存任务项处理失败: %d, 未获取到分享链接,总耗时: %v", item.ID, elapsedTime)
2025-08-10 13:52:41 +08:00
return fmt.Errorf("转存成功但未获取到分享链接")
2025-08-09 23:47:30 +08:00
}
2025-08-10 13:52:41 +08:00
// 转存成功,更新输出数据
2025-08-09 23:47:30 +08:00
output := TransferOutput{
ResourceID: resourceID,
SaveURL: saveURL,
Success: true,
2025-08-11 01:34:07 +08:00
Time: utils.GetCurrentTimeString(),
2025-08-09 23:47:30 +08:00
}
outputJSON, _ := json.Marshal(output)
item.OutputData = string(outputJSON)
2025-10-28 11:07:00 +08:00
elapsedTime := time.Since(startTime)
utils.InfoWithFields(map[string]interface{}{
"task_item_id": item.ID,
"resource_id": resourceID,
"save_url": saveURL,
"transfer_duration_ms": transferDuration.Milliseconds(),
"total_duration_ms": elapsedTime.Milliseconds(),
}, "转存任务项处理完成: %d, 资源ID: %d, 转存链接: %s转存耗时: %v总耗时: %v", item.ID, resourceID, saveURL, transferDuration, elapsedTime)
2025-08-09 23:47:30 +08:00
return nil
}
// validateInput 验证输入数据
func (tp *TransferProcessor) validateInput(input *TransferInput) error {
if strings.TrimSpace(input.Title) == "" {
return fmt.Errorf("标题不能为空")
}
if strings.TrimSpace(input.URL) == "" {
return fmt.Errorf("链接不能为空")
}
// 验证URL格式
if !tp.isValidURL(input.URL) {
return fmt.Errorf("链接格式不正确")
}
return nil
}
// isValidURL 验证URL格式
func (tp *TransferProcessor) isValidURL(url string) bool {
2025-09-05 01:28:24 +08:00
patterns := []string{
`https://pan\.quark\.cn/s/[a-zA-Z0-9]+`, // 夸克网盘
`https://pan\.xunlei\.com/s/.+`, // 迅雷网盘
}
for _, pattern := range patterns {
matched, _ := regexp.MatchString(pattern, url)
if matched {
return true
}
}
return false
2025-08-09 23:47:30 +08:00
}
// checkResourceExists 检查资源是否已存在
func (tp *TransferProcessor) checkResourceExists(url string) (bool, *entity.Resource, error) {
// 根据URL查找资源
resource, err := tp.repoMgr.ResourceRepository.GetByURL(url)
if err != nil {
// 如果是未找到记录的错误,则表示资源不存在
if strings.Contains(err.Error(), "record not found") {
return false, nil, nil
}
return false, nil, err
}
return true, resource, nil
}
// performTransfer 执行转存操作
2025-09-05 01:28:24 +08:00
func (tp *TransferProcessor) performTransfer(ctx context.Context, input *TransferInput, cks []*entity.Cks) (uint, string, error) {
// 从 cks 中,挑选出,能够转存的账号,
urlType := pan.ExtractServiceType(input.URL)
if urlType == pan.NotFound {
return 0, "", fmt.Errorf("未识别资源类型: %v", input.URL)
}
serviceType := ""
switch urlType {
case pan.Quark:
serviceType = "quark"
case pan.Xunlei:
serviceType = "xunlei"
default:
serviceType = ""
}
var account *entity.Cks
for _, ck := range cks {
if ck.ServiceType == serviceType {
account = ck
}
}
if account == nil {
return 0, "", fmt.Errorf("为找到匹配的账号: %v", serviceType)
2025-08-09 23:47:30 +08:00
}
2025-08-10 13:52:41 +08:00
// 先执行转存操作
2025-09-05 01:28:24 +08:00
saveData, err := tp.transferToCloud(ctx, input.URL, account)
2025-08-10 13:52:41 +08:00
if err != nil {
utils.Error("云端转存失败: %v", err)
return 0, "", fmt.Errorf("转存失败: %v", err)
}
// 验证转存链接是否有效
2025-09-05 01:28:24 +08:00
if saveData.SaveURL == "" {
2025-08-10 13:52:41 +08:00
utils.Error("转存成功但未获取到分享链接")
return 0, "", fmt.Errorf("转存成功但未获取到分享链接")
}
// 转存成功,创建资源记录
2025-08-09 23:47:30 +08:00
var categoryID *uint
if input.CategoryID != 0 {
categoryID = &input.CategoryID
}
2025-09-05 01:28:24 +08:00
// 确定平台ID 根据 serviceType 确认 panId
panID, _ := tp.repoMgr.PanRepository.FindIdByServiceType(serviceType)
panIdInt := uint(panID)
2025-08-12 00:27:10 +08:00
2025-08-09 23:47:30 +08:00
resource := &entity.Resource{
Title: input.Title,
URL: input.URL,
CategoryID: categoryID,
2025-09-05 01:28:24 +08:00
PanID: &panIdInt, // 设置平台ID
SaveURL: saveData.SaveURL, // 直接设置转存链接
2025-08-09 23:47:30 +08:00
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
// 保存资源到数据库
err = tp.repoMgr.ResourceRepository.Create(resource)
if err != nil {
2025-08-10 13:52:41 +08:00
utils.Error("保存转存成功的资源失败: %v", err)
2025-08-09 23:47:30 +08:00
return 0, "", fmt.Errorf("保存资源失败: %v", err)
}
// 添加标签关联
if len(input.Tags) > 0 {
err = tp.addResourceTags(resource.ID, input.Tags)
if err != nil {
utils.Error("添加资源标签失败: %v", err)
2025-08-10 13:52:41 +08:00
// 标签添加失败不影响资源创建,只记录错误
2025-08-09 23:47:30 +08:00
}
}
2025-09-05 01:28:24 +08:00
utils.Info("转存成功,资源已创建 - 资源ID: %d, 转存链接: %s", resource.ID, saveData.SaveURL)
return resource.ID, saveData.SaveURL, nil
2025-08-09 23:47:30 +08:00
}
// ShareInfo 分享信息结构
type ShareInfo struct {
PanType string
ShareID string
URL string
}
2025-09-05 01:28:24 +08:00
// // parseShareURL 解析分享链接
// func (tp *TransferProcessor) parseShareURL(url string) (*ShareInfo, error) {
// // 解析夸克网盘链接
// quarkPattern := `https://pan\.quark\.cn/s/([a-zA-Z0-9]+)`
// re := regexp.MustCompile(quarkPattern)
// matches := re.FindStringSubmatch(url)
2025-08-09 23:47:30 +08:00
2025-09-05 01:28:24 +08:00
// if len(matches) >= 2 {
// return &ShareInfo{
// PanType: "quark",
// ShareID: matches[1],
// URL: url,
// }, nil
// }
2025-08-09 23:47:30 +08:00
2025-09-05 01:28:24 +08:00
// return nil, fmt.Errorf("不支持的分享链接格式: %s", url)
// }
2025-08-09 23:47:30 +08:00
// addResourceTags 添加资源标签
func (tp *TransferProcessor) addResourceTags(resourceID uint, tagIDs []uint) error {
for _, tagID := range tagIDs {
// 创建资源标签关联
resourceTag := &entity.ResourceTag{
ResourceID: resourceID,
TagID: tagID,
}
err := tp.repoMgr.ResourceRepository.CreateResourceTag(resourceTag)
if err != nil {
return fmt.Errorf("创建资源标签关联失败: %v", err)
}
}
return nil
}
// transferToCloud 执行云端转存
2025-09-05 01:28:24 +08:00
func (tp *TransferProcessor) transferToCloud(ctx context.Context, url string, account *entity.Cks) (*TransferResult, error) {
2025-08-12 00:27:10 +08:00
2025-09-05 01:28:24 +08:00
// 创建网盘服务工厂
factory := pan.NewPanFactory()
2025-08-12 00:27:10 +08:00
2025-09-05 01:28:24 +08:00
service, err := factory.CreatePanService(url, &pan.PanConfig{
URL: url,
ExpiredType: 0,
IsType: 0,
Cookie: account.Ck,
})
service.SetCKSRepository(tp.repoMgr.CksRepository, *account)
2025-08-12 00:27:10 +08:00
2025-09-05 01:28:24 +08:00
// 提取分享ID
shareID, _ := pan.ExtractShareId(url)
2025-08-12 00:27:10 +08:00
2025-09-05 01:28:24 +08:00
// 执行转存
transferResult, err := service.Transfer(shareID) // 有些链接还需要其他信息从 url 中自行解析
2025-08-10 13:52:41 +08:00
if err != nil {
2025-09-05 01:28:24 +08:00
utils.Error("转存失败: %v", err)
return nil, fmt.Errorf("转存失败: %v", err)
2025-08-10 13:52:41 +08:00
}
2025-09-05 01:28:24 +08:00
if transferResult == nil || !transferResult.Success {
errMsg := "转存失败"
if transferResult != nil && transferResult.Message != "" {
errMsg = transferResult.Message
}
return nil, fmt.Errorf("转存失败: %v", errMsg)
2025-08-10 13:52:41 +08:00
}
2025-09-05 01:28:24 +08:00
// 提取转存链接
var saveURL string
var fid string
2025-08-10 13:52:41 +08:00
2025-09-05 01:28:24 +08:00
if data, ok := transferResult.Data.(map[string]interface{}); ok {
if v, ok := data["shareUrl"]; ok {
saveURL, _ = v.(string)
}
if v, ok := data["fid"]; ok {
fid, _ = v.(string)
2025-08-10 13:52:41 +08:00
}
2025-08-09 23:47:30 +08:00
}
2025-09-05 01:28:24 +08:00
if saveURL == "" {
saveURL = transferResult.ShareURL
2025-08-10 13:52:41 +08:00
}
2025-09-05 01:28:24 +08:00
if saveURL == "" {
return nil, fmt.Errorf("转存失败: %v", "转存成功但未获取到分享链接")
}
2025-08-10 13:52:41 +08:00
2025-09-05 01:28:24 +08:00
utils.Info("转存成功 - 资源ID: %d, 转存链接: %s", transferResult.Fid, saveURL)
2025-08-10 13:52:41 +08:00
2025-09-05 01:28:24 +08:00
return &TransferResult{
Success: true,
SaveURL: saveURL,
Fid: fid,
}, nil
2025-08-10 13:52:41 +08:00
}
// getQuarkPanID 获取夸克网盘ID
func (tp *TransferProcessor) getQuarkPanID() (uint, error) {
// 通过FindAll方法查找所有平台然后过滤出quark平台
pans, err := tp.repoMgr.PanRepository.FindAll()
if err != nil {
return 0, fmt.Errorf("查询平台信息失败: %v", err)
}
for _, p := range pans {
if p.Name == "quark" {
return p.ID, nil
}
}
return 0, fmt.Errorf("未找到quark平台")
}
// TransferResult 转存结果
type TransferResult struct {
Success bool `json:"success"`
SaveURL string `json:"save_url"`
2025-09-05 01:28:24 +08:00
Fid string `json:"fid`
2025-08-10 13:52:41 +08:00
ErrorMsg string `json:"error_msg"`
}