fix: 修复部分链接检测失败的问题

This commit is contained in:
ctwj
2025-07-29 01:29:53 +08:00
parent f9ecbad0a7
commit 78b147da47
10 changed files with 420 additions and 33 deletions

View File

@@ -16,6 +16,7 @@ const (
BaiduPan
UC
NotFound
Xunlei
)
// String 返回服务类型的字符串表示
@@ -29,6 +30,8 @@ func (s ServiceType) String() string {
return "baidu"
case UC:
return "uc"
case Xunlei:
return "xunlei"
default:
return "unknown"
}
@@ -173,6 +176,7 @@ func ExtractServiceType(url string) ServiceType {
"pan.baidu.com": BaiduPan,
"drive.uc.cn": UC,
"fast.uc.cn": UC,
"pan.xunlei.com": Xunlei,
}
for pattern, serviceType := range patterns {

View File

@@ -171,17 +171,19 @@ func ToCksResponseList(cksList []entity.Cks) []dto.CksResponse {
// ToReadyResourceResponse 将ReadyResource实体转换为ReadyResourceResponse
func ToReadyResourceResponse(resource *entity.ReadyResource) dto.ReadyResourceResponse {
return dto.ReadyResourceResponse{
ID: resource.ID,
Title: resource.Title,
URL: resource.URL,
Category: resource.Category,
Tags: resource.Tags,
Img: resource.Img,
Source: resource.Source,
Extra: resource.Extra,
Key: resource.Key,
CreateTime: resource.CreateTime,
IP: resource.IP,
ID: resource.ID,
Title: resource.Title,
Description: resource.Description,
URL: resource.URL,
Category: resource.Category,
Tags: resource.Tags,
Img: resource.Img,
Source: resource.Source,
Extra: resource.Extra,
Key: resource.Key,
ErrorMsg: resource.ErrorMsg,
CreateTime: resource.CreateTime,
IP: resource.IP,
}
}

View File

@@ -10,6 +10,7 @@ type ReadyResourceRequest struct {
Img string `json:"img" example:"https://example.com/image.jpg"`
Source string `json:"source" example:"数据来源"`
Extra string `json:"extra" example:"额外信息"`
ErrorMsg string `json:"error_msg" example:"错误信息"`
}
// BatchReadyResourceRequest 批量待处理资源请求

View File

@@ -89,6 +89,7 @@ type ReadyResourceResponse struct {
Source string `json:"source"`
Extra string `json:"extra"`
Key string `json:"key"`
ErrorMsg string `json:"error_msg"`
CreateTime time.Time `json:"create_time"`
IP *string `json:"ip"`
}

View File

@@ -18,6 +18,7 @@ type ReadyResource struct {
Source string `json:"source" gorm:"size:100;comment:数据来源"`
Extra string `json:"extra" gorm:"type:text;comment:额外附加数据"`
Key string `json:"key" gorm:"size:64;index;comment:资源组标识相同key表示同一组资源"`
ErrorMsg string `json:"error_msg" gorm:"type:text;comment:处理失败时的错误信息"`
CreateTime time.Time `json:"create_time" gorm:"default:CURRENT_TIMESTAMP"`
IP *string `json:"ip" gorm:"size:45;comment:IP地址"`
CreatedAt time.Time `json:"created_at"`

View File

@@ -21,6 +21,9 @@ type ReadyResourceRepository interface {
FindAllWithinDays(days int) ([]entity.ReadyResource, error)
BatchFindByURLs(urls []string) ([]entity.ReadyResource, error)
GenerateUniqueKey() (string, error)
FindWithErrors() ([]entity.ReadyResource, error)
FindWithoutErrors() ([]entity.ReadyResource, error)
ClearErrorMsg(id uint) error
}
// ReadyResourceRepositoryImpl ReadyResource的Repository实现
@@ -113,3 +116,22 @@ func (r *ReadyResourceRepositoryImpl) GenerateUniqueKey() (string, error) {
}
return "", gorm.ErrInvalidData
}
// FindWithErrors 查找有错误信息的资源
func (r *ReadyResourceRepositoryImpl) FindWithErrors() ([]entity.ReadyResource, error) {
var resources []entity.ReadyResource
err := r.db.Where("error_msg != '' AND error_msg IS NOT NULL").Find(&resources).Error
return resources, err
}
// FindWithoutErrors 查找没有错误信息的资源
func (r *ReadyResourceRepositoryImpl) FindWithoutErrors() ([]entity.ReadyResource, error) {
var resources []entity.ReadyResource
err := r.db.Where("error_msg = '' OR error_msg IS NULL").Find(&resources).Error
return resources, err
}
// ClearErrorMsg 清除指定资源的错误信息
func (r *ReadyResourceRepositoryImpl) ClearErrorMsg(id uint) error {
return r.db.Model(&entity.ReadyResource{}).Where("id = ?", id).Update("error_msg", "").Error
}

View File

@@ -286,3 +286,182 @@ func DeleteReadyResourcesByKey(c *gin.Context) {
"message": "资源组删除成功",
})
}
// getRetryableErrorCount 统计可重试的错误数量
func getRetryableErrorCount(resources []entity.ReadyResource) int {
count := 0
for _, resource := range resources {
if resource.ErrorMsg != "" {
errorMsg := strings.ToUpper(resource.ErrorMsg)
// 检查错误类型标记
if strings.Contains(resource.ErrorMsg, "[NO_ACCOUNT]") ||
strings.Contains(resource.ErrorMsg, "[NO_VALID_ACCOUNT]") ||
strings.Contains(resource.ErrorMsg, "[TRANSFER_FAILED]") ||
strings.Contains(resource.ErrorMsg, "[LINK_CHECK_FAILED]") {
count++
} else if strings.Contains(errorMsg, "没有可用的网盘账号") ||
strings.Contains(errorMsg, "没有有效的网盘账号") ||
strings.Contains(errorMsg, "网盘信息获取失败") ||
strings.Contains(errorMsg, "链接检查失败") {
count++
}
}
}
return count
}
// GetReadyResourcesWithErrors 获取有错误信息的待处理资源
func GetReadyResourcesWithErrors(c *gin.Context) {
// 获取分页参数
pageStr := c.DefaultQuery("page", "1")
pageSizeStr := c.DefaultQuery("page_size", "100")
page, err := strconv.Atoi(pageStr)
if err != nil || page < 1 {
page = 1
}
pageSize, err := strconv.Atoi(pageSizeStr)
if err != nil || pageSize < 1 || pageSize > 1000 {
pageSize = 100
}
// 获取有错误的资源
resources, err := repoManager.ReadyResourceRepository.FindWithErrors()
if err != nil {
ErrorResponse(c, err.Error(), http.StatusInternalServerError)
return
}
responses := converter.ToReadyResourceResponseList(resources)
// 统计错误类型
errorTypeStats := make(map[string]int)
for _, resource := range resources {
if resource.ErrorMsg != "" {
// 尝试从错误信息中提取错误类型
if len(resource.ErrorMsg) > 0 && resource.ErrorMsg[0] == '[' {
endIndex := strings.Index(resource.ErrorMsg, "]")
if endIndex > 0 {
errorType := resource.ErrorMsg[1:endIndex]
errorTypeStats[errorType]++
} else {
errorTypeStats["UNKNOWN"]++
}
} else {
// 如果没有错误类型标记,尝试从错误信息中推断
errorMsg := strings.ToUpper(resource.ErrorMsg)
if strings.Contains(errorMsg, "不支持的链接") {
errorTypeStats["UNSUPPORTED_LINK"]++
} else if strings.Contains(errorMsg, "链接无效") {
errorTypeStats["INVALID_LINK"]++
} else if strings.Contains(errorMsg, "没有可用的网盘账号") {
errorTypeStats["NO_ACCOUNT"]++
} else if strings.Contains(errorMsg, "没有有效的网盘账号") {
errorTypeStats["NO_VALID_ACCOUNT"]++
} else if strings.Contains(errorMsg, "网盘信息获取失败") {
errorTypeStats["TRANSFER_FAILED"]++
} else if strings.Contains(errorMsg, "创建网盘服务失败") {
errorTypeStats["SERVICE_CREATION_FAILED"]++
} else if strings.Contains(errorMsg, "处理标签失败") {
errorTypeStats["TAG_PROCESSING_FAILED"]++
} else if strings.Contains(errorMsg, "处理分类失败") {
errorTypeStats["CATEGORY_PROCESSING_FAILED"]++
} else if strings.Contains(errorMsg, "资源保存失败") {
errorTypeStats["RESOURCE_SAVE_FAILED"]++
} else if strings.Contains(errorMsg, "未找到对应的平台ID") {
errorTypeStats["PLATFORM_NOT_FOUND"]++
} else if strings.Contains(errorMsg, "链接检查失败") {
errorTypeStats["LINK_CHECK_FAILED"]++
} else {
errorTypeStats["UNKNOWN"]++
}
}
}
}
SuccessResponse(c, gin.H{
"data": responses,
"page": page,
"page_size": pageSize,
"total": len(resources),
"count": len(resources),
"error_stats": errorTypeStats,
"retryable_count": getRetryableErrorCount(resources),
})
}
// ClearErrorMsg 清除指定资源的错误信息
func ClearErrorMsg(c *gin.Context) {
idStr := c.Param("id")
id, err := strconv.ParseUint(idStr, 10, 32)
if err != nil {
ErrorResponse(c, "无效的ID", http.StatusBadRequest)
return
}
err = repoManager.ReadyResourceRepository.ClearErrorMsg(uint(id))
if err != nil {
ErrorResponse(c, err.Error(), http.StatusInternalServerError)
return
}
SuccessResponse(c, gin.H{"message": "错误信息已清除"})
}
// RetryFailedResources 重试失败的资源
func RetryFailedResources(c *gin.Context) {
// 获取有错误的资源
resources, err := repoManager.ReadyResourceRepository.FindWithErrors()
if err != nil {
ErrorResponse(c, err.Error(), http.StatusInternalServerError)
return
}
if len(resources) == 0 {
SuccessResponse(c, gin.H{
"message": "没有需要重试的资源",
"count": 0,
})
return
}
// 只重试可重试的错误
clearedCount := 0
skippedCount := 0
for _, resource := range resources {
isRetryable := false
errorMsg := strings.ToUpper(resource.ErrorMsg)
// 检查错误类型标记
if strings.Contains(resource.ErrorMsg, "[NO_ACCOUNT]") ||
strings.Contains(resource.ErrorMsg, "[NO_VALID_ACCOUNT]") ||
strings.Contains(resource.ErrorMsg, "[TRANSFER_FAILED]") ||
strings.Contains(resource.ErrorMsg, "[LINK_CHECK_FAILED]") {
isRetryable = true
} else if strings.Contains(errorMsg, "没有可用的网盘账号") ||
strings.Contains(errorMsg, "没有有效的网盘账号") ||
strings.Contains(errorMsg, "网盘信息获取失败") ||
strings.Contains(errorMsg, "链接检查失败") {
isRetryable = true
}
if isRetryable {
if err := repoManager.ReadyResourceRepository.ClearErrorMsg(resource.ID); err == nil {
clearedCount++
}
} else {
skippedCount++
}
}
SuccessResponse(c, gin.H{
"message": "已清除可重试资源的错误信息,资源将在下次调度时重新处理",
"total_count": len(resources),
"cleared_count": clearedCount,
"skipped_count": skippedCount,
"retryable_count": getRetryableErrorCount(resources),
})
}

View File

@@ -201,6 +201,9 @@ func main() {
api.DELETE("/ready-resources", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.ClearReadyResources)
api.GET("/ready-resources/key/:key", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetReadyResourcesByKey)
api.DELETE("/ready-resources/key/:key", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.DeleteReadyResourcesByKey)
api.GET("/ready-resources/errors", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetReadyResourcesWithErrors)
api.POST("/ready-resources/:id/clear-error", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.ClearErrorMsg)
api.POST("/ready-resources/retry-failed", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.RetryFailedResources)
// 用户管理(仅管理员)
api.GET("/users", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetUsers)

153
utils/errors.go Normal file
View File

@@ -0,0 +1,153 @@
package utils
import "fmt"
// ErrorType 错误类型枚举
type ErrorType string
const (
// ErrorTypeUnsupportedLink 不支持的链接
ErrorTypeUnsupportedLink ErrorType = "UNSUPPORTED_LINK"
// ErrorTypeInvalidLink 无效链接
ErrorTypeInvalidLink ErrorType = "INVALID_LINK"
// ErrorTypeNoAccount 没有可用账号
ErrorTypeNoAccount ErrorType = "NO_ACCOUNT"
// ErrorTypeNoValidAccount 没有有效账号
ErrorTypeNoValidAccount ErrorType = "NO_VALID_ACCOUNT"
// ErrorTypeServiceCreation 服务创建失败
ErrorTypeServiceCreation ErrorType = "SERVICE_CREATION_FAILED"
// ErrorTypeTransferFailed 转存失败
ErrorTypeTransferFailed ErrorType = "TRANSFER_FAILED"
// ErrorTypeTagProcessing 标签处理失败
ErrorTypeTagProcessing ErrorType = "TAG_PROCESSING_FAILED"
// ErrorTypeCategoryProcessing 分类处理失败
ErrorTypeCategoryProcessing ErrorType = "CATEGORY_PROCESSING_FAILED"
// ErrorTypeResourceSave 资源保存失败
ErrorTypeResourceSave ErrorType = "RESOURCE_SAVE_FAILED"
// ErrorTypePlatformNotFound 平台未找到
ErrorTypePlatformNotFound ErrorType = "PLATFORM_NOT_FOUND"
// ErrorTypeLinkCheckFailed 链接检查失败
ErrorTypeLinkCheckFailed ErrorType = "LINK_CHECK_FAILED"
)
// ResourceError 资源处理错误
type ResourceError struct {
Type ErrorType `json:"type"`
Message string `json:"message"`
URL string `json:"url,omitempty"`
Details string `json:"details,omitempty"`
}
// Error 实现error接口
func (e *ResourceError) Error() string {
if e.Details != "" {
return fmt.Sprintf("[%s] %s: %s", e.Type, e.Message, e.Details)
}
return fmt.Sprintf("[%s] %s", e.Type, e.Message)
}
// NewResourceError 创建新的资源错误
func NewResourceError(errorType ErrorType, message string, url string, details string) *ResourceError {
return &ResourceError{
Type: errorType,
Message: message,
URL: url,
Details: details,
}
}
// NewUnsupportedLinkError 创建不支持的链接错误
func NewUnsupportedLinkError(url string) *ResourceError {
return NewResourceError(ErrorTypeUnsupportedLink, "不支持的链接地址", url, "")
}
// NewInvalidLinkError 创建无效链接错误
func NewInvalidLinkError(url string, details string) *ResourceError {
return NewResourceError(ErrorTypeInvalidLink, "链接无效", url, details)
}
// NewNoAccountError 创建没有账号错误
func NewNoAccountError(platform string) *ResourceError {
return NewResourceError(ErrorTypeNoAccount, "没有可用的网盘账号", "", fmt.Sprintf("平台: %s", platform))
}
// NewNoValidAccountError 创建没有有效账号错误
func NewNoValidAccountError(platform string) *ResourceError {
return NewResourceError(ErrorTypeNoValidAccount, "没有有效的网盘账号", "", fmt.Sprintf("平台: %s", platform))
}
// NewServiceCreationError 创建服务创建失败错误
func NewServiceCreationError(url string, details string) *ResourceError {
return NewResourceError(ErrorTypeServiceCreation, "创建网盘服务失败", url, details)
}
// NewTransferFailedError 创建转存失败错误
func NewTransferFailedError(url string, details string) *ResourceError {
return NewResourceError(ErrorTypeTransferFailed, "网盘信息获取失败", url, details)
}
// NewTagProcessingError 创建标签处理失败错误
func NewTagProcessingError(details string) *ResourceError {
return NewResourceError(ErrorTypeTagProcessing, "处理标签失败", "", details)
}
// NewCategoryProcessingError 创建分类处理失败错误
func NewCategoryProcessingError(details string) *ResourceError {
return NewResourceError(ErrorTypeCategoryProcessing, "处理分类失败", "", details)
}
// NewResourceSaveError 创建资源保存失败错误
func NewResourceSaveError(url string, details string) *ResourceError {
return NewResourceError(ErrorTypeResourceSave, "资源保存失败", url, details)
}
// NewPlatformNotFoundError 创建平台未找到错误
func NewPlatformNotFoundError(platform string) *ResourceError {
return NewResourceError(ErrorTypePlatformNotFound, "未找到对应的平台ID", "", fmt.Sprintf("平台: %s", platform))
}
// NewLinkCheckError 创建链接检查失败错误
func NewLinkCheckError(url string, details string) *ResourceError {
return NewResourceError(ErrorTypeLinkCheckFailed, "链接检查失败", url, details)
}
// IsResourceError 检查是否为资源错误
func IsResourceError(err error) bool {
_, ok := err.(*ResourceError)
return ok
}
// GetResourceError 获取资源错误
func GetResourceError(err error) *ResourceError {
if resourceErr, ok := err.(*ResourceError); ok {
return resourceErr
}
return nil
}
// GetErrorType 获取错误类型
func GetErrorType(err error) ErrorType {
if resourceErr := GetResourceError(err); resourceErr != nil {
return resourceErr.Type
}
return ""
}
// IsRetryableError 检查是否为可重试的错误
func IsRetryableError(err error) bool {
errorType := GetErrorType(err)
switch errorType {
case ErrorTypeNoAccount, ErrorTypeNoValidAccount, ErrorTypeTransferFailed, ErrorTypeLinkCheckFailed:
return true
default:
return false
}
}
// GetErrorSummary 获取错误摘要
func GetErrorSummary(err error) string {
if resourceErr := GetResourceError(err); resourceErr != nil {
return fmt.Sprintf("%s: %s", resourceErr.Type, resourceErr.Message)
}
return err.Error()
}

View File

@@ -352,8 +352,9 @@ func (s *Scheduler) processReadyResources() {
return
}
// 获取所有待处理资源
// 获取所有没有错误的待处理资源
readyResources, err := s.readyResourceRepo.FindAll()
// readyResources, err := s.readyResourceRepo.FindWithoutErrors()
if err != nil {
Error("获取待处理资源失败: %v", err)
return
@@ -384,10 +385,21 @@ func (s *Scheduler) processReadyResources() {
if err := s.convertReadyResourceToResource(readyResource, factory); err != nil {
Error("处理资源失败 (ID: %d): %v", readyResource.ID, err)
// 保存完整的错误信息
readyResource.ErrorMsg = err.Error()
if updateErr := s.readyResourceRepo.Update(&readyResource); updateErr != nil {
Error("更新错误信息失败 (ID: %d): %v", readyResource.ID, updateErr)
} else {
Info("已保存错误信息到资源 (ID: %d): %s", readyResource.ID, err.Error())
}
} else {
// 处理成功删除readyResource
s.readyResourceRepo.Delete(readyResource.ID)
processedCount++
Info("成功处理资源: %s", readyResource.URL)
}
s.readyResourceRepo.Delete(readyResource.ID)
processedCount++
Info("成功处理资源: %s", readyResource.URL)
}
Info("待处理资源处理完成,共处理 %d 个资源", processedCount)
@@ -401,7 +413,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
shareID, serviceType := panutils.ExtractShareId(readyResource.URL)
if serviceType == panutils.NotFound {
Warn("不支持的链接地址: %s", readyResource.URL)
return nil
return NewUnsupportedLinkError(readyResource.URL)
}
Debug("检测到服务类型: %s, 分享ID: %s", serviceType.String(), shareID)
@@ -420,24 +432,32 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
// 不是夸克,直接保存,
if serviceType != panutils.Quark {
// 检测是否有效
checkResult, _ := commonutils.CheckURL(readyResource.URL)
checkResult, err := commonutils.CheckURL(readyResource.URL)
if err != nil {
Error("链接检查失败: %v", err)
return NewLinkCheckError(readyResource.URL, err.Error())
}
if !checkResult.Status {
Warn("链接无效: %s", readyResource.URL)
return nil
return NewInvalidLinkError(readyResource.URL, "链接状态检查失败")
}
return nil
} else {
// 获取夸克网盘账号的 cookie
accounts, err := s.cksRepo.FindByPanID(*s.getPanIDByServiceType(serviceType))
panID := s.getPanIDByServiceType(serviceType)
if panID == nil {
Error("未找到对应的平台ID")
return NewPlatformNotFoundError(serviceType.String())
}
accounts, err := s.cksRepo.FindByPanID(*panID)
if err != nil {
Error("获取夸克网盘账号失败: %v", err)
return err
return NewServiceCreationError(readyResource.URL, fmt.Sprintf("获取网盘账号失败: %v", err))
}
if len(accounts) == 0 {
Error("没有可用的夸克网盘账号")
return fmt.Errorf("没有可用的夸克网盘账号")
return NewNoAccountError(serviceType.String())
}
// 选择第一个有效的账号
@@ -451,7 +471,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
if selectedAccount == nil {
Error("没有有效的夸克网盘账号")
return fmt.Errorf("没有有效的夸克网盘账号")
return NewNoValidAccountError(serviceType.String())
}
Debug("使用夸克网盘账号: %d, Cookie: %s", selectedAccount.ID, selectedAccount.Ck[:20]+"...")
@@ -471,32 +491,32 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
panService, err := factory.CreatePanService(readyResource.URL, config)
if err != nil {
Error("获取网盘服务失败: %v", err)
return err
return NewServiceCreationError(readyResource.URL, err.Error())
}
// 统一处理:尝试转存获取标题
result, err := panService.Transfer(shareID)
if err != nil {
Error("网盘信息获取失败: %v", err)
return err
return NewTransferFailedError(readyResource.URL, err.Error())
}
if !result.Success {
Error("网盘信息获取失败: %s", result.Message)
return nil
return NewTransferFailedError(readyResource.URL, result.Message)
}
// 如果获取到了标题,更新资源标题
if result.Title != "" {
resource.Title = result.Title
}
// if result.Title != "" {
// resource.Title = result.Title
// }
}
// 处理标签
tagIDs, err := s.handleTags(readyResource.Tags)
if err != nil {
Error("处理标签失败: %v", err)
return err
return NewTagProcessingError(err.Error())
}
// 如果没有标签tagIDs 可能为 nil这是正常的
if tagIDs == nil {
@@ -506,7 +526,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
categoryID, err := s.resolveCategory(readyResource.Category, tagIDs)
if err != nil {
Error("处理分类失败: %v", err)
return err
return NewCategoryProcessingError(err.Error())
}
if categoryID != nil {
resource.CategoryID = categoryID
@@ -515,7 +535,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
err = s.resourceRepo.Create(resource)
if err != nil {
Error("资源保存失败: %v", err)
return err
return NewResourceSaveError(readyResource.URL, err.Error())
}
// 插入 resource_tags 关联
if len(tagIDs) > 0 {
@@ -523,6 +543,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
err := s.resourceRepo.CreateResourceTag(resource.ID, tagID)
if err != nil {
Error("插入资源标签关联失败: %v", err)
// 这里不返回错误,因为资源已经保存成功,标签关联失败不影响主要功能
}
}
} else {