mirror of
https://github.com/ctwj/urldb.git
synced 2025-11-26 03:44:55 +08:00
fix: 修复部分链接检测失败的问题
This commit is contained in:
@@ -16,6 +16,7 @@ const (
|
|||||||
BaiduPan
|
BaiduPan
|
||||||
UC
|
UC
|
||||||
NotFound
|
NotFound
|
||||||
|
Xunlei
|
||||||
)
|
)
|
||||||
|
|
||||||
// String 返回服务类型的字符串表示
|
// String 返回服务类型的字符串表示
|
||||||
@@ -29,6 +30,8 @@ func (s ServiceType) String() string {
|
|||||||
return "baidu"
|
return "baidu"
|
||||||
case UC:
|
case UC:
|
||||||
return "uc"
|
return "uc"
|
||||||
|
case Xunlei:
|
||||||
|
return "xunlei"
|
||||||
default:
|
default:
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
@@ -173,6 +176,7 @@ func ExtractServiceType(url string) ServiceType {
|
|||||||
"pan.baidu.com": BaiduPan,
|
"pan.baidu.com": BaiduPan,
|
||||||
"drive.uc.cn": UC,
|
"drive.uc.cn": UC,
|
||||||
"fast.uc.cn": UC,
|
"fast.uc.cn": UC,
|
||||||
|
"pan.xunlei.com": Xunlei,
|
||||||
}
|
}
|
||||||
|
|
||||||
for pattern, serviceType := range patterns {
|
for pattern, serviceType := range patterns {
|
||||||
|
|||||||
@@ -173,6 +173,7 @@ func ToReadyResourceResponse(resource *entity.ReadyResource) dto.ReadyResourceRe
|
|||||||
return dto.ReadyResourceResponse{
|
return dto.ReadyResourceResponse{
|
||||||
ID: resource.ID,
|
ID: resource.ID,
|
||||||
Title: resource.Title,
|
Title: resource.Title,
|
||||||
|
Description: resource.Description,
|
||||||
URL: resource.URL,
|
URL: resource.URL,
|
||||||
Category: resource.Category,
|
Category: resource.Category,
|
||||||
Tags: resource.Tags,
|
Tags: resource.Tags,
|
||||||
@@ -180,6 +181,7 @@ func ToReadyResourceResponse(resource *entity.ReadyResource) dto.ReadyResourceRe
|
|||||||
Source: resource.Source,
|
Source: resource.Source,
|
||||||
Extra: resource.Extra,
|
Extra: resource.Extra,
|
||||||
Key: resource.Key,
|
Key: resource.Key,
|
||||||
|
ErrorMsg: resource.ErrorMsg,
|
||||||
CreateTime: resource.CreateTime,
|
CreateTime: resource.CreateTime,
|
||||||
IP: resource.IP,
|
IP: resource.IP,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ type ReadyResourceRequest struct {
|
|||||||
Img string `json:"img" example:"https://example.com/image.jpg"`
|
Img string `json:"img" example:"https://example.com/image.jpg"`
|
||||||
Source string `json:"source" example:"数据来源"`
|
Source string `json:"source" example:"数据来源"`
|
||||||
Extra string `json:"extra" example:"额外信息"`
|
Extra string `json:"extra" example:"额外信息"`
|
||||||
|
ErrorMsg string `json:"error_msg" example:"错误信息"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchReadyResourceRequest 批量待处理资源请求
|
// BatchReadyResourceRequest 批量待处理资源请求
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ type ReadyResourceResponse struct {
|
|||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
Extra string `json:"extra"`
|
Extra string `json:"extra"`
|
||||||
Key string `json:"key"`
|
Key string `json:"key"`
|
||||||
|
ErrorMsg string `json:"error_msg"`
|
||||||
CreateTime time.Time `json:"create_time"`
|
CreateTime time.Time `json:"create_time"`
|
||||||
IP *string `json:"ip"`
|
IP *string `json:"ip"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ type ReadyResource struct {
|
|||||||
Source string `json:"source" gorm:"size:100;comment:数据来源"`
|
Source string `json:"source" gorm:"size:100;comment:数据来源"`
|
||||||
Extra string `json:"extra" gorm:"type:text;comment:额外附加数据"`
|
Extra string `json:"extra" gorm:"type:text;comment:额外附加数据"`
|
||||||
Key string `json:"key" gorm:"size:64;index;comment:资源组标识,相同key表示同一组资源"`
|
Key string `json:"key" gorm:"size:64;index;comment:资源组标识,相同key表示同一组资源"`
|
||||||
|
ErrorMsg string `json:"error_msg" gorm:"type:text;comment:处理失败时的错误信息"`
|
||||||
CreateTime time.Time `json:"create_time" gorm:"default:CURRENT_TIMESTAMP"`
|
CreateTime time.Time `json:"create_time" gorm:"default:CURRENT_TIMESTAMP"`
|
||||||
IP *string `json:"ip" gorm:"size:45;comment:IP地址"`
|
IP *string `json:"ip" gorm:"size:45;comment:IP地址"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
|||||||
@@ -21,6 +21,9 @@ type ReadyResourceRepository interface {
|
|||||||
FindAllWithinDays(days int) ([]entity.ReadyResource, error)
|
FindAllWithinDays(days int) ([]entity.ReadyResource, error)
|
||||||
BatchFindByURLs(urls []string) ([]entity.ReadyResource, error)
|
BatchFindByURLs(urls []string) ([]entity.ReadyResource, error)
|
||||||
GenerateUniqueKey() (string, error)
|
GenerateUniqueKey() (string, error)
|
||||||
|
FindWithErrors() ([]entity.ReadyResource, error)
|
||||||
|
FindWithoutErrors() ([]entity.ReadyResource, error)
|
||||||
|
ClearErrorMsg(id uint) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadyResourceRepositoryImpl ReadyResource的Repository实现
|
// ReadyResourceRepositoryImpl ReadyResource的Repository实现
|
||||||
@@ -113,3 +116,22 @@ func (r *ReadyResourceRepositoryImpl) GenerateUniqueKey() (string, error) {
|
|||||||
}
|
}
|
||||||
return "", gorm.ErrInvalidData
|
return "", gorm.ErrInvalidData
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FindWithErrors 查找有错误信息的资源
|
||||||
|
func (r *ReadyResourceRepositoryImpl) FindWithErrors() ([]entity.ReadyResource, error) {
|
||||||
|
var resources []entity.ReadyResource
|
||||||
|
err := r.db.Where("error_msg != '' AND error_msg IS NOT NULL").Find(&resources).Error
|
||||||
|
return resources, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindWithoutErrors 查找没有错误信息的资源
|
||||||
|
func (r *ReadyResourceRepositoryImpl) FindWithoutErrors() ([]entity.ReadyResource, error) {
|
||||||
|
var resources []entity.ReadyResource
|
||||||
|
err := r.db.Where("error_msg = '' OR error_msg IS NULL").Find(&resources).Error
|
||||||
|
return resources, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearErrorMsg 清除指定资源的错误信息
|
||||||
|
func (r *ReadyResourceRepositoryImpl) ClearErrorMsg(id uint) error {
|
||||||
|
return r.db.Model(&entity.ReadyResource{}).Where("id = ?", id).Update("error_msg", "").Error
|
||||||
|
}
|
||||||
|
|||||||
@@ -286,3 +286,182 @@ func DeleteReadyResourcesByKey(c *gin.Context) {
|
|||||||
"message": "资源组删除成功",
|
"message": "资源组删除成功",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getRetryableErrorCount 统计可重试的错误数量
|
||||||
|
func getRetryableErrorCount(resources []entity.ReadyResource) int {
|
||||||
|
count := 0
|
||||||
|
|
||||||
|
for _, resource := range resources {
|
||||||
|
if resource.ErrorMsg != "" {
|
||||||
|
errorMsg := strings.ToUpper(resource.ErrorMsg)
|
||||||
|
// 检查错误类型标记
|
||||||
|
if strings.Contains(resource.ErrorMsg, "[NO_ACCOUNT]") ||
|
||||||
|
strings.Contains(resource.ErrorMsg, "[NO_VALID_ACCOUNT]") ||
|
||||||
|
strings.Contains(resource.ErrorMsg, "[TRANSFER_FAILED]") ||
|
||||||
|
strings.Contains(resource.ErrorMsg, "[LINK_CHECK_FAILED]") {
|
||||||
|
count++
|
||||||
|
} else if strings.Contains(errorMsg, "没有可用的网盘账号") ||
|
||||||
|
strings.Contains(errorMsg, "没有有效的网盘账号") ||
|
||||||
|
strings.Contains(errorMsg, "网盘信息获取失败") ||
|
||||||
|
strings.Contains(errorMsg, "链接检查失败") {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetReadyResourcesWithErrors 获取有错误信息的待处理资源
|
||||||
|
func GetReadyResourcesWithErrors(c *gin.Context) {
|
||||||
|
// 获取分页参数
|
||||||
|
pageStr := c.DefaultQuery("page", "1")
|
||||||
|
pageSizeStr := c.DefaultQuery("page_size", "100")
|
||||||
|
|
||||||
|
page, err := strconv.Atoi(pageStr)
|
||||||
|
if err != nil || page < 1 {
|
||||||
|
page = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
pageSize, err := strconv.Atoi(pageSizeStr)
|
||||||
|
if err != nil || pageSize < 1 || pageSize > 1000 {
|
||||||
|
pageSize = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
// 获取有错误的资源
|
||||||
|
resources, err := repoManager.ReadyResourceRepository.FindWithErrors()
|
||||||
|
if err != nil {
|
||||||
|
ErrorResponse(c, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
responses := converter.ToReadyResourceResponseList(resources)
|
||||||
|
|
||||||
|
// 统计错误类型
|
||||||
|
errorTypeStats := make(map[string]int)
|
||||||
|
for _, resource := range resources {
|
||||||
|
if resource.ErrorMsg != "" {
|
||||||
|
// 尝试从错误信息中提取错误类型
|
||||||
|
if len(resource.ErrorMsg) > 0 && resource.ErrorMsg[0] == '[' {
|
||||||
|
endIndex := strings.Index(resource.ErrorMsg, "]")
|
||||||
|
if endIndex > 0 {
|
||||||
|
errorType := resource.ErrorMsg[1:endIndex]
|
||||||
|
errorTypeStats[errorType]++
|
||||||
|
} else {
|
||||||
|
errorTypeStats["UNKNOWN"]++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// 如果没有错误类型标记,尝试从错误信息中推断
|
||||||
|
errorMsg := strings.ToUpper(resource.ErrorMsg)
|
||||||
|
if strings.Contains(errorMsg, "不支持的链接") {
|
||||||
|
errorTypeStats["UNSUPPORTED_LINK"]++
|
||||||
|
} else if strings.Contains(errorMsg, "链接无效") {
|
||||||
|
errorTypeStats["INVALID_LINK"]++
|
||||||
|
} else if strings.Contains(errorMsg, "没有可用的网盘账号") {
|
||||||
|
errorTypeStats["NO_ACCOUNT"]++
|
||||||
|
} else if strings.Contains(errorMsg, "没有有效的网盘账号") {
|
||||||
|
errorTypeStats["NO_VALID_ACCOUNT"]++
|
||||||
|
} else if strings.Contains(errorMsg, "网盘信息获取失败") {
|
||||||
|
errorTypeStats["TRANSFER_FAILED"]++
|
||||||
|
} else if strings.Contains(errorMsg, "创建网盘服务失败") {
|
||||||
|
errorTypeStats["SERVICE_CREATION_FAILED"]++
|
||||||
|
} else if strings.Contains(errorMsg, "处理标签失败") {
|
||||||
|
errorTypeStats["TAG_PROCESSING_FAILED"]++
|
||||||
|
} else if strings.Contains(errorMsg, "处理分类失败") {
|
||||||
|
errorTypeStats["CATEGORY_PROCESSING_FAILED"]++
|
||||||
|
} else if strings.Contains(errorMsg, "资源保存失败") {
|
||||||
|
errorTypeStats["RESOURCE_SAVE_FAILED"]++
|
||||||
|
} else if strings.Contains(errorMsg, "未找到对应的平台ID") {
|
||||||
|
errorTypeStats["PLATFORM_NOT_FOUND"]++
|
||||||
|
} else if strings.Contains(errorMsg, "链接检查失败") {
|
||||||
|
errorTypeStats["LINK_CHECK_FAILED"]++
|
||||||
|
} else {
|
||||||
|
errorTypeStats["UNKNOWN"]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SuccessResponse(c, gin.H{
|
||||||
|
"data": responses,
|
||||||
|
"page": page,
|
||||||
|
"page_size": pageSize,
|
||||||
|
"total": len(resources),
|
||||||
|
"count": len(resources),
|
||||||
|
"error_stats": errorTypeStats,
|
||||||
|
"retryable_count": getRetryableErrorCount(resources),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearErrorMsg 清除指定资源的错误信息
|
||||||
|
func ClearErrorMsg(c *gin.Context) {
|
||||||
|
idStr := c.Param("id")
|
||||||
|
id, err := strconv.ParseUint(idStr, 10, 32)
|
||||||
|
if err != nil {
|
||||||
|
ErrorResponse(c, "无效的ID", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
err = repoManager.ReadyResourceRepository.ClearErrorMsg(uint(id))
|
||||||
|
if err != nil {
|
||||||
|
ErrorResponse(c, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
SuccessResponse(c, gin.H{"message": "错误信息已清除"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetryFailedResources 重试失败的资源
|
||||||
|
func RetryFailedResources(c *gin.Context) {
|
||||||
|
// 获取有错误的资源
|
||||||
|
resources, err := repoManager.ReadyResourceRepository.FindWithErrors()
|
||||||
|
if err != nil {
|
||||||
|
ErrorResponse(c, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(resources) == 0 {
|
||||||
|
SuccessResponse(c, gin.H{
|
||||||
|
"message": "没有需要重试的资源",
|
||||||
|
"count": 0,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 只重试可重试的错误
|
||||||
|
clearedCount := 0
|
||||||
|
skippedCount := 0
|
||||||
|
|
||||||
|
for _, resource := range resources {
|
||||||
|
isRetryable := false
|
||||||
|
errorMsg := strings.ToUpper(resource.ErrorMsg)
|
||||||
|
|
||||||
|
// 检查错误类型标记
|
||||||
|
if strings.Contains(resource.ErrorMsg, "[NO_ACCOUNT]") ||
|
||||||
|
strings.Contains(resource.ErrorMsg, "[NO_VALID_ACCOUNT]") ||
|
||||||
|
strings.Contains(resource.ErrorMsg, "[TRANSFER_FAILED]") ||
|
||||||
|
strings.Contains(resource.ErrorMsg, "[LINK_CHECK_FAILED]") {
|
||||||
|
isRetryable = true
|
||||||
|
} else if strings.Contains(errorMsg, "没有可用的网盘账号") ||
|
||||||
|
strings.Contains(errorMsg, "没有有效的网盘账号") ||
|
||||||
|
strings.Contains(errorMsg, "网盘信息获取失败") ||
|
||||||
|
strings.Contains(errorMsg, "链接检查失败") {
|
||||||
|
isRetryable = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if isRetryable {
|
||||||
|
if err := repoManager.ReadyResourceRepository.ClearErrorMsg(resource.ID); err == nil {
|
||||||
|
clearedCount++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
skippedCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SuccessResponse(c, gin.H{
|
||||||
|
"message": "已清除可重试资源的错误信息,资源将在下次调度时重新处理",
|
||||||
|
"total_count": len(resources),
|
||||||
|
"cleared_count": clearedCount,
|
||||||
|
"skipped_count": skippedCount,
|
||||||
|
"retryable_count": getRetryableErrorCount(resources),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
3
main.go
3
main.go
@@ -201,6 +201,9 @@ func main() {
|
|||||||
api.DELETE("/ready-resources", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.ClearReadyResources)
|
api.DELETE("/ready-resources", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.ClearReadyResources)
|
||||||
api.GET("/ready-resources/key/:key", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetReadyResourcesByKey)
|
api.GET("/ready-resources/key/:key", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetReadyResourcesByKey)
|
||||||
api.DELETE("/ready-resources/key/:key", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.DeleteReadyResourcesByKey)
|
api.DELETE("/ready-resources/key/:key", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.DeleteReadyResourcesByKey)
|
||||||
|
api.GET("/ready-resources/errors", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetReadyResourcesWithErrors)
|
||||||
|
api.POST("/ready-resources/:id/clear-error", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.ClearErrorMsg)
|
||||||
|
api.POST("/ready-resources/retry-failed", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.RetryFailedResources)
|
||||||
|
|
||||||
// 用户管理(仅管理员)
|
// 用户管理(仅管理员)
|
||||||
api.GET("/users", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetUsers)
|
api.GET("/users", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetUsers)
|
||||||
|
|||||||
153
utils/errors.go
Normal file
153
utils/errors.go
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
package utils
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// ErrorType 错误类型枚举
|
||||||
|
type ErrorType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ErrorTypeUnsupportedLink 不支持的链接
|
||||||
|
ErrorTypeUnsupportedLink ErrorType = "UNSUPPORTED_LINK"
|
||||||
|
// ErrorTypeInvalidLink 无效链接
|
||||||
|
ErrorTypeInvalidLink ErrorType = "INVALID_LINK"
|
||||||
|
// ErrorTypeNoAccount 没有可用账号
|
||||||
|
ErrorTypeNoAccount ErrorType = "NO_ACCOUNT"
|
||||||
|
// ErrorTypeNoValidAccount 没有有效账号
|
||||||
|
ErrorTypeNoValidAccount ErrorType = "NO_VALID_ACCOUNT"
|
||||||
|
// ErrorTypeServiceCreation 服务创建失败
|
||||||
|
ErrorTypeServiceCreation ErrorType = "SERVICE_CREATION_FAILED"
|
||||||
|
// ErrorTypeTransferFailed 转存失败
|
||||||
|
ErrorTypeTransferFailed ErrorType = "TRANSFER_FAILED"
|
||||||
|
// ErrorTypeTagProcessing 标签处理失败
|
||||||
|
ErrorTypeTagProcessing ErrorType = "TAG_PROCESSING_FAILED"
|
||||||
|
// ErrorTypeCategoryProcessing 分类处理失败
|
||||||
|
ErrorTypeCategoryProcessing ErrorType = "CATEGORY_PROCESSING_FAILED"
|
||||||
|
// ErrorTypeResourceSave 资源保存失败
|
||||||
|
ErrorTypeResourceSave ErrorType = "RESOURCE_SAVE_FAILED"
|
||||||
|
// ErrorTypePlatformNotFound 平台未找到
|
||||||
|
ErrorTypePlatformNotFound ErrorType = "PLATFORM_NOT_FOUND"
|
||||||
|
// ErrorTypeLinkCheckFailed 链接检查失败
|
||||||
|
ErrorTypeLinkCheckFailed ErrorType = "LINK_CHECK_FAILED"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResourceError 资源处理错误
|
||||||
|
type ResourceError struct {
|
||||||
|
Type ErrorType `json:"type"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
URL string `json:"url,omitempty"`
|
||||||
|
Details string `json:"details,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error 实现error接口
|
||||||
|
func (e *ResourceError) Error() string {
|
||||||
|
if e.Details != "" {
|
||||||
|
return fmt.Sprintf("[%s] %s: %s", e.Type, e.Message, e.Details)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("[%s] %s", e.Type, e.Message)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewResourceError 创建新的资源错误
|
||||||
|
func NewResourceError(errorType ErrorType, message string, url string, details string) *ResourceError {
|
||||||
|
return &ResourceError{
|
||||||
|
Type: errorType,
|
||||||
|
Message: message,
|
||||||
|
URL: url,
|
||||||
|
Details: details,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewUnsupportedLinkError 创建不支持的链接错误
|
||||||
|
func NewUnsupportedLinkError(url string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeUnsupportedLink, "不支持的链接地址", url, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewInvalidLinkError 创建无效链接错误
|
||||||
|
func NewInvalidLinkError(url string, details string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeInvalidLink, "链接无效", url, details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewNoAccountError 创建没有账号错误
|
||||||
|
func NewNoAccountError(platform string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeNoAccount, "没有可用的网盘账号", "", fmt.Sprintf("平台: %s", platform))
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewNoValidAccountError 创建没有有效账号错误
|
||||||
|
func NewNoValidAccountError(platform string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeNoValidAccount, "没有有效的网盘账号", "", fmt.Sprintf("平台: %s", platform))
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewServiceCreationError 创建服务创建失败错误
|
||||||
|
func NewServiceCreationError(url string, details string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeServiceCreation, "创建网盘服务失败", url, details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTransferFailedError 创建转存失败错误
|
||||||
|
func NewTransferFailedError(url string, details string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeTransferFailed, "网盘信息获取失败", url, details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTagProcessingError 创建标签处理失败错误
|
||||||
|
func NewTagProcessingError(details string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeTagProcessing, "处理标签失败", "", details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCategoryProcessingError 创建分类处理失败错误
|
||||||
|
func NewCategoryProcessingError(details string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeCategoryProcessing, "处理分类失败", "", details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewResourceSaveError 创建资源保存失败错误
|
||||||
|
func NewResourceSaveError(url string, details string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeResourceSave, "资源保存失败", url, details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPlatformNotFoundError 创建平台未找到错误
|
||||||
|
func NewPlatformNotFoundError(platform string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypePlatformNotFound, "未找到对应的平台ID", "", fmt.Sprintf("平台: %s", platform))
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLinkCheckError 创建链接检查失败错误
|
||||||
|
func NewLinkCheckError(url string, details string) *ResourceError {
|
||||||
|
return NewResourceError(ErrorTypeLinkCheckFailed, "链接检查失败", url, details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsResourceError 检查是否为资源错误
|
||||||
|
func IsResourceError(err error) bool {
|
||||||
|
_, ok := err.(*ResourceError)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetResourceError 获取资源错误
|
||||||
|
func GetResourceError(err error) *ResourceError {
|
||||||
|
if resourceErr, ok := err.(*ResourceError); ok {
|
||||||
|
return resourceErr
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetErrorType 获取错误类型
|
||||||
|
func GetErrorType(err error) ErrorType {
|
||||||
|
if resourceErr := GetResourceError(err); resourceErr != nil {
|
||||||
|
return resourceErr.Type
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsRetryableError 检查是否为可重试的错误
|
||||||
|
func IsRetryableError(err error) bool {
|
||||||
|
errorType := GetErrorType(err)
|
||||||
|
switch errorType {
|
||||||
|
case ErrorTypeNoAccount, ErrorTypeNoValidAccount, ErrorTypeTransferFailed, ErrorTypeLinkCheckFailed:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetErrorSummary 获取错误摘要
|
||||||
|
func GetErrorSummary(err error) string {
|
||||||
|
if resourceErr := GetResourceError(err); resourceErr != nil {
|
||||||
|
return fmt.Sprintf("%s: %s", resourceErr.Type, resourceErr.Message)
|
||||||
|
}
|
||||||
|
return err.Error()
|
||||||
|
}
|
||||||
@@ -352,8 +352,9 @@ func (s *Scheduler) processReadyResources() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// 获取所有待处理资源
|
// 获取所有没有错误的待处理资源
|
||||||
readyResources, err := s.readyResourceRepo.FindAll()
|
readyResources, err := s.readyResourceRepo.FindAll()
|
||||||
|
// readyResources, err := s.readyResourceRepo.FindWithoutErrors()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("获取待处理资源失败: %v", err)
|
Error("获取待处理资源失败: %v", err)
|
||||||
return
|
return
|
||||||
@@ -384,11 +385,22 @@ func (s *Scheduler) processReadyResources() {
|
|||||||
|
|
||||||
if err := s.convertReadyResourceToResource(readyResource, factory); err != nil {
|
if err := s.convertReadyResourceToResource(readyResource, factory); err != nil {
|
||||||
Error("处理资源失败 (ID: %d): %v", readyResource.ID, err)
|
Error("处理资源失败 (ID: %d): %v", readyResource.ID, err)
|
||||||
|
|
||||||
|
// 保存完整的错误信息
|
||||||
|
readyResource.ErrorMsg = err.Error()
|
||||||
|
|
||||||
|
if updateErr := s.readyResourceRepo.Update(&readyResource); updateErr != nil {
|
||||||
|
Error("更新错误信息失败 (ID: %d): %v", readyResource.ID, updateErr)
|
||||||
|
} else {
|
||||||
|
Info("已保存错误信息到资源 (ID: %d): %s", readyResource.ID, err.Error())
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// 处理成功,删除readyResource
|
||||||
s.readyResourceRepo.Delete(readyResource.ID)
|
s.readyResourceRepo.Delete(readyResource.ID)
|
||||||
processedCount++
|
processedCount++
|
||||||
Info("成功处理资源: %s", readyResource.URL)
|
Info("成功处理资源: %s", readyResource.URL)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Info("待处理资源处理完成,共处理 %d 个资源", processedCount)
|
Info("待处理资源处理完成,共处理 %d 个资源", processedCount)
|
||||||
}
|
}
|
||||||
@@ -401,7 +413,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
|
|||||||
shareID, serviceType := panutils.ExtractShareId(readyResource.URL)
|
shareID, serviceType := panutils.ExtractShareId(readyResource.URL)
|
||||||
if serviceType == panutils.NotFound {
|
if serviceType == panutils.NotFound {
|
||||||
Warn("不支持的链接地址: %s", readyResource.URL)
|
Warn("不支持的链接地址: %s", readyResource.URL)
|
||||||
return nil
|
return NewUnsupportedLinkError(readyResource.URL)
|
||||||
}
|
}
|
||||||
|
|
||||||
Debug("检测到服务类型: %s, 分享ID: %s", serviceType.String(), shareID)
|
Debug("检测到服务类型: %s, 分享ID: %s", serviceType.String(), shareID)
|
||||||
@@ -420,24 +432,32 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
|
|||||||
// 不是夸克,直接保存,
|
// 不是夸克,直接保存,
|
||||||
if serviceType != panutils.Quark {
|
if serviceType != panutils.Quark {
|
||||||
// 检测是否有效
|
// 检测是否有效
|
||||||
checkResult, _ := commonutils.CheckURL(readyResource.URL)
|
checkResult, err := commonutils.CheckURL(readyResource.URL)
|
||||||
|
if err != nil {
|
||||||
|
Error("链接检查失败: %v", err)
|
||||||
|
return NewLinkCheckError(readyResource.URL, err.Error())
|
||||||
|
}
|
||||||
if !checkResult.Status {
|
if !checkResult.Status {
|
||||||
Warn("链接无效: %s", readyResource.URL)
|
Warn("链接无效: %s", readyResource.URL)
|
||||||
return nil
|
return NewInvalidLinkError(readyResource.URL, "链接状态检查失败")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
|
||||||
} else {
|
} else {
|
||||||
// 获取夸克网盘账号的 cookie
|
// 获取夸克网盘账号的 cookie
|
||||||
accounts, err := s.cksRepo.FindByPanID(*s.getPanIDByServiceType(serviceType))
|
panID := s.getPanIDByServiceType(serviceType)
|
||||||
|
if panID == nil {
|
||||||
|
Error("未找到对应的平台ID")
|
||||||
|
return NewPlatformNotFoundError(serviceType.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
accounts, err := s.cksRepo.FindByPanID(*panID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("获取夸克网盘账号失败: %v", err)
|
Error("获取夸克网盘账号失败: %v", err)
|
||||||
return err
|
return NewServiceCreationError(readyResource.URL, fmt.Sprintf("获取网盘账号失败: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(accounts) == 0 {
|
if len(accounts) == 0 {
|
||||||
Error("没有可用的夸克网盘账号")
|
Error("没有可用的夸克网盘账号")
|
||||||
return fmt.Errorf("没有可用的夸克网盘账号")
|
return NewNoAccountError(serviceType.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
// 选择第一个有效的账号
|
// 选择第一个有效的账号
|
||||||
@@ -451,7 +471,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
|
|||||||
|
|
||||||
if selectedAccount == nil {
|
if selectedAccount == nil {
|
||||||
Error("没有有效的夸克网盘账号")
|
Error("没有有效的夸克网盘账号")
|
||||||
return fmt.Errorf("没有有效的夸克网盘账号")
|
return NewNoValidAccountError(serviceType.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
Debug("使用夸克网盘账号: %d, Cookie: %s", selectedAccount.ID, selectedAccount.Ck[:20]+"...")
|
Debug("使用夸克网盘账号: %d, Cookie: %s", selectedAccount.ID, selectedAccount.Ck[:20]+"...")
|
||||||
@@ -471,32 +491,32 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
|
|||||||
panService, err := factory.CreatePanService(readyResource.URL, config)
|
panService, err := factory.CreatePanService(readyResource.URL, config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("获取网盘服务失败: %v", err)
|
Error("获取网盘服务失败: %v", err)
|
||||||
return err
|
return NewServiceCreationError(readyResource.URL, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
// 统一处理:尝试转存获取标题
|
// 统一处理:尝试转存获取标题
|
||||||
result, err := panService.Transfer(shareID)
|
result, err := panService.Transfer(shareID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("网盘信息获取失败: %v", err)
|
Error("网盘信息获取失败: %v", err)
|
||||||
return err
|
return NewTransferFailedError(readyResource.URL, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
if !result.Success {
|
if !result.Success {
|
||||||
Error("网盘信息获取失败: %s", result.Message)
|
Error("网盘信息获取失败: %s", result.Message)
|
||||||
return nil
|
return NewTransferFailedError(readyResource.URL, result.Message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 如果获取到了标题,更新资源标题
|
// 如果获取到了标题,更新资源标题
|
||||||
if result.Title != "" {
|
// if result.Title != "" {
|
||||||
resource.Title = result.Title
|
// resource.Title = result.Title
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
// 处理标签
|
// 处理标签
|
||||||
tagIDs, err := s.handleTags(readyResource.Tags)
|
tagIDs, err := s.handleTags(readyResource.Tags)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("处理标签失败: %v", err)
|
Error("处理标签失败: %v", err)
|
||||||
return err
|
return NewTagProcessingError(err.Error())
|
||||||
}
|
}
|
||||||
// 如果没有标签,tagIDs 可能为 nil,这是正常的
|
// 如果没有标签,tagIDs 可能为 nil,这是正常的
|
||||||
if tagIDs == nil {
|
if tagIDs == nil {
|
||||||
@@ -506,7 +526,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
|
|||||||
categoryID, err := s.resolveCategory(readyResource.Category, tagIDs)
|
categoryID, err := s.resolveCategory(readyResource.Category, tagIDs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("处理分类失败: %v", err)
|
Error("处理分类失败: %v", err)
|
||||||
return err
|
return NewCategoryProcessingError(err.Error())
|
||||||
}
|
}
|
||||||
if categoryID != nil {
|
if categoryID != nil {
|
||||||
resource.CategoryID = categoryID
|
resource.CategoryID = categoryID
|
||||||
@@ -515,7 +535,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
|
|||||||
err = s.resourceRepo.Create(resource)
|
err = s.resourceRepo.Create(resource)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("资源保存失败: %v", err)
|
Error("资源保存失败: %v", err)
|
||||||
return err
|
return NewResourceSaveError(readyResource.URL, err.Error())
|
||||||
}
|
}
|
||||||
// 插入 resource_tags 关联
|
// 插入 resource_tags 关联
|
||||||
if len(tagIDs) > 0 {
|
if len(tagIDs) > 0 {
|
||||||
@@ -523,6 +543,7 @@ func (s *Scheduler) convertReadyResourceToResource(readyResource entity.ReadyRes
|
|||||||
err := s.resourceRepo.CreateResourceTag(resource.ID, tagID)
|
err := s.resourceRepo.CreateResourceTag(resource.ID, tagID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Error("插入资源标签关联失败: %v", err)
|
Error("插入资源标签关联失败: %v", err)
|
||||||
|
// 这里不返回错误,因为资源已经保存成功,标签关联失败不影响主要功能
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
Reference in New Issue
Block a user