injectlib/urldb
1
0
Fork 0
mirror of https://github.com/ctwj/urldb.git synced 2025-11-25 11:29:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

update: Finish Auto save

Browse Source
This commit is contained in:
ctwj
2025-07-24 01:05:46 +08:00
parent 42ffc1e2e8
commit e2d4960c4c
10 changed files with 211 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
common/pan_factory.go
View File

@@ -42,6 +42,7 @@ type PanConfig struct {
ExpiredType int `json:"expiredType"` // 1: 分享永久, 2: 临时
AdFid string `json:"adFid"` // 夸克专用 - 分享时带上这个文件的fid
Stoken string `json:"stoken"`
Cookie string `json:"cookie"`
}
// TransferResult 转存结果

44
common/quark_pan.go
View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"log"
"strconv"
"strings"
"sync"
"time"
@@ -42,6 +43,7 @@ func NewQuarkPanService(config *PanConfig) *QuarkPanService {
"Referer": "https://pan.quark.cn/",
"Referrer-Policy": "strict-origin-when-cross-origin",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Cookie": config.Cookie,
})
})
@@ -66,6 +68,25 @@ func (q *QuarkPanService) UpdateConfig(config *PanConfig) {
defer q.configMutex.Unlock()
q.config = config
// 设置Cookie到header
if config.Cookie != "" {
q.SetHeader("Cookie", config.Cookie)
}
}
// SetCookie 设置Cookie
func (q *QuarkPanService) SetCookie(cookie string) {
q.SetHeader("Cookie", cookie)
q.configMutex.Lock()
if q.config != nil {
q.config.Cookie = cookie
}
q.configMutex.Unlock()
}
// GetCookie 获取当前Cookie
func (q *QuarkPanService) GetCookie() string {
return q.GetHeader("Cookie")
}
// GetServiceType 获取服务类型
@@ -383,11 +404,23 @@ func (q *QuarkPanService) getShareSave(shareID, stoken string, fidList, fidToken
return &response.Data, nil
}
// 生成指定长度的时间戳
func (q *QuarkPanService) generateTimestamp(length int) int64 {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
timestampStr := strconv.FormatInt(timestamp, 10)
if len(timestampStr) > length {
timestampStr = timestampStr[:length]
}
timestamp, _ = strconv.ParseInt(timestampStr, 10, 64)
return timestamp
}
// getShareBtn 分享按钮
func (q *QuarkPanService) getShareBtn(fidList []string, title string) (*ShareBtnResult, error) {
data := map[string]interface{}{
"fid_list": fidList,
"title": title,
"url_type": 1,
"expired_type": 1, // 永久分享
}
@@ -397,7 +430,7 @@ func (q *QuarkPanService) getShareBtn(fidList []string, title string) (*ShareBtn
"uc_param_str": "",
}
respData, err := q.HTTPPost("https://drive-pc.quark.cn/1/clouddrive/share/create", data, queryParams)
respData, err := q.HTTPPost("https://drive-pc.quark.cn/1/clouddrive/share", data, queryParams)
if err != nil {
return nil, err
}
@@ -427,9 +460,11 @@ func (q *QuarkPanService) getShareTask(taskID string, retryIndex int) (*TaskResu
"uc_param_str": "",
"task_id": taskID,
"retry_index": fmt.Sprintf("%d", retryIndex),
"__dt": "21192",
"__t": fmt.Sprintf("%d", q.generateTimestamp(13)),
}
respData, err := q.HTTPGet("https://drive-pc.quark.cn/1/clouddrive/share/sharepage/task", queryParams)
respData, err := q.HTTPGet("https://drive-pc.quark.cn/1/clouddrive/task", queryParams)
if err != nil {
return nil, err
}
@@ -457,10 +492,13 @@ func (q *QuarkPanService) getSharePassword(shareID string) (*PasswordResult, err
"pr": "ucpro",
"fr": "pc",
"uc_param_str": "",
}
data := map[string]interface{}{
"share_id": shareID,
}
respData, err := q.HTTPGet("https://drive-pc.quark.cn/1/clouddrive/share/sharepage/password", queryParams)
respData, err := q.HTTPPost("https://drive-pc.quark.cn/1/clouddrive/share/password", data, queryParams)
if err != nil {
return nil, err
}

2
db/converter/converter.go
View File

@@ -14,7 +14,7 @@ func ToResourceResponse(resource *entity.Resource) dto.ResourceResponse {
Description: resource.Description,
URL: resource.URL,
PanID: resource.PanID,
QuarkURL: resource.QuarkURL,
SaveURL: resource.SaveURL,
FileSize: resource.FileSize,
CategoryID: resource.CategoryID,
ViewCount: resource.ViewCount,

4
db/dto/request.go
View File

@@ -52,7 +52,7 @@ type CreateResourceRequest struct {
Description string `json:"description"`
URL string `json:"url"`
PanID *uint `json:"pan_id"`
QuarkURL string `json:"quark_url"`
SaveURL string `json:"save_url"`
FileSize string `json:"file_size"`
CategoryID *uint `json:"category_id"`
IsValid bool `json:"is_valid"`
@@ -69,7 +69,7 @@ type UpdateResourceRequest struct {
Description string `json:"description"`
URL string `json:"url"`
PanID *uint `json:"pan_id"`
QuarkURL string `json:"quark_url"`
SaveURL string `json:"save_url"`
FileSize string `json:"file_size"`
CategoryID *uint `json:"category_id"`
IsValid bool `json:"is_valid"`

2
db/dto/response.go
View File

@@ -17,7 +17,7 @@ type ResourceResponse struct {
Description string `json:"description"`
URL string `json:"url"`
PanID *uint `json:"pan_id"`
QuarkURL string `json:"quark_url"`
SaveURL string `json:"save_url"`
FileSize string `json:"file_size"`
CategoryID *uint `json:"category_id"`
CategoryName string `json:"category_name"`

4
db/entity/resource.go
View File

@@ -13,7 +13,7 @@ type Resource struct {
Description string `json:"description" gorm:"type:text;comment:资源描述"`
URL string `json:"url" gorm:"size:128;comment:资源链接"`
PanID *uint `json:"pan_id" gorm:"comment:平台ID"`
QuarkURL string `json:"quark_url" gorm:"size:500;comment:夸克链接"`
SaveURL string `json:"save_url" gorm:"size:500;comment:转存后的链接"`
FileSize string `json:"file_size" gorm:"size:100;comment:文件大小"`
CategoryID *uint `json:"category_id" gorm:"comment:分类ID"`
ViewCount int `json:"view_count" gorm:"default:0;comment:浏览次数"`
@@ -25,6 +25,8 @@ type Resource struct {
Cover string `json:"cover" gorm:"size:500;comment:封面"`
Author string `json:"author" gorm:"size:100;comment:作者"`
ErrorMsg string `json:"error_msg" gorm:"size:255;comment:转存失败原因"`
CkID *uint `json:"ck_id" gorm:"comment:账号ID"`
Fid string `json:"fid" gorm:"size:128;comment:网盘文件ID"`
// 关联关系
Category Category `json:"category" gorm:"foreignKey:CategoryID"`

15
db/repo/resource_repository.go
View File

@@ -32,6 +32,7 @@ type ResourceRepository interface {
InvalidateCache() error
FindExists(url string, excludeID ...uint) (bool, error)
BatchFindByURLs(urls []string) ([]entity.Resource, error)
GetResourcesForTransfer(panID uint, sinceTime time.Time) ([]*entity.Resource, error)
}
// ResourceRepositoryImpl Resource的Repository实现
@@ -354,3 +355,17 @@ func (r *ResourceRepositoryImpl) BatchFindByURLs(urls []string) ([]entity.Resour
err := r.db.Where("url IN ?", urls).Find(&resources).Error
return resources, err
}
// GetResourcesForTransfer 获取需要转存的资源
func (r *ResourceRepositoryImpl) GetResourcesForTransfer(panID uint, sinceTime time.Time) ([]*entity.Resource, error) {
var resources []*entity.Resource
query := r.db.Where("pan_id = ? AND (save_url = '' OR save_url IS NULL) AND (error_msg = '' OR error_msg IS NULL)", panID)
if !sinceTime.IsZero() {
query = query.Where("created_at >= ?", sinceTime)
}
err := query.Order("created_at DESC").Find(&resources).Error
if err != nil {
return nil, err
}
return resources, nil
}

6
handlers/resource_handler.go
View File

@@ -120,7 +120,7 @@ func CreateResource(c *gin.Context) {
Description: req.Description,
URL: req.URL,
PanID: req.PanID,
QuarkURL: req.QuarkURL,
SaveURL: req.SaveURL,
FileSize: req.FileSize,
CategoryID: req.CategoryID,
IsValid: req.IsValid,
@@ -185,8 +185,8 @@ func UpdateResource(c *gin.Context) {
if req.PanID != nil {
resource.PanID = req.PanID
}
if req.QuarkURL != "" {
resource.QuarkURL = req.QuarkURL
if req.SaveURL != "" {
resource.SaveURL = req.SaveURL
}
if req.FileSize != "" {
resource.FileSize = req.FileSize

183
utils/scheduler.go
View File

@@ -10,6 +10,7 @@ import (
commonutils "github.com/ctwj/urldb/common/utils"
"github.com/ctwj/urldb/db/entity"
"github.com/ctwj/urldb/db/repo"
"gorm.io/gorm"
)
// Scheduler 定时任务管理器
@@ -284,7 +285,7 @@ func (s *Scheduler) StartReadyResourceScheduler() {
go func() {
// 获取系统配置中的间隔时间
config, err := s.systemConfigRepo.GetOrCreateDefault()
interval := 5 * time.Minute // 默认5分钟
interval := 3 * time.Minute // 默认5分钟
if err == nil && config.AutoProcessInterval > 0 {
interval = time.Duration(config.AutoProcessInterval) * time.Minute
}
@@ -658,22 +659,45 @@ func (s *Scheduler) processAutoTransfer() {
return
}
// 获取所有有效的网盘账号
// 获取quark平台ID
panRepoImpl, ok := s.panRepo.(interface{ GetDB() *gorm.DB })
if !ok {
Error("panRepo不支持GetDB方法")
return
}
var quarkPan entity.Pan
err = panRepoImpl.GetDB().Where("name = ?", "quark").First(&quarkPan).Error
if err != nil {
Error("未找到quark平台: %v", err)
return
}
quarkPanID := quarkPan.ID
// 获取所有账号
accounts, err := s.cksRepo.FindAll()
if err != nil {
Error("获取网盘账号失败: %v", err)
return
}
if len(accounts) == 0 {
Info("没有可用的网盘账号")
// 过滤：只保留已激活、quark平台、剩余空间足够的账号
minSpaceBytes := int64(config.AutoTransferMinSpace) * 1024 * 1024 * 1024
var validAccounts []entity.Cks
for _, acc := range accounts {
if acc.IsValid && acc.PanID == quarkPanID && acc.LeftSpace >= minSpaceBytes {
validAccounts = append(validAccounts, acc)
}
}
if len(validAccounts) == 0 {
Info("没有可用的quark网盘账号")
return
}
Info("找到 %d 个网盘账号，开始自动转存处理...", len(accounts))
Info("找到 %d 个可用quark网盘账号，开始自动转存处理...", len(validAccounts))
// 获取需要转存的资源
resources, err := s.getResourcesForTransfer(config)
resources, err := s.getResourcesForTransfer(config, quarkPanID)
if err != nil {
Error("获取需要转存的资源失败: %v", err)
return
@@ -686,55 +710,120 @@ func (s *Scheduler) processAutoTransfer() {
Info("找到 %d 个需要转存的资源", len(resources))
// 执行自动转存
transferCount := 0
for _, resource := range resources {
if err := s.transferResource(resource, accounts, config); err != nil {
Error("转存资源失败 (ID: %d): %v", resource.ID, err)
} else {
transferCount++
Info("成功转存资源: %s", resource.Title)
}
// 并发自动转存
resourceCh := make(chan *entity.Resource, len(resources))
for _, res := range resources {
resourceCh <- res
}
close(resourceCh)
Info("自动转存处理完成，共转存 %d 个资源", transferCount)
var wg sync.WaitGroup
for _, account := range validAccounts {
wg.Add(1)
go func(acc entity.Cks) {
defer wg.Done()
factory := panutils.GetInstance() // 使用单例模式
for res := range resourceCh {
if err := s.transferResource(res, []entity.Cks{acc}, config, factory); err != nil {
Error("转存资源失败 (ID: %d): %v", res.ID, err)
} else {
Info("成功转存资源: %s", res.Title)
}
}
}(account)
}
wg.Wait()
Info("自动转存处理完成，账号数: %d，资源数: %d", len(validAccounts), len(resources))
}
// getResourcesForTransfer 获取需要转存的资源
func (s *Scheduler) getResourcesForTransfer(config *entity.SystemConfig) ([]*entity.Resource, error) {
// TODO: 实现获取需要转存的资源逻辑
// 1. 获取所有有效的资源
// 2. 根据配置的转存限制天数过滤资源
// 3. 排除已经转存过的资源
// 4. 按优先级排序（可以根据浏览次数、创建时间等）
Info("获取需要转存的资源 - 限制天数: %d", config.AutoTransferLimitDays)
// 临时返回空数组，等待具体实现
return []*entity.Resource{}, nil
}
// transferResource 转存单个资源
func (s *Scheduler) transferResource(resource *entity.Resource, accounts []entity.Cks, config *entity.SystemConfig) error {
// TODO: 实现单个资源的转存逻辑
// 1. 选择合适的网盘账号（根据剩余空间、VIP状态等）
// 2. 检查账号剩余空间是否满足最小空间要求
// 3. 调用网盘API进行转存
// 4. 更新资源状态和转存记录
// 5. 更新账号使用空间
Info("开始转存资源: %s (ID: %d)", resource.Title, resource.ID)
// 选择最佳账号
selectedAccount := s.selectBestAccount(accounts, config)
if selectedAccount == nil {
return fmt.Errorf("没有合适的网盘账号")
func (s *Scheduler) getResourcesForTransfer(config *entity.SystemConfig, quarkPanID uint) ([]*entity.Resource, error) {
days := config.AutoTransferLimitDays
var sinceTime time.Time
if days > 0 {
sinceTime = time.Now().AddDate(0, 0, -days)
} else {
sinceTime = time.Time{}
}
Info("选择账号: %s (剩余空间: %d GB)", selectedAccount.Username, selectedAccount.LeftSpace/1024/1024/1024)
repoImpl, ok := s.resourceRepo.(*repo.ResourceRepositoryImpl)
if !ok {
return nil, fmt.Errorf("resourceRepo不是ResourceRepositoryImpl类型")
}
return repoImpl.GetResourcesForTransfer(quarkPanID, sinceTime)
}
// TODO: 执行实际的转存操作
// 这里需要调用网盘API进行转存
var resourceUpdateMutex sync.Mutex // 全局互斥锁，保证多协程安全
// transferResource 转存单个资源
func (s *Scheduler) transferResource(resource *entity.Resource, accounts []entity.Cks, config *entity.SystemConfig, factory *panutils.PanFactory) error {
if len(accounts) == 0 {
return fmt.Errorf("没有可用的网盘账号")
}
account := accounts[0]
service, err := factory.CreatePanService(resource.URL, &panutils.PanConfig{
URL: resource.URL,
ExpiredType: 0,
IsType: 0,
Cookie: account.Ck,
})
if err != nil {
return fmt.Errorf("创建网盘服务失败: %v", err)
}
shareID, _ := commonutils.ExtractShareIdString(resource.URL)
result, err := service.Transfer(shareID)
if err != nil {
resourceUpdateMutex.Lock()
defer resourceUpdateMutex.Unlock()
s.resourceRepo.Update(&entity.Resource{
ID: resource.ID,
ErrorMsg: err.Error(),
})
return fmt.Errorf("转存失败: %v", err)
}
if result == nil || !result.Success {
errMsg := "转存失败"
if result != nil && result.Message != "" {
errMsg = result.Message
}
resourceUpdateMutex.Lock()
defer resourceUpdateMutex.Unlock()
s.resourceRepo.Update(&entity.Resource{
ID: resource.ID,
ErrorMsg: errMsg,
})
return fmt.Errorf("转存失败: %s", errMsg)
}
// 提取转存链接、fid等
var saveURL, fid string
if data, ok := result.Data.(map[string]interface{}); ok {
if v, ok := data["shareUrl"]; ok {
saveURL, _ = v.(string)
}
if v, ok := data["fid"]; ok {
fid, _ = v.(string)
}
}
if saveURL == "" {
saveURL = result.ShareURL
}
resourceUpdateMutex.Lock()
defer resourceUpdateMutex.Unlock()
err = s.resourceRepo.Update(&entity.Resource{
ID: resource.ID,
SaveURL: saveURL,
CkID: &account.ID,
Fid: fid,
ErrorMsg: "",
})
if err != nil {
return fmt.Errorf("保存转存结果失败: %v", err)
}
return nil
}

8
web/pages/index.vue
View File

@@ -287,7 +287,9 @@ const route = useRoute()
const router = useRouter()
// 响应式数据
const searchQuery = ref(route.query.search as string || '')
const initSerch = route.query.search || ''
const oldQuery = ref(initSerch)
const searchQuery = ref(oldQuery)
const currentPage = ref(parseInt(route.query.page as string) || 1)
const pageSize = ref(200)
const selectedPlatform = ref(route.query.platform as string || '')
@@ -346,6 +348,10 @@ const handleSearch = async (e?: any) => {
if (e && e.target && typeof e.target.value === 'string') {
searchQuery.value = e.target.value
}
if (oldQuery.value === searchQuery.value) {
return
}
oldQuery.value = searchQuery.value
currentPage.value = 1
// 更新URL参数