mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2025-11-25 03:15:19 +08:00
Compare commits
12 Commits
renovate/g
...
2844797684
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2844797684 | ||
|
|
9f4e439478 | ||
|
|
9d09ee133d | ||
|
|
d88f0e8f3c | ||
|
|
0857478516 | ||
|
|
66d9809057 | ||
|
|
db8a7e8caf | ||
|
|
8f18e34da0 | ||
|
|
525f26dc23 | ||
|
|
a0fcfa3ed2 | ||
|
|
15f276537c | ||
|
|
623a12050e |
@@ -2,6 +2,7 @@ package flags
|
||||
|
||||
var (
|
||||
DataDir string
|
||||
ConfigPath string
|
||||
Debug bool
|
||||
NoPrefix bool
|
||||
Dev bool
|
||||
|
||||
@@ -27,7 +27,8 @@ func Execute() {
|
||||
}
|
||||
|
||||
func init() {
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data folder")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data directory (relative paths are resolved against the current working directory)")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.ConfigPath, "config", "", "path to config.json (relative to current working directory; defaults to [data directory]/config.json, where [data directory] is set by --data)")
|
||||
RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode")
|
||||
RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix")
|
||||
RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode")
|
||||
|
||||
@@ -17,7 +17,7 @@ type Addition struct {
|
||||
var config = driver.Config{
|
||||
Name: "115 Cloud",
|
||||
DefaultRoot: "0",
|
||||
LinkCacheType: 2,
|
||||
LinkCacheMode: driver.LinkCacheUA,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -19,7 +19,7 @@ type Addition struct {
|
||||
var config = driver.Config{
|
||||
Name: "115 Open",
|
||||
DefaultRoot: "0",
|
||||
LinkCacheType: 2,
|
||||
LinkCacheMode: driver.LinkCacheUA,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -41,7 +41,9 @@ func (d *Pan123) GetAddition() driver.Additional {
|
||||
}
|
||||
|
||||
func (d *Pan123) Init(ctx context.Context) error {
|
||||
_, err := d.Request(UserInfo, http.MethodGet, nil, nil)
|
||||
_, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) {
|
||||
req.SetHeader("platform", "web")
|
||||
}, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,8 @@ type Addition struct {
|
||||
//OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"`
|
||||
//OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
|
||||
AccessToken string
|
||||
UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
|
||||
UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
|
||||
Platform string `json:"platform" type:"string" default:"web" help:"the platform header value, sent with API requests"`
|
||||
}
|
||||
|
||||
var config = driver.Config{
|
||||
|
||||
@@ -203,7 +203,7 @@ do:
|
||||
"referer": "https://www.123pan.com/",
|
||||
"authorization": "Bearer " + d.AccessToken,
|
||||
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) openlist-client",
|
||||
"platform": "web",
|
||||
"platform": d.Platform,
|
||||
"app-version": "3",
|
||||
//"user-agent": base.UserAgent,
|
||||
})
|
||||
|
||||
@@ -524,4 +524,25 @@ func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode {
|
||||
root, sub := d.getRootAndPath(path)
|
||||
dsts, ok := d.pathMap[root]
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
for _, dst := range dsts {
|
||||
storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(dst, sub))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mode := storage.Config().LinkCacheMode
|
||||
if mode == -1 {
|
||||
return storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(actualPath)
|
||||
} else {
|
||||
return mode
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var _ driver.Driver = (*Alias)(nil)
|
||||
|
||||
@@ -26,6 +26,7 @@ var config = driver.Config{
|
||||
NoUpload: false,
|
||||
DefaultRoot: "/",
|
||||
ProxyRangeOption: true,
|
||||
LinkCacheMode: driver.LinkCacheAuto,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -299,10 +299,7 @@ func (d *AliyundriveOpen) GetDetails(ctx context.Context) (*model.StorageDetails
|
||||
total := utils.Json.Get(res, "personal_space_info", "total_size").ToUint64()
|
||||
used := utils.Json.Get(res, "personal_space_info", "used_size").ToUint64()
|
||||
return &model.StorageDetails{
|
||||
DiskUsage: model.DiskUsage{
|
||||
TotalSpace: total,
|
||||
FreeSpace: total - used,
|
||||
},
|
||||
DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -5,11 +5,13 @@ import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
stdpath "path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
@@ -31,6 +33,8 @@ type BaiduNetdisk struct {
|
||||
vipType int // 会员类型,0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M)
|
||||
}
|
||||
|
||||
var ErrUploadIDExpired = errors.New("uploadid expired")
|
||||
|
||||
func (d *BaiduNetdisk) Config() driver.Config {
|
||||
return config
|
||||
}
|
||||
@@ -214,7 +218,6 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
|
||||
|
||||
// cal md5 for first 256k data
|
||||
const SliceSize int64 = 256 * utils.KB
|
||||
// cal md5
|
||||
blockList := make([]string, 0, count)
|
||||
byteSize := sliceSize
|
||||
fileMd5H := md5.New()
|
||||
@@ -244,7 +247,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
|
||||
}
|
||||
if tmpF != nil {
|
||||
if written != streamSize {
|
||||
return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize)
|
||||
return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize)
|
||||
}
|
||||
_, err = tmpF.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
@@ -258,31 +261,14 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
|
||||
mtime := stream.ModTime().Unix()
|
||||
ctime := stream.CreateTime().Unix()
|
||||
|
||||
// step.1 预上传
|
||||
// 尝试获取之前的进度
|
||||
// step.1 尝试读取已保存进度
|
||||
precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5)
|
||||
if !ok {
|
||||
params := map[string]string{
|
||||
"method": "precreate",
|
||||
}
|
||||
form := map[string]string{
|
||||
"path": path,
|
||||
"size": strconv.FormatInt(streamSize, 10),
|
||||
"isdir": "0",
|
||||
"autoinit": "1",
|
||||
"rtype": "3",
|
||||
"block_list": blockListStr,
|
||||
"content-md5": contentMd5,
|
||||
"slice-md5": sliceMd5,
|
||||
}
|
||||
joinTime(form, ctime, mtime)
|
||||
|
||||
log.Debugf("[baidu_netdisk] precreate data: %s", form)
|
||||
_, err = d.postForm("/xpan/file", params, form, &precreateResp)
|
||||
// 没有进度,走预上传
|
||||
precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debugf("%+v", precreateResp)
|
||||
if precreateResp.ReturnType == 2 {
|
||||
// rapid upload, since got md5 match from baidu server
|
||||
// 修复时间,具体原因见 Put 方法注释的 **注意**
|
||||
@@ -291,45 +277,81 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
|
||||
return fileToObj(precreateResp.File), nil
|
||||
}
|
||||
}
|
||||
|
||||
// step.2 上传分片
|
||||
threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread,
|
||||
retry.Attempts(1),
|
||||
retry.Delay(time.Second),
|
||||
retry.DelayType(retry.BackOffDelay))
|
||||
uploadLoop:
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread,
|
||||
retry.Attempts(1),
|
||||
retry.Delay(time.Second),
|
||||
retry.DelayType(retry.BackOffDelay))
|
||||
|
||||
for i, partseq := range precreateResp.BlockList {
|
||||
if utils.IsCanceled(upCtx) {
|
||||
break
|
||||
cacheReaderAt, okReaderAt := cache.(io.ReaderAt)
|
||||
if !okReaderAt {
|
||||
return nil, fmt.Errorf("cache object must implement io.ReaderAt interface for upload operations")
|
||||
}
|
||||
|
||||
i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize
|
||||
if partseq+1 == count {
|
||||
byteSize = lastBlockSize
|
||||
totalParts := len(precreateResp.BlockList)
|
||||
doneParts := 0 // 新增计数器
|
||||
|
||||
for i, partseq := range precreateResp.BlockList {
|
||||
if utils.IsCanceled(upCtx) || partseq < 0 {
|
||||
continue
|
||||
}
|
||||
i, partseq := i, partseq
|
||||
offset, size := int64(partseq)*sliceSize, sliceSize
|
||||
if partseq+1 == count {
|
||||
size = lastBlockSize
|
||||
}
|
||||
threadG.Go(func(ctx context.Context) error {
|
||||
params := map[string]string{
|
||||
"method": "upload",
|
||||
"access_token": d.AccessToken,
|
||||
"type": "tmpfile",
|
||||
"path": path,
|
||||
"uploadid": precreateResp.Uploadid,
|
||||
"partseq": strconv.Itoa(partseq),
|
||||
}
|
||||
section := io.NewSectionReader(cacheReaderAt, offset, size)
|
||||
err := d.uploadSlice(ctx, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
precreateResp.BlockList[i] = -1
|
||||
doneParts++ // 直接递增计数器
|
||||
if totalParts > 0 {
|
||||
up(float64(doneParts) * 100.0 / float64(totalParts))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
threadG.Go(func(ctx context.Context) error {
|
||||
params := map[string]string{
|
||||
"method": "upload",
|
||||
"access_token": d.AccessToken,
|
||||
"type": "tmpfile",
|
||||
"path": path,
|
||||
"uploadid": precreateResp.Uploadid,
|
||||
"partseq": strconv.Itoa(partseq),
|
||||
}
|
||||
err := d.uploadSlice(ctx, params, stream.GetName(),
|
||||
driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList)))
|
||||
precreateResp.BlockList[i] = -1
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err = threadG.Wait(); err != nil {
|
||||
// 如果属于用户主动取消,则保存上传进度
|
||||
|
||||
err = threadG.Wait()
|
||||
if err == nil {
|
||||
break uploadLoop
|
||||
}
|
||||
|
||||
// 保存进度(所有错误都会保存)
|
||||
precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 })
|
||||
base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5)
|
||||
|
||||
if errors.Is(err, context.Canceled) {
|
||||
precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 })
|
||||
return nil, err
|
||||
}
|
||||
if errors.Is(err, ErrUploadIDExpired) {
|
||||
log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch")
|
||||
// 重新 precreate(所有分片都要重传)
|
||||
newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime)
|
||||
if err2 != nil {
|
||||
return nil, err2
|
||||
}
|
||||
if newPre.ReturnType == 2 {
|
||||
return fileToObj(newPre.File), nil
|
||||
}
|
||||
precreateResp = newPre
|
||||
// 覆盖掉旧的进度
|
||||
base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5)
|
||||
continue uploadLoop
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
@@ -343,9 +365,46 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
|
||||
// 修复时间,具体原因见 Put 方法注释的 **注意**
|
||||
newFile.Ctime = ctime
|
||||
newFile.Mtime = mtime
|
||||
// 上传成功清理进度
|
||||
base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5)
|
||||
return fileToObj(newFile), nil
|
||||
}
|
||||
|
||||
// precreate 执行预上传操作,支持首次上传和 uploadid 过期重试
|
||||
func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) {
|
||||
params := map[string]string{"method": "precreate"}
|
||||
form := map[string]string{
|
||||
"path": path,
|
||||
"size": strconv.FormatInt(streamSize, 10),
|
||||
"isdir": "0",
|
||||
"autoinit": "1",
|
||||
"rtype": "3",
|
||||
"block_list": blockListStr,
|
||||
}
|
||||
|
||||
// 只有在首次上传时才包含 content-md5 和 slice-md5
|
||||
if contentMd5 != "" && sliceMd5 != "" {
|
||||
form["content-md5"] = contentMd5
|
||||
form["slice-md5"] = sliceMd5
|
||||
}
|
||||
|
||||
joinTime(form, ctime, mtime)
|
||||
|
||||
var precreateResp PrecreateResp
|
||||
_, err := d.postForm("/xpan/file", params, form, &precreateResp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 修复时间,具体原因见 Put 方法注释的 **注意**
|
||||
if precreateResp.ReturnType == 2 {
|
||||
precreateResp.File.Ctime = ctime
|
||||
precreateResp.File.Mtime = mtime
|
||||
}
|
||||
|
||||
return &precreateResp, nil
|
||||
}
|
||||
|
||||
func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error {
|
||||
res, err := base.RestyClient.R().
|
||||
SetContext(ctx).
|
||||
@@ -358,8 +417,16 @@ func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string
|
||||
log.Debugln(res.RawResponse.Status + res.String())
|
||||
errCode := utils.Json.Get(res.Body(), "error_code").ToInt()
|
||||
errNo := utils.Json.Get(res.Body(), "errno").ToInt()
|
||||
respStr := res.String()
|
||||
lower := strings.ToLower(respStr)
|
||||
// 合并 uploadid 过期检测逻辑
|
||||
if strings.Contains(lower, "uploadid") &&
|
||||
(strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) {
|
||||
return ErrUploadIDExpired
|
||||
}
|
||||
|
||||
if errCode != 0 || errNo != 0 {
|
||||
return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String())
|
||||
return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ type Addition struct {
|
||||
var config = driver.Config{
|
||||
Name: "BaiduPhoto",
|
||||
LocalSort: true,
|
||||
LinkCacheType: 2,
|
||||
LinkCacheMode: driver.LinkCacheUA,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -25,6 +25,7 @@ func InitClient() {
|
||||
}),
|
||||
).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify})
|
||||
NoRedirectClient.SetHeader("user-agent", UserAgent)
|
||||
net.SetRestyProxyIfConfigured(NoRedirectClient)
|
||||
|
||||
RestyClient = NewRestyClient()
|
||||
HttpClient = net.NewHttpClient()
|
||||
@@ -37,5 +38,7 @@ func NewRestyClient() *resty.Client {
|
||||
SetRetryResetReaders(true).
|
||||
SetTimeout(DefaultTimeout).
|
||||
SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify})
|
||||
|
||||
net.SetRestyProxyIfConfigured(client)
|
||||
return client
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ var config = driver.Config{
|
||||
Name: "FebBox",
|
||||
NoUpload: true,
|
||||
DefaultRoot: "0",
|
||||
LinkCacheType: 1,
|
||||
LinkCacheMode: driver.LinkCacheIP,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -27,6 +27,14 @@ import (
|
||||
|
||||
// do others that not defined in Driver interface
|
||||
|
||||
// Google Drive API field constants
|
||||
const (
|
||||
// File list query fields
|
||||
FilesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
// Single file query fields
|
||||
FileInfoFields = "id,name,mimeType,size,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
)
|
||||
|
||||
type googleDriveServiceAccount struct {
|
||||
// Type string `json:"type"`
|
||||
// ProjectID string `json:"project_id"`
|
||||
@@ -235,7 +243,7 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) {
|
||||
}
|
||||
query := map[string]string{
|
||||
"orderBy": orderBy,
|
||||
"fields": "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken",
|
||||
"fields": FilesListFields,
|
||||
"pageSize": "1000",
|
||||
"q": fmt.Sprintf("'%s' in parents and trashed = false", id),
|
||||
//"includeItemsFromAllDrives": "true",
|
||||
@@ -249,11 +257,82 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) {
|
||||
return nil, err
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
|
||||
// Batch process shortcuts, API calls only for file shortcuts
|
||||
shortcutTargetIds := make([]string, 0)
|
||||
shortcutIndices := make([]int, 0)
|
||||
|
||||
// Collect target IDs of all file shortcuts (skip folder shortcuts)
|
||||
for i := range resp.Files {
|
||||
if resp.Files[i].MimeType == "application/vnd.google-apps.shortcut" &&
|
||||
resp.Files[i].ShortcutDetails.TargetId != "" &&
|
||||
resp.Files[i].ShortcutDetails.TargetMimeType != "application/vnd.google-apps.folder" {
|
||||
shortcutTargetIds = append(shortcutTargetIds, resp.Files[i].ShortcutDetails.TargetId)
|
||||
shortcutIndices = append(shortcutIndices, i)
|
||||
}
|
||||
}
|
||||
|
||||
// Batch get target file info (only for file shortcuts)
|
||||
if len(shortcutTargetIds) > 0 {
|
||||
targetFiles := d.batchGetTargetFilesInfo(shortcutTargetIds)
|
||||
// Update shortcut file info
|
||||
for j, targetId := range shortcutTargetIds {
|
||||
if targetFile, exists := targetFiles[targetId]; exists {
|
||||
fileIndex := shortcutIndices[j]
|
||||
if targetFile.Size != "" {
|
||||
resp.Files[fileIndex].Size = targetFile.Size
|
||||
}
|
||||
if targetFile.MD5Checksum != "" {
|
||||
resp.Files[fileIndex].MD5Checksum = targetFile.MD5Checksum
|
||||
}
|
||||
if targetFile.SHA1Checksum != "" {
|
||||
resp.Files[fileIndex].SHA1Checksum = targetFile.SHA1Checksum
|
||||
}
|
||||
if targetFile.SHA256Checksum != "" {
|
||||
resp.Files[fileIndex].SHA256Checksum = targetFile.SHA256Checksum
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res = append(res, resp.Files...)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// getTargetFileInfo gets target file details for shortcuts
|
||||
func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) {
|
||||
var targetFile File
|
||||
url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s", targetId)
|
||||
query := map[string]string{
|
||||
"fields": FileInfoFields,
|
||||
}
|
||||
_, err := d.request(url, http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParams(query)
|
||||
}, &targetFile)
|
||||
if err != nil {
|
||||
return File{}, err
|
||||
}
|
||||
return targetFile, nil
|
||||
}
|
||||
|
||||
// batchGetTargetFilesInfo batch gets target file info, sequential processing to avoid concurrency complexity
|
||||
func (d *GoogleDrive) batchGetTargetFilesInfo(targetIds []string) map[string]File {
|
||||
if len(targetIds) == 0 {
|
||||
return make(map[string]File)
|
||||
}
|
||||
|
||||
result := make(map[string]File)
|
||||
// Sequential processing to avoid concurrency complexity
|
||||
for _, targetId := range targetIds {
|
||||
file, err := d.getTargetFileInfo(targetId)
|
||||
if err == nil {
|
||||
result[targetId] = file
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, url string, up driver.UpdateProgress) error {
|
||||
defaultChunkSize := d.ChunkSize * 1024 * 1024
|
||||
ss, err := stream.NewStreamSectionReader(file, int(defaultChunkSize), &up)
|
||||
|
||||
@@ -26,11 +26,6 @@ type OpenList struct {
|
||||
}
|
||||
|
||||
func (d *OpenList) Config() driver.Config {
|
||||
if d.PassUAToUpsteam {
|
||||
c := config
|
||||
c.LinkCacheType = 2 // add User-Agent to cache key
|
||||
return c
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
@@ -115,19 +110,29 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs)
|
||||
|
||||
func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
|
||||
var resp common.Resp[FsGetResp]
|
||||
headers := map[string]string{
|
||||
"User-Agent": base.UserAgent,
|
||||
}
|
||||
// if PassUAToUpsteam is true, then pass the user-agent to the upstream
|
||||
userAgent := base.UserAgent
|
||||
if d.PassUAToUpsteam {
|
||||
userAgent = args.Header.Get("user-agent")
|
||||
if userAgent == "" {
|
||||
userAgent = base.UserAgent
|
||||
userAgent := args.Header.Get("user-agent")
|
||||
if userAgent != "" {
|
||||
headers["User-Agent"] = base.UserAgent
|
||||
}
|
||||
}
|
||||
// if PassIPToUpsteam is true, then pass the ip address to the upstream
|
||||
if d.PassIPToUpsteam {
|
||||
ip := args.IP
|
||||
if ip != "" {
|
||||
headers["X-Forwarded-For"] = ip
|
||||
headers["X-Real-Ip"] = ip
|
||||
}
|
||||
}
|
||||
_, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) {
|
||||
req.SetResult(&resp).SetBody(FsGetReq{
|
||||
Path: file.GetPath(),
|
||||
Password: d.MetaPassword,
|
||||
}).SetHeader("user-agent", userAgent)
|
||||
}).SetHeaders(headers)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -360,8 +365,15 @@ func (d *OpenList) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.O
|
||||
return err
|
||||
}
|
||||
|
||||
//func (d *OpenList) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
|
||||
// return nil, errs.NotSupport
|
||||
//}
|
||||
func (d *OpenList) ResolveLinkCacheMode(_ string) driver.LinkCacheMode {
|
||||
var mode driver.LinkCacheMode
|
||||
if d.PassIPToUpsteam {
|
||||
mode |= driver.LinkCacheIP
|
||||
}
|
||||
if d.PassUAToUpsteam {
|
||||
mode |= driver.LinkCacheUA
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
var _ driver.Driver = (*OpenList)(nil)
|
||||
|
||||
@@ -12,6 +12,7 @@ type Addition struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
Token string `json:"token"`
|
||||
PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"`
|
||||
PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"`
|
||||
ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"`
|
||||
}
|
||||
@@ -22,6 +23,7 @@ var config = driver.Config{
|
||||
DefaultRoot: "/",
|
||||
CheckStatus: true,
|
||||
ProxyRangeOption: true,
|
||||
LinkCacheMode: driver.LinkCacheAuto,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Strm struct {
|
||||
@@ -40,6 +41,9 @@ func (d *Strm) Init(ctx context.Context) error {
|
||||
if d.Paths == "" {
|
||||
return errors.New("paths is required")
|
||||
}
|
||||
if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 {
|
||||
return errors.New("SaveStrmLocalPath is required")
|
||||
}
|
||||
d.pathMap = make(map[string][]string)
|
||||
for _, path := range strings.Split(d.Paths, "\n") {
|
||||
path = strings.TrimSpace(path)
|
||||
@@ -48,6 +52,11 @@ func (d *Strm) Init(ctx context.Context) error {
|
||||
}
|
||||
k, v := getPair(path)
|
||||
d.pathMap[k] = append(d.pathMap[k], v)
|
||||
err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
|
||||
if err != nil {
|
||||
log.Errorf("insert strmTrie error: %v", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(d.pathMap) == 1 {
|
||||
for k := range d.pathMap {
|
||||
@@ -59,26 +68,51 @@ func (d *Strm) Init(ctx context.Context) error {
|
||||
d.autoFlatten = false
|
||||
}
|
||||
|
||||
d.supportSuffix = supportSuffix()
|
||||
if d.FilterFileTypes != "" {
|
||||
types := strings.Split(d.FilterFileTypes, ",")
|
||||
for _, ext := range types {
|
||||
ext = strings.ToLower(strings.TrimSpace(ext))
|
||||
if ext != "" {
|
||||
d.supportSuffix[ext] = struct{}{}
|
||||
}
|
||||
var supportTypes []string
|
||||
if d.FilterFileTypes == "" {
|
||||
d.FilterFileTypes = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac"
|
||||
}
|
||||
supportTypes = strings.Split(d.FilterFileTypes, ",")
|
||||
d.supportSuffix = map[string]struct{}{}
|
||||
for _, ext := range supportTypes {
|
||||
ext = strings.ToLower(strings.TrimSpace(ext))
|
||||
if ext != "" {
|
||||
d.supportSuffix[ext] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
d.downloadSuffix = downloadSuffix()
|
||||
if d.DownloadFileTypes != "" {
|
||||
downloadTypes := strings.Split(d.DownloadFileTypes, ",")
|
||||
for _, ext := range downloadTypes {
|
||||
ext = strings.ToLower(strings.TrimSpace(ext))
|
||||
if ext != "" {
|
||||
d.downloadSuffix[ext] = struct{}{}
|
||||
var downloadTypes []string
|
||||
if d.DownloadFileTypes == "" {
|
||||
d.DownloadFileTypes = "ass,srt,vtt,sub,strm"
|
||||
}
|
||||
downloadTypes = strings.Split(d.DownloadFileTypes, ",")
|
||||
d.downloadSuffix = map[string]struct{}{}
|
||||
for _, ext := range downloadTypes {
|
||||
ext = strings.ToLower(strings.TrimSpace(ext))
|
||||
if ext != "" {
|
||||
d.downloadSuffix[ext] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
if d.Version != 3 {
|
||||
types := strings.Split("mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac", ",")
|
||||
for _, ext := range types {
|
||||
if _, ok := d.supportSuffix[ext]; !ok {
|
||||
d.supportSuffix[ext] = struct{}{}
|
||||
supportTypes = append(supportTypes, ext)
|
||||
}
|
||||
}
|
||||
d.FilterFileTypes = strings.Join(supportTypes, ",")
|
||||
|
||||
types = strings.Split("ass,srt,vtt,sub,strm", ",")
|
||||
for _, ext := range types {
|
||||
if _, ok := d.downloadSuffix[ext]; !ok {
|
||||
d.supportSuffix[ext] = struct{}{}
|
||||
downloadTypes = append(downloadTypes, ext)
|
||||
}
|
||||
}
|
||||
d.DownloadFileTypes = strings.Join(downloadTypes, ",")
|
||||
d.Version = 3
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -87,6 +121,9 @@ func (d *Strm) Drop(ctx context.Context) error {
|
||||
d.pathMap = nil
|
||||
d.downloadSuffix = nil
|
||||
d.supportSuffix = nil
|
||||
for _, path := range strings.Split(d.Paths, "\n") {
|
||||
RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
164
drivers/strm/hook.go
Normal file
164
drivers/strm/hook.go
Normal file
@@ -0,0 +1,164 @@
|
||||
package strm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
stdpath "path"
|
||||
"strings"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/tchap/go-patricia/v2/patricia"
|
||||
)
|
||||
|
||||
var strmTrie = patricia.NewTrie()
|
||||
|
||||
func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) {
|
||||
path = utils.FixAndCleanPath(path)
|
||||
updateLocal := func(driver *Strm, basePath string, objs []model.Obj) {
|
||||
relParent := strings.TrimPrefix(basePath, driver.MountPath)
|
||||
localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent)
|
||||
for _, obj := range objs {
|
||||
localPath := stdpath.Join(localParentPath, obj.GetName())
|
||||
generateStrm(ctx, driver, obj, localPath)
|
||||
}
|
||||
deleteExtraFiles(localParentPath, objs)
|
||||
}
|
||||
|
||||
_ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error {
|
||||
strmDrivers := item.([]*Strm)
|
||||
needPath := string(needPathPrefix)
|
||||
restPath := strings.TrimPrefix(path, needPath)
|
||||
if len(restPath) > 0 && restPath[0] != '/' {
|
||||
return nil
|
||||
}
|
||||
for _, strmDriver := range strmDrivers {
|
||||
strmObjs, _ := utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) {
|
||||
ret := strmDriver.convert2strmObj(ctx, path, obj)
|
||||
return &ret, nil
|
||||
})
|
||||
updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func InsertStrm(dstPath string, d *Strm) error {
|
||||
prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
|
||||
existing := strmTrie.Get(prefix)
|
||||
|
||||
if existing == nil {
|
||||
if !strmTrie.Insert(prefix, []*Strm{d}) {
|
||||
return errors.New("failed to insert strm")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if lst, ok := existing.([]*Strm); ok {
|
||||
strmTrie.Set(prefix, append(lst, d))
|
||||
} else {
|
||||
return errors.New("invalid trie item type")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func RemoveStrm(dstPath string, d *Strm) {
|
||||
prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
|
||||
existing := strmTrie.Get(prefix)
|
||||
if existing == nil {
|
||||
return
|
||||
}
|
||||
lst, ok := existing.([]*Strm)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if len(lst) == 1 && lst[0] == d {
|
||||
strmTrie.Delete(prefix)
|
||||
return
|
||||
}
|
||||
|
||||
for i, di := range lst {
|
||||
if di == d {
|
||||
newList := append(lst[:i], lst[i+1:]...)
|
||||
strmTrie.Set(prefix, newList)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) {
|
||||
link, err := driver.Link(ctx, obj, model.LinkArgs{})
|
||||
if err != nil {
|
||||
log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err)
|
||||
return
|
||||
}
|
||||
seekableStream, err := stream.NewSeekableStream(&stream.FileStream{
|
||||
Obj: obj,
|
||||
Ctx: ctx,
|
||||
}, link)
|
||||
if err != nil {
|
||||
_ = link.Close()
|
||||
log.Warnf("failed to generate strm of obj %s: failed to get seekable stream: %v", localPath, err)
|
||||
return
|
||||
}
|
||||
defer seekableStream.Close()
|
||||
file, err := utils.CreateNestedFile(localPath)
|
||||
if err != nil {
|
||||
log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
if _, err := io.Copy(file, seekableStream); err != nil {
|
||||
log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteExtraFiles(localPath string, objs []model.Obj) {
|
||||
localFiles, err := getLocalFiles(localPath)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to read local files from %s: %v", localPath, err)
|
||||
return
|
||||
}
|
||||
|
||||
objsSet := make(map[string]struct{})
|
||||
for _, obj := range objs {
|
||||
if obj.IsDir() {
|
||||
continue
|
||||
}
|
||||
objsSet[stdpath.Join(localPath, obj.GetName())] = struct{}{}
|
||||
}
|
||||
|
||||
for _, localFile := range localFiles {
|
||||
if _, exists := objsSet[localFile]; !exists {
|
||||
err := os.Remove(localFile)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err)
|
||||
} else {
|
||||
log.Infof("Deleted file %s", localFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getLocalFiles(localPath string) ([]string, error) {
|
||||
var files []string
|
||||
entries, err := os.ReadDir(localPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
files = append(files, stdpath.Join(localPath, entry.Name()))
|
||||
}
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
op.RegisterObjsUpdateHook(UpdateLocalStrm)
|
||||
}
|
||||
@@ -8,10 +8,13 @@ import (
|
||||
type Addition struct {
|
||||
Paths string `json:"paths" required:"true" type:"text"`
|
||||
SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of the strm file"`
|
||||
FilterFileTypes string `json:"filterFileTypes" type:"text" default:"strm" required:"false" help:"Supports suffix name of strm file"`
|
||||
DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass" required:"false" help:"Files need to download with strm (usally subtitles)"`
|
||||
DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Files need to download with strm (usally subtitles)"`
|
||||
FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Supports suffix name of strm file"`
|
||||
EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"`
|
||||
LocalModel bool `json:"localModel" default:"false" help:"enable local mode"`
|
||||
WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"`
|
||||
SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"`
|
||||
SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"`
|
||||
Version int
|
||||
}
|
||||
|
||||
var config = driver.Config{
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
package strm
|
||||
|
||||
func supportSuffix() map[string]struct{} {
|
||||
return map[string]struct{}{
|
||||
// video
|
||||
"mp4": {},
|
||||
"mkv": {},
|
||||
"flv": {},
|
||||
"avi": {},
|
||||
"wmv": {},
|
||||
"ts": {},
|
||||
"rmvb": {},
|
||||
"webm": {},
|
||||
// audio
|
||||
"mp3": {},
|
||||
"flac": {},
|
||||
"aac": {},
|
||||
"wav": {},
|
||||
"ogg": {},
|
||||
"m4a": {},
|
||||
"wma": {},
|
||||
"alac": {},
|
||||
}
|
||||
}
|
||||
|
||||
func downloadSuffix() map[string]struct{} {
|
||||
return map[string]struct{}{
|
||||
// strm
|
||||
"strm": {},
|
||||
// subtitles
|
||||
"ass": {},
|
||||
"srt": {},
|
||||
"vtt": {},
|
||||
"sub": {},
|
||||
}
|
||||
}
|
||||
@@ -61,36 +61,12 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
|
||||
|
||||
var validObjs []model.Obj
|
||||
for _, obj := range objs {
|
||||
id, name, path := "", obj.GetName(), ""
|
||||
size := int64(0)
|
||||
if !obj.IsDir() {
|
||||
path = stdpath.Join(reqPath, obj.GetName())
|
||||
ext := strings.ToLower(utils.Ext(name))
|
||||
if _, ok := d.supportSuffix[ext]; ok {
|
||||
id = "strm"
|
||||
name = strings.TrimSuffix(name, ext) + "strm"
|
||||
size = int64(len(d.getLink(ctx, path)))
|
||||
} else if _, ok := d.downloadSuffix[ext]; ok {
|
||||
size = obj.GetSize()
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
objRes := model.Object{
|
||||
ID: id,
|
||||
Path: path,
|
||||
Name: name,
|
||||
Size: size,
|
||||
Modified: obj.ModTime(),
|
||||
IsFolder: obj.IsDir(),
|
||||
}
|
||||
|
||||
objRes := d.convert2strmObj(ctx, reqPath, obj)
|
||||
thumb, ok := model.GetThumb(obj)
|
||||
if !ok {
|
||||
validObjs = append(validObjs, &objRes)
|
||||
continue
|
||||
}
|
||||
|
||||
validObjs = append(validObjs, &model.ObjThumb{
|
||||
Object: objRes,
|
||||
Thumbnail: model.Thumbnail{
|
||||
@@ -101,6 +77,32 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
|
||||
return validObjs, nil
|
||||
}
|
||||
|
||||
func (d *Strm) convert2strmObj(ctx context.Context, reqPath string, obj model.Obj) model.Object {
|
||||
id, name, path := "", obj.GetName(), ""
|
||||
size := int64(0)
|
||||
if !obj.IsDir() {
|
||||
path = stdpath.Join(reqPath, obj.GetName())
|
||||
ext := strings.ToLower(utils.Ext(name))
|
||||
if _, ok := d.supportSuffix[ext]; ok {
|
||||
id = "strm"
|
||||
name = strings.TrimSuffix(name, ext) + "strm"
|
||||
size = int64(len(d.getLink(ctx, path)))
|
||||
} else if _, ok := d.downloadSuffix[ext]; ok {
|
||||
size = obj.GetSize()
|
||||
} else {
|
||||
|
||||
}
|
||||
}
|
||||
return model.Object{
|
||||
ID: id,
|
||||
Path: path,
|
||||
Name: name,
|
||||
Size: size,
|
||||
Modified: obj.ModTime(),
|
||||
IsFolder: obj.IsDir(),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Strm) getLink(ctx context.Context, path string) string {
|
||||
finalPath := path
|
||||
if d.EncodePath {
|
||||
@@ -110,7 +112,7 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
|
||||
signPath := sign.Sign(path)
|
||||
finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath)
|
||||
}
|
||||
if d.LocalModel {
|
||||
if d.WithoutUrl {
|
||||
return finalPath
|
||||
}
|
||||
apiUrl := d.SiteUrl
|
||||
@@ -119,7 +121,9 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
|
||||
} else {
|
||||
apiUrl = common.GetApiUrl(ctx)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(finalPath, "/") {
|
||||
finalPath = "/" + finalPath
|
||||
}
|
||||
return fmt.Sprintf("%s/d%s",
|
||||
apiUrl,
|
||||
finalPath)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -68,6 +69,7 @@ func (x *Thunder) Init(ctx context.Context) (err error) {
|
||||
PackageName: "com.xunlei.downloadprovider",
|
||||
UserAgent: "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)",
|
||||
DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)",
|
||||
Space: x.Space,
|
||||
refreshCTokenCk: func(token string) {
|
||||
x.CaptchaToken = token
|
||||
op.MustSaveDriverStorage(x)
|
||||
@@ -167,6 +169,7 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) {
|
||||
UserAgent: x.UserAgent,
|
||||
DownloadUserAgent: x.DownloadUserAgent,
|
||||
UseVideoUrl: x.UseVideoUrl,
|
||||
Space: x.Space,
|
||||
|
||||
refreshCTokenCk: func(token string) {
|
||||
x.CaptchaToken = token
|
||||
@@ -281,7 +284,7 @@ func (xc *XunLeiCommon) Link(ctx context.Context, file model.Obj, args model.Lin
|
||||
_, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetPathParam("fileID", file.GetID())
|
||||
//r.SetQueryParam("space", "")
|
||||
r.SetQueryParam("space", xc.Space)
|
||||
}, &lFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -322,6 +325,7 @@ func (xc *XunLeiCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirNam
|
||||
"kind": FOLDER,
|
||||
"name": dirName,
|
||||
"parent_id": parentDir.GetID(),
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, nil)
|
||||
return err
|
||||
@@ -331,8 +335,9 @@ func (xc *XunLeiCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) erro
|
||||
_, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetBody(&base.Json{
|
||||
"to": base.Json{"parent_id": dstDir.GetID()},
|
||||
"ids": []string{srcObj.GetID()},
|
||||
"to": base.Json{"parent_id": dstDir.GetID()},
|
||||
"ids": []string{srcObj.GetID()},
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, nil)
|
||||
return err
|
||||
@@ -342,7 +347,10 @@ func (xc *XunLeiCommon) Rename(ctx context.Context, srcObj model.Obj, newName st
|
||||
_, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetPathParam("fileID", srcObj.GetID())
|
||||
r.SetBody(&base.Json{"name": newName})
|
||||
r.SetBody(&base.Json{
|
||||
"name": newName,
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, nil)
|
||||
return err
|
||||
}
|
||||
@@ -351,8 +359,9 @@ func (xc *XunLeiCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) erro
|
||||
_, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetBody(&base.Json{
|
||||
"to": base.Json{"parent_id": dstDir.GetID()},
|
||||
"ids": []string{srcObj.GetID()},
|
||||
"to": base.Json{"parent_id": dstDir.GetID()},
|
||||
"ids": []string{srcObj.GetID()},
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, nil)
|
||||
return err
|
||||
@@ -362,6 +371,7 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error {
|
||||
_, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetPathParam("fileID", obj.GetID())
|
||||
r.SetQueryParam("space", xc.Space)
|
||||
r.SetBody("{}")
|
||||
}, nil)
|
||||
return err
|
||||
@@ -387,6 +397,7 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi
|
||||
"size": file.GetSize(),
|
||||
"hash": gcid,
|
||||
"upload_type": UPLOAD_TYPE_RESUMABLE,
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
@@ -430,7 +441,7 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.
|
||||
_, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetQueryParams(map[string]string{
|
||||
"space": "",
|
||||
"space": xc.Space,
|
||||
"__type": "drive",
|
||||
"refresh": "true",
|
||||
"__sync": "true",
|
||||
@@ -440,6 +451,17 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.
|
||||
"limit": "100",
|
||||
"filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`,
|
||||
})
|
||||
// 获取硬盘挂载目录等
|
||||
if xc.Space != "" {
|
||||
r.SetQueryParamsFromValues(url.Values{
|
||||
"with": []string{
|
||||
"withCategoryDiskMountPath",
|
||||
"withCategoryDriveCachePath",
|
||||
"withCategoryHistoryDownloadPath",
|
||||
"withReadOnlyFS",
|
||||
},
|
||||
})
|
||||
}
|
||||
}, &fileList)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -576,6 +598,7 @@ func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, par
|
||||
"name": fileName,
|
||||
"parent_id": parentDir.GetID(),
|
||||
"upload_type": UPLOAD_TYPE_URL,
|
||||
"space": xc.Space,
|
||||
"url": base.Json{
|
||||
"url": fileUrl,
|
||||
},
|
||||
@@ -602,6 +625,7 @@ func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) (
|
||||
"type": "offline",
|
||||
"limit": "10000",
|
||||
"page_token": nextPageToken,
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, &resp)
|
||||
|
||||
@@ -618,6 +642,7 @@ func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string
|
||||
SetQueryParams(map[string]string{
|
||||
"task_ids": strings.Join(taskIDs, ","),
|
||||
"delete_files": strconv.FormatBool(deleteFiles),
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -46,6 +46,8 @@ type ExpertAddition struct {
|
||||
|
||||
//优先使用视频链接代替下载链接
|
||||
UseVideoUrl bool `json:"use_video_url"`
|
||||
|
||||
Space string `json:"space" default:"" help:"device id for remote device"`
|
||||
}
|
||||
|
||||
// 登录特征,用于判断是否重新登录
|
||||
@@ -80,6 +82,8 @@ type Addition struct {
|
||||
CreditKey string `json:"credit_key" help:"credit key,used for login"`
|
||||
// 登录设备ID
|
||||
DeviceID string `json:"device_id" default:""`
|
||||
|
||||
Space string `json:"space" default:"" help:"device id for remote device"`
|
||||
}
|
||||
|
||||
// 登录特征,用于判断是否重新登录
|
||||
@@ -90,7 +94,6 @@ func (i *Addition) GetIdentity() string {
|
||||
var config = driver.Config{
|
||||
Name: "Thunder",
|
||||
LocalSort: true,
|
||||
OnlyProxy: true,
|
||||
}
|
||||
|
||||
var configExpert = driver.Config{
|
||||
|
||||
@@ -68,6 +68,7 @@ type Common struct {
|
||||
UserAgent string
|
||||
DownloadUserAgent string
|
||||
UseVideoUrl bool
|
||||
Space string
|
||||
|
||||
// 验证码token刷新成功回调
|
||||
refreshCTokenCk func(token string)
|
||||
|
||||
@@ -39,7 +39,21 @@ func InitConfig() {
|
||||
if !filepath.IsAbs(dataDir) {
|
||||
flags.DataDir = filepath.Join(pwd, flags.DataDir)
|
||||
}
|
||||
configPath := filepath.Join(flags.DataDir, "config.json")
|
||||
// Determine config file path: use flags.ConfigPath if provided, otherwise default to <dataDir>/config.json
|
||||
configPath := flags.ConfigPath
|
||||
if configPath == "" {
|
||||
configPath = filepath.Join(flags.DataDir, "config.json")
|
||||
} else {
|
||||
// if relative, resolve relative to working directory
|
||||
if !filepath.IsAbs(configPath) {
|
||||
if absPath, err := filepath.Abs(configPath); err == nil {
|
||||
configPath = absPath
|
||||
} else {
|
||||
configPath = filepath.Join(pwd, configPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
configPath = filepath.Clean(configPath)
|
||||
log.Infof("reading config file: %s", configPath)
|
||||
if !utils.Exists(configPath) {
|
||||
log.Infof("config file not exists, creating default config file")
|
||||
@@ -126,6 +140,10 @@ func InitConfig() {
|
||||
log.Fatalf("create temp dir error: %+v", err)
|
||||
}
|
||||
log.Debugf("config: %+v", conf.Conf)
|
||||
|
||||
// Validate and display proxy configuration status
|
||||
validateProxyConfig()
|
||||
|
||||
base.InitClient()
|
||||
initURL()
|
||||
}
|
||||
@@ -165,3 +183,14 @@ func CleanTempDir() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// validateProxyConfig validates proxy configuration and displays status at startup
|
||||
func validateProxyConfig() {
|
||||
if conf.Conf.ProxyAddress != "" {
|
||||
if _, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
|
||||
log.Infof("Proxy enabled: %s", conf.Conf.ProxyAddress)
|
||||
} else {
|
||||
log.Errorf("Invalid proxy address format: %s, error: %v", conf.Conf.ProxyAddress, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
17
internal/cache/typed_cache.go
vendored
17
internal/cache/typed_cache.go
vendored
@@ -43,23 +43,14 @@ func (c *TypedCache[T]) SetTypeWithExpirable(key, typeKey string, value T, exp E
|
||||
}
|
||||
}
|
||||
|
||||
// Prefer to use typeKeys for lookup; if none match, use fallbackTypeKey for lookup
|
||||
func (c *TypedCache[T]) GetType(key, fallbackTypeKey string, typeKeys ...string) (T, bool) {
|
||||
func (c *TypedCache[T]) GetType(key, typeKey string) (T, bool) {
|
||||
c.mu.RLock()
|
||||
cache, exists := c.entries[key]
|
||||
if !exists {
|
||||
c.mu.RUnlock()
|
||||
return *new(T), false
|
||||
}
|
||||
entry, exists := cache[fallbackTypeKey]
|
||||
if len(typeKeys) > 0 {
|
||||
for _, tk := range typeKeys {
|
||||
if entry, exists = cache[tk]; exists {
|
||||
fallbackTypeKey = tk
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
entry, exists := cache[typeKey]
|
||||
if !exists {
|
||||
c.mu.RUnlock()
|
||||
return *new(T), false
|
||||
@@ -72,8 +63,8 @@ func (c *TypedCache[T]) GetType(key, fallbackTypeKey string, typeKeys ...string)
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
if cache[fallbackTypeKey] == entry {
|
||||
delete(cache, fallbackTypeKey)
|
||||
if cache[typeKey] == entry {
|
||||
delete(cache, typeKey)
|
||||
if len(cache) == 0 {
|
||||
delete(c.entries, key)
|
||||
}
|
||||
|
||||
@@ -131,6 +131,7 @@ type Config struct {
|
||||
FTP FTP `json:"ftp" envPrefix:"FTP_"`
|
||||
SFTP SFTP `json:"sftp" envPrefix:"SFTP_"`
|
||||
LastLaunchedVersion string `json:"last_launched_version"`
|
||||
ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"`
|
||||
}
|
||||
|
||||
func DefaultConfig(dataDir string) *Config {
|
||||
@@ -244,5 +245,6 @@ func DefaultConfig(dataDir string) *Config {
|
||||
Listen: ":5222",
|
||||
},
|
||||
LastLaunchedVersion: "",
|
||||
ProxyAddress: "",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,3 +60,7 @@ func DeleteSharingById(id string) error {
|
||||
s := model.SharingDB{ID: id}
|
||||
return errors.WithStack(db.Where(s).Delete(&s).Error)
|
||||
}
|
||||
|
||||
func DeleteSharingsByCreatorId(creatorId uint) error {
|
||||
return errors.WithStack(db.Where("creator_id = ?", creatorId).Delete(&model.SharingDB{}).Error)
|
||||
}
|
||||
|
||||
@@ -17,11 +17,23 @@ type Config struct {
|
||||
ProxyRangeOption bool `json:"-"`
|
||||
// if the driver returns Link without URL, this should be set to true
|
||||
NoLinkURL bool `json:"-"`
|
||||
// LinkCacheType=1 add IP to cache key
|
||||
//
|
||||
// LinkCacheType=2 add UserAgent to cache key
|
||||
LinkCacheType uint8 `json:"-"`
|
||||
// Link cache behaviour:
|
||||
// - LinkCacheAuto: let driver decide per-path (implement driver.LinkCacheModeResolver)
|
||||
// - LinkCacheNone: no extra info added to cache key (default)
|
||||
// - flags (OR-able) can add more attributes to cache key (IP, UA, ...)
|
||||
LinkCacheMode `json:"-"`
|
||||
}
|
||||
type LinkCacheMode int8
|
||||
|
||||
const (
|
||||
LinkCacheAuto LinkCacheMode = -1 // Let the driver decide per-path (use driver.LinkCacheModeResolver)
|
||||
LinkCacheNone LinkCacheMode = 0 // No extra info added to cache key (default)
|
||||
)
|
||||
|
||||
const (
|
||||
LinkCacheIP LinkCacheMode = 1 << iota // include client IP in cache key
|
||||
LinkCacheUA // include User-Agent in cache key
|
||||
)
|
||||
|
||||
func (c Config) MustProxy() bool {
|
||||
return c.OnlyProxy || c.NoLinkURL
|
||||
|
||||
@@ -213,3 +213,8 @@ type WithDetails interface {
|
||||
type Reference interface {
|
||||
InitReference(storage Driver) error
|
||||
}
|
||||
|
||||
type LinkCacheModeResolver interface {
|
||||
// ResolveLinkCacheMode returns the LinkCacheMode for the given path.
|
||||
ResolveLinkCacheMode(path string) LinkCacheMode
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ func (s *Sharing) Valid() bool {
|
||||
if len(s.Files) == 0 {
|
||||
return false
|
||||
}
|
||||
if !s.Creator.CanShare() {
|
||||
if s.Creator == nil || !s.Creator.CanShare() {
|
||||
return false
|
||||
}
|
||||
if s.Expires != nil && !s.Expires.IsZero() && s.Expires.Before(time.Now()) {
|
||||
|
||||
@@ -283,11 +283,15 @@ func HttpClient() *http.Client {
|
||||
}
|
||||
|
||||
func NewHttpClient() *http.Client {
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify},
|
||||
}
|
||||
|
||||
SetProxyIfConfigured(transport)
|
||||
|
||||
return &http.Client{
|
||||
Timeout: time.Hour * 48,
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify},
|
||||
},
|
||||
Timeout: time.Hour * 48,
|
||||
Transport: transport,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,12 +7,15 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
||||
"github.com/go-resty/resty/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -350,3 +353,23 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea
|
||||
// return an io.ReadCloser that is limited to `length` bytes.
|
||||
return &LimitedReadCloser{readCloser, length_int}, nil
|
||||
}
|
||||
|
||||
// SetProxyIfConfigured sets proxy for HTTP Transport if configured
|
||||
func SetProxyIfConfigured(transport *http.Transport) {
|
||||
// If proxy address is configured, override environment variable settings
|
||||
if conf.Conf.ProxyAddress != "" {
|
||||
if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
|
||||
transport.Proxy = http.ProxyURL(proxyURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetRestyProxyIfConfigured sets proxy for Resty client if configured
|
||||
func SetRestyProxyIfConfigured(client *resty.Client) {
|
||||
// If proxy address is configured, override environment variable settings
|
||||
if conf.Conf.ProxyAddress != "" {
|
||||
if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
|
||||
client.SetProxy(proxyURL.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
|
||||
model.WrapObjsName(files)
|
||||
// call hooks
|
||||
go func(reqPath string, files []model.Obj) {
|
||||
HandleObjsUpdateHook(reqPath, files)
|
||||
HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files)
|
||||
}(utils.GetFullPath(storage.GetStorage().MountPath, path), files)
|
||||
|
||||
// sort objs
|
||||
@@ -168,23 +168,19 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li
|
||||
return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status)
|
||||
}
|
||||
|
||||
typeKey := args.Type
|
||||
var typeKeys []string
|
||||
switch storage.Config().LinkCacheType {
|
||||
case 1:
|
||||
if args.IP != "" {
|
||||
typeKey += "/" + args.IP
|
||||
typeKeys = []string{typeKey}
|
||||
}
|
||||
case 2:
|
||||
if ua := args.Header.Get("User-Agent"); ua != "" {
|
||||
typeKey += "/" + ua
|
||||
typeKeys = []string{typeKey}
|
||||
}
|
||||
mode := storage.Config().LinkCacheMode
|
||||
if mode == -1 {
|
||||
mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path)
|
||||
}
|
||||
typeKey := args.Type
|
||||
if mode&driver.LinkCacheIP == 1 {
|
||||
typeKey += "/" + args.IP
|
||||
}
|
||||
if mode&driver.LinkCacheUA == 1 {
|
||||
typeKey += "/" + args.Header.Get("User-Agent")
|
||||
}
|
||||
|
||||
key := Key(storage, path)
|
||||
if ol, exists := Cache.linkCache.GetType(key, args.Type, typeKeys...); exists {
|
||||
if ol, exists := Cache.linkCache.GetType(key, typeKey); exists {
|
||||
if ol.link.Expiration != nil ||
|
||||
ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference {
|
||||
return ol.link, ol.obj, nil
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package op
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -13,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
// Obj
|
||||
type ObjsUpdateHook = func(parent string, objs []model.Obj)
|
||||
type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj)
|
||||
|
||||
var (
|
||||
objsUpdateHooks = make([]ObjsUpdateHook, 0)
|
||||
@@ -23,9 +24,9 @@ func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
|
||||
objsUpdateHooks = append(objsUpdateHooks, hook)
|
||||
}
|
||||
|
||||
func HandleObjsUpdateHook(parent string, objs []model.Obj) {
|
||||
func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) {
|
||||
for _, hook := range objsUpdateHooks {
|
||||
hook(parent, objs)
|
||||
hook(ctx, parent, objs)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -137,3 +137,7 @@ func DeleteSharing(sid string) error {
|
||||
sharingCache.Del(sid)
|
||||
return db.DeleteSharingById(sid)
|
||||
}
|
||||
|
||||
func DeleteSharingsByCreatorId(creatorId uint) error {
|
||||
return db.DeleteSharingsByCreatorId(creatorId)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/singleflight"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var userG singleflight.Group[*model.User]
|
||||
@@ -78,6 +79,9 @@ func DeleteUserById(id uint) error {
|
||||
return errs.DeleteAdminOrGuest
|
||||
}
|
||||
Cache.DeleteUser(old.Username)
|
||||
if err := DeleteSharingsByCreatorId(id); err != nil {
|
||||
return errors.WithMessage(err, "failed to delete user's sharings")
|
||||
}
|
||||
return db.DeleteUserById(id)
|
||||
}
|
||||
|
||||
|
||||
@@ -199,14 +199,13 @@ func Config(ctx context.Context) searcher.Config {
|
||||
return instance.Config()
|
||||
}
|
||||
|
||||
func Update(parent string, objs []model.Obj) {
|
||||
func Update(ctx context.Context, parent string, objs []model.Obj) {
|
||||
if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() {
|
||||
return
|
||||
}
|
||||
if isIgnorePath(parent) {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
// only update when index have built
|
||||
progress, err := Progress()
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user