feat(thunder): allow setting space (#1219)

allows access to files on remote devices via Thunder's tunneling service.

drivers/123/driver.go

@@ -41,7 +41,9 @@ func (d *Pan123) GetAddition() driver.Additional {
 }
 
 func (d *Pan123) Init(ctx context.Context) error {
-	_, err := d.Request(UserInfo, http.MethodGet, nil, nil)
+	_, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) {
+		req.SetHeader("platform", "web")
+	}, nil)
 	return err
 }
 

drivers/123/meta.go

@@ -12,7 +12,8 @@ type Addition struct {
 	//OrderBy        string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"`
 	//OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
 	AccessToken  string
-	UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
+	UploadThread int    `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
+	Platform     string `json:"platform" type:"string" default:"web" help:"the platform header value, sent with API requests"`
 }
 
 var config = driver.Config{

drivers/123/util.go

@@ -203,7 +203,7 @@ do:
 		"referer":       "https://www.123pan.com/",
 		"authorization": "Bearer " + d.AccessToken,
 		"user-agent":    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) openlist-client",
-		"platform":      "web",
+		"platform":      d.Platform,
 		"app-version":   "3",
 		//"user-agent":    base.UserAgent,
 	})

drivers/strm/driver.go

@@ -15,6 +15,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/OpenListTeam/OpenList/v4/server/common"
+	log "github.com/sirupsen/logrus"
 )
 
 type Strm struct {
@@ -40,6 +41,9 @@ func (d *Strm) Init(ctx context.Context) error {
 	if d.Paths == "" {
 		return errors.New("paths is required")
 	}
+	if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 {
+		return errors.New("SaveStrmLocalPath is required")
+	}
 	d.pathMap = make(map[string][]string)
 	for _, path := range strings.Split(d.Paths, "\n") {
 		path = strings.TrimSpace(path)
@@ -48,6 +52,11 @@ func (d *Strm) Init(ctx context.Context) error {
 		}
 		k, v := getPair(path)
 		d.pathMap[k] = append(d.pathMap[k], v)
+		err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
+		if err != nil {
+			log.Errorf("insert strmTrie error: %v", err)
+			continue
+		}
 	}
 	if len(d.pathMap) == 1 {
 		for k := range d.pathMap {
@@ -87,6 +96,9 @@ func (d *Strm) Drop(ctx context.Context) error {
 	d.pathMap = nil
 	d.downloadSuffix = nil
 	d.supportSuffix = nil
+	for _, path := range strings.Split(d.Paths, "\n") {
+		RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
+	}
 	return nil
 }
 

drivers/strm/hook.go

@@ -0,0 +1,164 @@
+package strm
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	stdpath "path"
+	"strings"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	"github.com/OpenListTeam/OpenList/v4/internal/stream"
+	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	log "github.com/sirupsen/logrus"
+	"github.com/tchap/go-patricia/v2/patricia"
+)
+
+var strmTrie = patricia.NewTrie()
+
+func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) {
+	path = utils.FixAndCleanPath(path)
+	updateLocal := func(driver *Strm, basePath string, objs []model.Obj) {
+		relParent := strings.TrimPrefix(basePath, driver.MountPath)
+		localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent)
+		for _, obj := range objs {
+			localPath := stdpath.Join(localParentPath, obj.GetName())
+			generateStrm(ctx, driver, obj, localPath)
+		}
+		deleteExtraFiles(localParentPath, objs)
+	}
+
+	_ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error {
+		strmDrivers := item.([]*Strm)
+		needPath := string(needPathPrefix)
+		restPath := strings.TrimPrefix(path, needPath)
+		if len(restPath) > 0 && restPath[0] != '/' {
+			return nil
+		}
+		for _, strmDriver := range strmDrivers {
+			strmObjs, _ := utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) {
+				ret := strmDriver.convert2strmObj(ctx, path, obj)
+				return &ret, nil
+			})
+			updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs)
+		}
+		return nil
+	})
+}
+
+func InsertStrm(dstPath string, d *Strm) error {
+	prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
+	existing := strmTrie.Get(prefix)
+
+	if existing == nil {
+		if !strmTrie.Insert(prefix, []*Strm{d}) {
+			return errors.New("failed to insert strm")
+		}
+		return nil
+	}
+	if lst, ok := existing.([]*Strm); ok {
+		strmTrie.Set(prefix, append(lst, d))
+	} else {
+		return errors.New("invalid trie item type")
+	}
+
+	return nil
+}
+
+func RemoveStrm(dstPath string, d *Strm) {
+	prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
+	existing := strmTrie.Get(prefix)
+	if existing == nil {
+		return
+	}
+	lst, ok := existing.([]*Strm)
+	if !ok {
+		return
+	}
+	if len(lst) == 1 && lst[0] == d {
+		strmTrie.Delete(prefix)
+		return
+	}
+
+	for i, di := range lst {
+		if di == d {
+			newList := append(lst[:i], lst[i+1:]...)
+			strmTrie.Set(prefix, newList)
+			return
+		}
+	}
+}
+
+func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) {
+	link, err := driver.Link(ctx, obj, model.LinkArgs{})
+	if err != nil {
+		log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err)
+		return
+	}
+	seekableStream, err := stream.NewSeekableStream(&stream.FileStream{
+		Obj: obj,
+		Ctx: ctx,
+	}, link)
+	if err != nil {
+		_ = link.Close()
+		log.Warnf("failed to generate strm of obj %s: failed to get seekable stream: %v", localPath, err)
+		return
+	}
+	defer seekableStream.Close()
+	file, err := utils.CreateNestedFile(localPath)
+	if err != nil {
+		log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err)
+		return
+	}
+	defer file.Close()
+	if _, err := io.Copy(file, seekableStream); err != nil {
+		log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err)
+	}
+}
+
+func deleteExtraFiles(localPath string, objs []model.Obj) {
+	localFiles, err := getLocalFiles(localPath)
+	if err != nil {
+		log.Errorf("Failed to read local files from %s: %v", localPath, err)
+		return
+	}
+
+	objsSet := make(map[string]struct{})
+	for _, obj := range objs {
+		if obj.IsDir() {
+			continue
+		}
+		objsSet[stdpath.Join(localPath, obj.GetName())] = struct{}{}
+	}
+
+	for _, localFile := range localFiles {
+		if _, exists := objsSet[localFile]; !exists {
+			err := os.Remove(localFile)
+			if err != nil {
+				log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err)
+			} else {
+				log.Infof("Deleted file %s", localFile)
+			}
+		}
+	}
+}
+
+func getLocalFiles(localPath string) ([]string, error) {
+	var files []string
+	entries, err := os.ReadDir(localPath)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			files = append(files, stdpath.Join(localPath, entry.Name()))
+		}
+	}
+	return files, nil
+}
+
+func init() {
+	op.RegisterObjsUpdateHook(UpdateLocalStrm)
+}

drivers/strm/meta.go

@@ -11,7 +11,9 @@ type Addition struct {
 	FilterFileTypes   string `json:"filterFileTypes" type:"text" default:"strm" required:"false" help:"Supports suffix name of strm file"`
 	DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass" required:"false" help:"Files need to download with strm (usally subtitles)"`
 	EncodePath        bool   `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"`
-	LocalModel        bool   `json:"localModel" default:"false" help:"enable local mode"`
+	WithoutUrl        bool   `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"`
+	SaveStrmToLocal   bool   `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"`
+	SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"`
 }
 
 var config = driver.Config{

drivers/strm/util.go

@@ -61,36 +61,12 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
 
 	var validObjs []model.Obj
 	for _, obj := range objs {
-		id, name, path := "", obj.GetName(), ""
-		size := int64(0)
-		if !obj.IsDir() {
-			path = stdpath.Join(reqPath, obj.GetName())
-			ext := strings.ToLower(utils.Ext(name))
-			if _, ok := d.supportSuffix[ext]; ok {
-				id = "strm"
-				name = strings.TrimSuffix(name, ext) + "strm"
-				size = int64(len(d.getLink(ctx, path)))
-			} else if _, ok := d.downloadSuffix[ext]; ok {
-				size = obj.GetSize()
-			} else {
-				continue
-			}
-		}
-		objRes := model.Object{
-			ID:       id,
-			Path:     path,
-			Name:     name,
-			Size:     size,
-			Modified: obj.ModTime(),
-			IsFolder: obj.IsDir(),
-		}
-
+		objRes := d.convert2strmObj(ctx, reqPath, obj)
 		thumb, ok := model.GetThumb(obj)
 		if !ok {
 			validObjs = append(validObjs, &objRes)
 			continue
 		}
-
 		validObjs = append(validObjs, &model.ObjThumb{
 			Object: objRes,
 			Thumbnail: model.Thumbnail{
@@ -101,6 +77,32 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
 	return validObjs, nil
 }
 
+func (d *Strm) convert2strmObj(ctx context.Context, reqPath string, obj model.Obj) model.Object {
+	id, name, path := "", obj.GetName(), ""
+	size := int64(0)
+	if !obj.IsDir() {
+		path = stdpath.Join(reqPath, obj.GetName())
+		ext := strings.ToLower(utils.Ext(name))
+		if _, ok := d.supportSuffix[ext]; ok {
+			id = "strm"
+			name = strings.TrimSuffix(name, ext) + "strm"
+			size = int64(len(d.getLink(ctx, path)))
+		} else if _, ok := d.downloadSuffix[ext]; ok {
+			size = obj.GetSize()
+		} else {
+
+		}
+	}
+	return model.Object{
+		ID:       id,
+		Path:     path,
+		Name:     name,
+		Size:     size,
+		Modified: obj.ModTime(),
+		IsFolder: obj.IsDir(),
+	}
+}
+
 func (d *Strm) getLink(ctx context.Context, path string) string {
 	finalPath := path
 	if d.EncodePath {
@@ -110,7 +112,7 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
 		signPath := sign.Sign(path)
 		finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath)
 	}
-	if d.LocalModel {
+	if d.WithoutUrl {
 		return finalPath
 	}
 	apiUrl := d.SiteUrl
@@ -119,7 +121,9 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
 	} else {
 		apiUrl = common.GetApiUrl(ctx)
 	}
-
+	if !strings.HasPrefix(finalPath, "/") {
+		finalPath = "/" + finalPath
+	}
 	return fmt.Sprintf("%s/d%s",
 		apiUrl,
 		finalPath)

drivers/thunder/driver.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 
@@ -68,6 +69,7 @@ func (x *Thunder) Init(ctx context.Context) (err error) {
 				PackageName:       "com.xunlei.downloadprovider",
 				UserAgent:         "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)",
 				DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)",
+				Space:             x.Space,
 				refreshCTokenCk: func(token string) {
 					x.CaptchaToken = token
 					op.MustSaveDriverStorage(x)
@@ -167,6 +169,7 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) {
 				UserAgent:         x.UserAgent,
 				DownloadUserAgent: x.DownloadUserAgent,
 				UseVideoUrl:       x.UseVideoUrl,
+				Space:             x.Space,
 
 				refreshCTokenCk: func(token string) {
 					x.CaptchaToken = token
@@ -281,7 +284,7 @@ func (xc *XunLeiCommon) Link(ctx context.Context, file model.Obj, args model.Lin
 	_, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) {
 		r.SetContext(ctx)
 		r.SetPathParam("fileID", file.GetID())
-		//r.SetQueryParam("space", "")
+		r.SetQueryParam("space", xc.Space)
 	}, &lFile)
 	if err != nil {
 		return nil, err
@@ -322,6 +325,7 @@ func (xc *XunLeiCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirNam
 			"kind":      FOLDER,
 			"name":      dirName,
 			"parent_id": parentDir.GetID(),
+			"space":     xc.Space,
 		})
 	}, nil)
 	return err
@@ -331,8 +335,9 @@ func (xc *XunLeiCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) erro
 	_, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) {
 		r.SetContext(ctx)
 		r.SetBody(&base.Json{
-			"to":  base.Json{"parent_id": dstDir.GetID()},
-			"ids": []string{srcObj.GetID()},
+			"to":    base.Json{"parent_id": dstDir.GetID()},
+			"ids":   []string{srcObj.GetID()},
+			"space": xc.Space,
 		})
 	}, nil)
 	return err
@@ -342,7 +347,10 @@ func (xc *XunLeiCommon) Rename(ctx context.Context, srcObj model.Obj, newName st
 	_, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) {
 		r.SetContext(ctx)
 		r.SetPathParam("fileID", srcObj.GetID())
-		r.SetBody(&base.Json{"name": newName})
+		r.SetBody(&base.Json{
+			"name":  newName,
+			"space": xc.Space,
+		})
 	}, nil)
 	return err
 }
@@ -351,8 +359,9 @@ func (xc *XunLeiCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) erro
 	_, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) {
 		r.SetContext(ctx)
 		r.SetBody(&base.Json{
-			"to":  base.Json{"parent_id": dstDir.GetID()},
-			"ids": []string{srcObj.GetID()},
+			"to":    base.Json{"parent_id": dstDir.GetID()},
+			"ids":   []string{srcObj.GetID()},
+			"space": xc.Space,
 		})
 	}, nil)
 	return err
@@ -362,6 +371,7 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error {
 	_, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) {
 		r.SetContext(ctx)
 		r.SetPathParam("fileID", obj.GetID())
+		r.SetQueryParam("space", xc.Space)
 		r.SetBody("{}")
 	}, nil)
 	return err
@@ -387,6 +397,7 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi
 			"size":        file.GetSize(),
 			"hash":        gcid,
 			"upload_type": UPLOAD_TYPE_RESUMABLE,
+			"space":       xc.Space,
 		})
 	}, &resp)
 	if err != nil {
@@ -430,7 +441,7 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.
 		_, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) {
 			r.SetContext(ctx)
 			r.SetQueryParams(map[string]string{
-				"space":      "",
+				"space":      xc.Space,
 				"__type":     "drive",
 				"refresh":    "true",
 				"__sync":     "true",
@@ -440,6 +451,17 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.
 				"limit":      "100",
 				"filters":    `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`,
 			})
+			// 获取硬盘挂载目录等
+			if xc.Space != "" {
+				r.SetQueryParamsFromValues(url.Values{
+					"with": []string{
+						"withCategoryDiskMountPath",
+						"withCategoryDriveCachePath",
+						"withCategoryHistoryDownloadPath",
+						"withReadOnlyFS",
+					},
+				})
+			}
 		}, &fileList)
 		if err != nil {
 			return nil, err
@@ -576,6 +598,7 @@ func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, par
 			"name":        fileName,
 			"parent_id":   parentDir.GetID(),
 			"upload_type": UPLOAD_TYPE_URL,
+			"space":       xc.Space,
 			"url": base.Json{
 				"url": fileUrl,
 			},
@@ -602,6 +625,7 @@ func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) (
 				"type":       "offline",
 				"limit":      "10000",
 				"page_token": nextPageToken,
+				"space":      xc.Space,
 			})
 	}, &resp)
 
@@ -618,6 +642,7 @@ func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string
 			SetQueryParams(map[string]string{
 				"task_ids":     strings.Join(taskIDs, ","),
 				"delete_files": strconv.FormatBool(deleteFiles),
+				"space":        xc.Space,
 			})
 	}, nil)
 	if err != nil {

drivers/thunder/meta.go

@@ -46,6 +46,8 @@ type ExpertAddition struct {
 
 	//优先使用视频链接代替下载链接
 	UseVideoUrl bool `json:"use_video_url"`
+
+	Space string `json:"space" default:"" help:"device id for remote device"`
 }
 
 // 登录特征,用于判断是否重新登录
@@ -80,6 +82,8 @@ type Addition struct {
 	CreditKey string `json:"credit_key" help:"credit key,used for login"`
 	// 登录设备ID
 	DeviceID string `json:"device_id" default:""`
+
+	Space string `json:"space" default:"" help:"device id for remote device"`
 }
 
 // 登录特征,用于判断是否重新登录
@@ -90,7 +94,6 @@ func (i *Addition) GetIdentity() string {
 var config = driver.Config{
 	Name:      "Thunder",
 	LocalSort: true,
-	OnlyProxy: true,
 }
 
 var configExpert = driver.Config{

drivers/thunder/util.go

@@ -68,6 +68,7 @@ type Common struct {
 	UserAgent         string
 	DownloadUserAgent string
 	UseVideoUrl       bool
+	Space             string
 
 	// 验证码token刷新成功回调
 	refreshCTokenCk func(token string)

internal/op/fs.go

@@ -57,7 +57,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
 		model.WrapObjsName(files)
 		// call hooks
 		go func(reqPath string, files []model.Obj) {
-			HandleObjsUpdateHook(reqPath, files)
+			HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files)
 		}(utils.GetFullPath(storage.GetStorage().MountPath, path), files)
 
 		// sort objs

internal/op/hook.go

@@ -1,6 +1,7 @@
 package op
 
 import (
+	"context"
 	"regexp"
 	"strings"
 
@@ -13,7 +14,7 @@ import (
 )
 
 // Obj
-type ObjsUpdateHook = func(parent string, objs []model.Obj)
+type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj)
 
 var (
 	objsUpdateHooks = make([]ObjsUpdateHook, 0)
@@ -23,9 +24,9 @@ func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
 	objsUpdateHooks = append(objsUpdateHooks, hook)
 }
 
-func HandleObjsUpdateHook(parent string, objs []model.Obj) {
+func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) {
 	for _, hook := range objsUpdateHooks {
-		hook(parent, objs)
+		hook(ctx, parent, objs)
 	}
 }
 

internal/search/build.go

@@ -199,14 +199,13 @@ func Config(ctx context.Context) searcher.Config {
 	return instance.Config()
 }
 
-func Update(parent string, objs []model.Obj) {
+func Update(ctx context.Context, parent string, objs []model.Obj) {
 	if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() {
 		return
 	}
 	if isIgnorePath(parent) {
 		return
 	}
-	ctx := context.Background()
 	// only update when index have built
 	progress, err := Progress()
 	if err != nil {