feat(strm): strm local file (#1127)

* feat(strm): strm local file

* feat: 代码优化

* feat: 访问被strm挂载路径时也更新

* fix: 路径最后带/判断缺失

* fix: 路径最后带/判断缺失

* refactor

* refactor

* fix: close seekable-stream in `generateStrm`

* refactor: lazy create local file

* 优化路径判断

---------

Co-authored-by: KirCute <kircute@foxmail.com>

drivers/strm/driver.go

@@ -15,6 +15,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/OpenListTeam/OpenList/v4/server/common"
+	log "github.com/sirupsen/logrus"
 )
 
 type Strm struct {
@@ -40,6 +41,9 @@ func (d *Strm) Init(ctx context.Context) error {
 	if d.Paths == "" {
 		return errors.New("paths is required")
 	}
+	if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 {
+		return errors.New("SaveStrmLocalPath is required")
+	}
 	d.pathMap = make(map[string][]string)
 	for _, path := range strings.Split(d.Paths, "\n") {
 		path = strings.TrimSpace(path)
@@ -48,6 +52,11 @@ func (d *Strm) Init(ctx context.Context) error {
 		}
 		k, v := getPair(path)
 		d.pathMap[k] = append(d.pathMap[k], v)
+		err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
+		if err != nil {
+			log.Errorf("insert strmTrie error: %v", err)
+			continue
+		}
 	}
 	if len(d.pathMap) == 1 {
 		for k := range d.pathMap {
@@ -87,6 +96,9 @@ func (d *Strm) Drop(ctx context.Context) error {
 	d.pathMap = nil
 	d.downloadSuffix = nil
 	d.supportSuffix = nil
+	for _, path := range strings.Split(d.Paths, "\n") {
+		RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
+	}
 	return nil
 }
 

drivers/strm/hook.go

@@ -0,0 +1,164 @@
+package strm
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	stdpath "path"
+	"strings"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	"github.com/OpenListTeam/OpenList/v4/internal/stream"
+	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	log "github.com/sirupsen/logrus"
+	"github.com/tchap/go-patricia/v2/patricia"
+)
+
+var strmTrie = patricia.NewTrie()
+
+func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) {
+	path = utils.FixAndCleanPath(path)
+	updateLocal := func(driver *Strm, basePath string, objs []model.Obj) {
+		relParent := strings.TrimPrefix(basePath, driver.MountPath)
+		localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent)
+		for _, obj := range objs {
+			localPath := stdpath.Join(localParentPath, obj.GetName())
+			generateStrm(ctx, driver, obj, localPath)
+		}
+		deleteExtraFiles(localParentPath, objs)
+	}
+
+	_ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error {
+		strmDrivers := item.([]*Strm)
+		needPath := string(needPathPrefix)
+		restPath := strings.TrimPrefix(path, needPath)
+		if len(restPath) > 0 && restPath[0] != '/' {
+			return nil
+		}
+		for _, strmDriver := range strmDrivers {
+			strmObjs, _ := utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) {
+				ret := strmDriver.convert2strmObj(ctx, path, obj)
+				return &ret, nil
+			})
+			updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs)
+		}
+		return nil
+	})
+}
+
+func InsertStrm(dstPath string, d *Strm) error {
+	prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
+	existing := strmTrie.Get(prefix)
+
+	if existing == nil {
+		if !strmTrie.Insert(prefix, []*Strm{d}) {
+			return errors.New("failed to insert strm")
+		}
+		return nil
+	}
+	if lst, ok := existing.([]*Strm); ok {
+		strmTrie.Set(prefix, append(lst, d))
+	} else {
+		return errors.New("invalid trie item type")
+	}
+
+	return nil
+}
+
+func RemoveStrm(dstPath string, d *Strm) {
+	prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
+	existing := strmTrie.Get(prefix)
+	if existing == nil {
+		return
+	}
+	lst, ok := existing.([]*Strm)
+	if !ok {
+		return
+	}
+	if len(lst) == 1 && lst[0] == d {
+		strmTrie.Delete(prefix)
+		return
+	}
+
+	for i, di := range lst {
+		if di == d {
+			newList := append(lst[:i], lst[i+1:]...)
+			strmTrie.Set(prefix, newList)
+			return
+		}
+	}
+}
+
+func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) {
+	link, err := driver.Link(ctx, obj, model.LinkArgs{})
+	if err != nil {
+		log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err)
+		return
+	}
+	seekableStream, err := stream.NewSeekableStream(&stream.FileStream{
+		Obj: obj,
+		Ctx: ctx,
+	}, link)
+	if err != nil {
+		_ = link.Close()
+		log.Warnf("failed to generate strm of obj %s: failed to get seekable stream: %v", localPath, err)
+		return
+	}
+	defer seekableStream.Close()
+	file, err := utils.CreateNestedFile(localPath)
+	if err != nil {
+		log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err)
+		return
+	}
+	defer file.Close()
+	if _, err := io.Copy(file, seekableStream); err != nil {
+		log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err)
+	}
+}
+
+func deleteExtraFiles(localPath string, objs []model.Obj) {
+	localFiles, err := getLocalFiles(localPath)
+	if err != nil {
+		log.Errorf("Failed to read local files from %s: %v", localPath, err)
+		return
+	}
+
+	objsSet := make(map[string]struct{})
+	for _, obj := range objs {
+		if obj.IsDir() {
+			continue
+		}
+		objsSet[stdpath.Join(localPath, obj.GetName())] = struct{}{}
+	}
+
+	for _, localFile := range localFiles {
+		if _, exists := objsSet[localFile]; !exists {
+			err := os.Remove(localFile)
+			if err != nil {
+				log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err)
+			} else {
+				log.Infof("Deleted file %s", localFile)
+			}
+		}
+	}
+}
+
+func getLocalFiles(localPath string) ([]string, error) {
+	var files []string
+	entries, err := os.ReadDir(localPath)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			files = append(files, stdpath.Join(localPath, entry.Name()))
+		}
+	}
+	return files, nil
+}
+
+func init() {
+	op.RegisterObjsUpdateHook(UpdateLocalStrm)
+}

drivers/strm/meta.go

@@ -11,7 +11,9 @@ type Addition struct {
 	FilterFileTypes   string `json:"filterFileTypes" type:"text" default:"strm" required:"false" help:"Supports suffix name of strm file"`
 	DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass" required:"false" help:"Files need to download with strm (usally subtitles)"`
 	EncodePath        bool   `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"`
-	LocalModel        bool   `json:"localModel" default:"false" help:"enable local mode"`
+	WithoutUrl        bool   `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"`
+	SaveStrmToLocal   bool   `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"`
+	SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"`
 }
 
 var config = driver.Config{

drivers/strm/util.go

@@ -61,36 +61,12 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
 
 	var validObjs []model.Obj
 	for _, obj := range objs {
-		id, name, path := "", obj.GetName(), ""
+		objRes := d.convert2strmObj(ctx, reqPath, obj)
-		size := int64(0)
-		if !obj.IsDir() {
-			path = stdpath.Join(reqPath, obj.GetName())
-			ext := strings.ToLower(utils.Ext(name))
-			if _, ok := d.supportSuffix[ext]; ok {
-				id = "strm"
-				name = strings.TrimSuffix(name, ext) + "strm"
-				size = int64(len(d.getLink(ctx, path)))
-			} else if _, ok := d.downloadSuffix[ext]; ok {
-				size = obj.GetSize()
-			} else {
-				continue
-			}
-		}
-		objRes := model.Object{
-			ID:       id,
-			Path:     path,
-			Name:     name,
-			Size:     size,
-			Modified: obj.ModTime(),
-			IsFolder: obj.IsDir(),
-		}
-
 		thumb, ok := model.GetThumb(obj)
 		if !ok {
 			validObjs = append(validObjs, &objRes)
 			continue
 		}
-
 		validObjs = append(validObjs, &model.ObjThumb{
 			Object: objRes,
 			Thumbnail: model.Thumbnail{
@@ -101,6 +77,32 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
 	return validObjs, nil
 }
 
+func (d *Strm) convert2strmObj(ctx context.Context, reqPath string, obj model.Obj) model.Object {
+	id, name, path := "", obj.GetName(), ""
+	size := int64(0)
+	if !obj.IsDir() {
+		path = stdpath.Join(reqPath, obj.GetName())
+		ext := strings.ToLower(utils.Ext(name))
+		if _, ok := d.supportSuffix[ext]; ok {
+			id = "strm"
+			name = strings.TrimSuffix(name, ext) + "strm"
+			size = int64(len(d.getLink(ctx, path)))
+		} else if _, ok := d.downloadSuffix[ext]; ok {
+			size = obj.GetSize()
+		} else {
+
+		}
+	}
+	return model.Object{
+		ID:       id,
+		Path:     path,
+		Name:     name,
+		Size:     size,
+		Modified: obj.ModTime(),
+		IsFolder: obj.IsDir(),
+	}
+}
+
 func (d *Strm) getLink(ctx context.Context, path string) string {
 	finalPath := path
 	if d.EncodePath {
@@ -110,7 +112,7 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
 		signPath := sign.Sign(path)
 		finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath)
 	}
-	if d.LocalModel {
+	if d.WithoutUrl {
 		return finalPath
 	}
 	apiUrl := d.SiteUrl
@@ -119,7 +121,9 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
 	} else {
 		apiUrl = common.GetApiUrl(ctx)
 	}
-
+	if !strings.HasPrefix(finalPath, "/") {
+		finalPath = "/" + finalPath
+	}
 	return fmt.Sprintf("%s/d%s",
 		apiUrl,
 		finalPath)

internal/op/fs.go

@@ -57,7 +57,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
 		model.WrapObjsName(files)
 		// call hooks
 		go func(reqPath string, files []model.Obj) {
-			HandleObjsUpdateHook(reqPath, files)
+			HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files)
 		}(utils.GetFullPath(storage.GetStorage().MountPath, path), files)
 
 		// sort objs

internal/op/hook.go

@@ -1,6 +1,7 @@
 package op
 
 import (
+	"context"
 	"regexp"
 	"strings"
 
@@ -13,7 +14,7 @@ import (
 )
 
 // Obj
-type ObjsUpdateHook = func(parent string, objs []model.Obj)
+type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj)
 
 var (
 	objsUpdateHooks = make([]ObjsUpdateHook, 0)
@@ -23,9 +24,9 @@ func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
 	objsUpdateHooks = append(objsUpdateHooks, hook)
 }
 
-func HandleObjsUpdateHook(parent string, objs []model.Obj) {
+func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) {
 	for _, hook := range objsUpdateHooks {
-		hook(parent, objs)
+		hook(ctx, parent, objs)
 	}
 }
 

internal/search/build.go

@@ -199,14 +199,13 @@ func Config(ctx context.Context) searcher.Config {
 	return instance.Config()
 }
 
-func Update(parent string, objs []model.Obj) {
+func Update(ctx context.Context, parent string, objs []model.Obj) {
 	if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() {
 		return
 	}
 	if isIgnorePath(parent) {
 		return
 	}
-	ctx := context.Background()
 	// only update when index have built
 	progress, err := Progress()
 	if err != nil {