From 66d98090574561f6de92bd11e1756793c78ebc44 Mon Sep 17 00:00:00 2001 From: Seven <53081179+Seven66677731@users.noreply.github.com> Date: Mon, 3 Nov 2025 10:48:15 +0800 Subject: [PATCH] feat(strm): strm local file (#1127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(strm): strm local file * feat: 代码优化 * feat: 访问被strm挂载路径时也更新 * fix: 路径最后带/判断缺失 * fix: 路径最后带/判断缺失 * refactor * refactor * fix: close seekable-stream in `generateStrm` * refactor: lazy create local file * 优化路径判断 --------- Co-authored-by: KirCute --- drivers/strm/driver.go | 12 +++ drivers/strm/hook.go | 164 +++++++++++++++++++++++++++++++++++++++ drivers/strm/meta.go | 4 +- drivers/strm/util.go | 58 +++++++------- internal/op/fs.go | 2 +- internal/op/hook.go | 7 +- internal/search/build.go | 3 +- 7 files changed, 216 insertions(+), 34 deletions(-) create mode 100644 drivers/strm/hook.go diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index e4482cf9..2caf18ef 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + log "github.com/sirupsen/logrus" ) type Strm struct { @@ -40,6 +41,9 @@ func (d *Strm) Init(ctx context.Context) error { if d.Paths == "" { return errors.New("paths is required") } + if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 { + return errors.New("SaveStrmLocalPath is required") + } d.pathMap = make(map[string][]string) for _, path := range strings.Split(d.Paths, "\n") { path = strings.TrimSpace(path) @@ -48,6 +52,11 @@ func (d *Strm) Init(ctx context.Context) error { } k, v := getPair(path) d.pathMap[k] = append(d.pathMap[k], v) + err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) + if err != nil { + log.Errorf("insert strmTrie error: %v", err) + continue + } } if len(d.pathMap) == 1 { for k := range d.pathMap { @@ -87,6 +96,9 @@ func (d *Strm) Drop(ctx context.Context) error { d.pathMap = nil d.downloadSuffix = nil d.supportSuffix = nil + for _, path := range strings.Split(d.Paths, "\n") { + RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) + } return nil } diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go new file mode 100644 index 00000000..7512d2a2 --- /dev/null +++ b/drivers/strm/hook.go @@ -0,0 +1,164 @@ +package strm + +import ( + "context" + "errors" + "io" + "os" + stdpath "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" + "github.com/tchap/go-patricia/v2/patricia" +) + +var strmTrie = patricia.NewTrie() + +func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) { + path = utils.FixAndCleanPath(path) + updateLocal := func(driver *Strm, basePath string, objs []model.Obj) { + relParent := strings.TrimPrefix(basePath, driver.MountPath) + localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent) + for _, obj := range objs { + localPath := stdpath.Join(localParentPath, obj.GetName()) + generateStrm(ctx, driver, obj, localPath) + } + deleteExtraFiles(localParentPath, objs) + } + + _ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error { + strmDrivers := item.([]*Strm) + needPath := string(needPathPrefix) + restPath := strings.TrimPrefix(path, needPath) + if len(restPath) > 0 && restPath[0] != '/' { + return nil + } + for _, strmDriver := range strmDrivers { + strmObjs, _ := utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) { + ret := strmDriver.convert2strmObj(ctx, path, obj) + return &ret, nil + }) + updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs) + } + return nil + }) +} + +func InsertStrm(dstPath string, d *Strm) error { + prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) + existing := strmTrie.Get(prefix) + + if existing == nil { + if !strmTrie.Insert(prefix, []*Strm{d}) { + return errors.New("failed to insert strm") + } + return nil + } + if lst, ok := existing.([]*Strm); ok { + strmTrie.Set(prefix, append(lst, d)) + } else { + return errors.New("invalid trie item type") + } + + return nil +} + +func RemoveStrm(dstPath string, d *Strm) { + prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) + existing := strmTrie.Get(prefix) + if existing == nil { + return + } + lst, ok := existing.([]*Strm) + if !ok { + return + } + if len(lst) == 1 && lst[0] == d { + strmTrie.Delete(prefix) + return + } + + for i, di := range lst { + if di == d { + newList := append(lst[:i], lst[i+1:]...) + strmTrie.Set(prefix, newList) + return + } + } +} + +func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) { + link, err := driver.Link(ctx, obj, model.LinkArgs{}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) + return + } + seekableStream, err := stream.NewSeekableStream(&stream.FileStream{ + Obj: obj, + Ctx: ctx, + }, link) + if err != nil { + _ = link.Close() + log.Warnf("failed to generate strm of obj %s: failed to get seekable stream: %v", localPath, err) + return + } + defer seekableStream.Close() + file, err := utils.CreateNestedFile(localPath) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) + return + } + defer file.Close() + if _, err := io.Copy(file, seekableStream); err != nil { + log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) + } +} + +func deleteExtraFiles(localPath string, objs []model.Obj) { + localFiles, err := getLocalFiles(localPath) + if err != nil { + log.Errorf("Failed to read local files from %s: %v", localPath, err) + return + } + + objsSet := make(map[string]struct{}) + for _, obj := range objs { + if obj.IsDir() { + continue + } + objsSet[stdpath.Join(localPath, obj.GetName())] = struct{}{} + } + + for _, localFile := range localFiles { + if _, exists := objsSet[localFile]; !exists { + err := os.Remove(localFile) + if err != nil { + log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err) + } else { + log.Infof("Deleted file %s", localFile) + } + } + } +} + +func getLocalFiles(localPath string) ([]string, error) { + var files []string + entries, err := os.ReadDir(localPath) + if err != nil { + return nil, err + } + for _, entry := range entries { + if !entry.IsDir() { + files = append(files, stdpath.Join(localPath, entry.Name())) + } + } + return files, nil +} + +func init() { + op.RegisterObjsUpdateHook(UpdateLocalStrm) +} diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go index d3c33164..9e7aba82 100644 --- a/drivers/strm/meta.go +++ b/drivers/strm/meta.go @@ -11,7 +11,9 @@ type Addition struct { FilterFileTypes string `json:"filterFileTypes" type:"text" default:"strm" required:"false" help:"Supports suffix name of strm file"` DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass" required:"false" help:"Files need to download with strm (usally subtitles)"` EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"` - LocalModel bool `json:"localModel" default:"false" help:"enable local mode"` + WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"` + SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"` + SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"` } var config = driver.Config{ diff --git a/drivers/strm/util.go b/drivers/strm/util.go index cfa441db..939c2098 100644 --- a/drivers/strm/util.go +++ b/drivers/strm/util.go @@ -61,36 +61,12 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] var validObjs []model.Obj for _, obj := range objs { - id, name, path := "", obj.GetName(), "" - size := int64(0) - if !obj.IsDir() { - path = stdpath.Join(reqPath, obj.GetName()) - ext := strings.ToLower(utils.Ext(name)) - if _, ok := d.supportSuffix[ext]; ok { - id = "strm" - name = strings.TrimSuffix(name, ext) + "strm" - size = int64(len(d.getLink(ctx, path))) - } else if _, ok := d.downloadSuffix[ext]; ok { - size = obj.GetSize() - } else { - continue - } - } - objRes := model.Object{ - ID: id, - Path: path, - Name: name, - Size: size, - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - } - + objRes := d.convert2strmObj(ctx, reqPath, obj) thumb, ok := model.GetThumb(obj) if !ok { validObjs = append(validObjs, &objRes) continue } - validObjs = append(validObjs, &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ @@ -101,6 +77,32 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] return validObjs, nil } +func (d *Strm) convert2strmObj(ctx context.Context, reqPath string, obj model.Obj) model.Object { + id, name, path := "", obj.GetName(), "" + size := int64(0) + if !obj.IsDir() { + path = stdpath.Join(reqPath, obj.GetName()) + ext := strings.ToLower(utils.Ext(name)) + if _, ok := d.supportSuffix[ext]; ok { + id = "strm" + name = strings.TrimSuffix(name, ext) + "strm" + size = int64(len(d.getLink(ctx, path))) + } else if _, ok := d.downloadSuffix[ext]; ok { + size = obj.GetSize() + } else { + + } + } + return model.Object{ + ID: id, + Path: path, + Name: name, + Size: size, + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + } +} + func (d *Strm) getLink(ctx context.Context, path string) string { finalPath := path if d.EncodePath { @@ -110,7 +112,7 @@ func (d *Strm) getLink(ctx context.Context, path string) string { signPath := sign.Sign(path) finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath) } - if d.LocalModel { + if d.WithoutUrl { return finalPath } apiUrl := d.SiteUrl @@ -119,7 +121,9 @@ func (d *Strm) getLink(ctx context.Context, path string) string { } else { apiUrl = common.GetApiUrl(ctx) } - + if !strings.HasPrefix(finalPath, "/") { + finalPath = "/" + finalPath + } return fmt.Sprintf("%s/d%s", apiUrl, finalPath) diff --git a/internal/op/fs.go b/internal/op/fs.go index c761828e..d9118047 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -57,7 +57,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li model.WrapObjsName(files) // call hooks go func(reqPath string, files []model.Obj) { - HandleObjsUpdateHook(reqPath, files) + HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files) }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) // sort objs diff --git a/internal/op/hook.go b/internal/op/hook.go index 22a06998..5cf01730 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -1,6 +1,7 @@ package op import ( + "context" "regexp" "strings" @@ -13,7 +14,7 @@ import ( ) // Obj -type ObjsUpdateHook = func(parent string, objs []model.Obj) +type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj) var ( objsUpdateHooks = make([]ObjsUpdateHook, 0) @@ -23,9 +24,9 @@ func RegisterObjsUpdateHook(hook ObjsUpdateHook) { objsUpdateHooks = append(objsUpdateHooks, hook) } -func HandleObjsUpdateHook(parent string, objs []model.Obj) { +func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) { for _, hook := range objsUpdateHooks { - hook(parent, objs) + hook(ctx, parent, objs) } } diff --git a/internal/search/build.go b/internal/search/build.go index c5c74e09..e851efc3 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -199,14 +199,13 @@ func Config(ctx context.Context) searcher.Config { return instance.Config() } -func Update(parent string, objs []model.Obj) { +func Update(ctx context.Context, parent string, objs []model.Obj) { if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() { return } if isIgnorePath(parent) { return } - ctx := context.Background() // only update when index have built progress, err := Progress() if err != nil {