From ca401b9af9883cca2ae7413717341961877d0423 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 24 Nov 2025 14:14:53 +0800 Subject: [PATCH] fix(local): assign non-CoW copy requests to the task module (#1669) * fix(local): assign non-CoW copy requests to the task module * fix build * fix cross device --- drivers/local/copy_namedpipes.go | 16 ++++++ drivers/local/copy_namedpipes_x.go | 9 ++++ drivers/local/driver.go | 25 +++------ drivers/local/util.go | 81 +++++++++++++++++++++++++++++- drivers/local/util_unix.go | 6 +++ drivers/local/util_windows.go | 4 ++ go.mod | 3 +- go.sum | 6 +-- 8 files changed, 126 insertions(+), 24 deletions(-) create mode 100644 drivers/local/copy_namedpipes.go create mode 100644 drivers/local/copy_namedpipes_x.go diff --git a/drivers/local/copy_namedpipes.go b/drivers/local/copy_namedpipes.go new file mode 100644 index 00000000..8217e3e0 --- /dev/null +++ b/drivers/local/copy_namedpipes.go @@ -0,0 +1,16 @@ +//go:build !windows && !plan9 && !netbsd && !aix && !illumos && !solaris && !js + +package local + +import ( + "os" + "path/filepath" + "syscall" +) + +func copyNamedPipe(dstPath string, mode os.FileMode, dirMode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(dstPath), dirMode); err != nil { + return err + } + return syscall.Mkfifo(dstPath, uint32(mode)) +} diff --git a/drivers/local/copy_namedpipes_x.go b/drivers/local/copy_namedpipes_x.go new file mode 100644 index 00000000..81f676f1 --- /dev/null +++ b/drivers/local/copy_namedpipes_x.go @@ -0,0 +1,9 @@ +//go:build windows || plan9 || netbsd || aix || illumos || solaris || js + +package local + +import "os" + +func copyNamedPipe(_ string, _, _ os.FileMode) error { + return nil +} diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 45badb2e..7189648a 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -23,7 +23,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/times" - cp "github.com/otiai10/copy" log "github.com/sirupsen/logrus" _ "golang.org/x/image/webp" ) @@ -297,16 +296,9 @@ func (d *Local) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return fmt.Errorf("the destination folder is a subfolder of the source folder") } err := os.Rename(srcPath, dstPath) - if err != nil && strings.Contains(err.Error(), "invalid cross-device link") { - // 跨设备移动，先复制再删除 - if err := d.Copy(ctx, srcObj, dstDir); err != nil { - return err - } - // 复制成功后直接删除源文件/文件夹 - if srcObj.IsDir() { - return os.RemoveAll(srcObj.GetPath()) - } - return os.Remove(srcObj.GetPath()) + if isCrossDeviceError(err) { + // 跨设备移动，变更为移动任务 + return errs.NotImplement } if err == nil { srcParent := filepath.Dir(srcPath) @@ -347,15 +339,14 @@ func (d *Local) Copy(_ context.Context, srcObj, dstDir model.Obj) error { if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } - // Copy using otiai10/copy to perform more secure & efficient copy - err := cp.Copy(srcPath, dstPath, cp.Options{ - Sync: true, // Sync file to disk after copy, may have performance penalty in filesystem such as ZFS - PreserveTimes: true, - PreserveOwner: true, - }) + info, err := os.Lstat(srcPath) if err != nil { return err } + // 复制regular文件会返回errs.NotImplement, 转为复制任务 + if err = d.tryCopy(srcPath, dstPath, info); err != nil { + return err + } if d.directoryMap.Has(filepath.Dir(dstPath)) { d.directoryMap.UpdateDirSize(filepath.Dir(dstPath)) diff --git a/drivers/local/util.go b/drivers/local/util.go index cbf73ad5..be86d9c9 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -3,6 +3,7 @@ package local import ( "bytes" "encoding/json" + "errors" "fmt" "io/fs" "os" @@ -14,7 +15,9 @@ import ( "strings" "sync" + "github.com/KarpelesLab/reflink" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/disintegration/imaging" @@ -148,7 +151,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { return nil, nil, err } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0666) + err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0o666) if err != nil { return nil, nil, err } @@ -405,3 +408,79 @@ func (m *DirectoryMap) DeleteDirNode(dirname string) error { return nil } + +func (d *Local) tryCopy(srcPath, dstPath string, info os.FileInfo) error { + if info.Mode()&os.ModeDevice != 0 { + return errors.New("cannot copy a device") + } else if info.Mode()&os.ModeSymlink != 0 { + return d.copySymlink(srcPath, dstPath) + } else if info.Mode()&os.ModeNamedPipe != 0 { + return copyNamedPipe(dstPath, info.Mode(), os.FileMode(d.mkdirPerm)) + } else if info.IsDir() { + return d.recurAndTryCopy(srcPath, dstPath) + } else { + return tryReflinkCopy(srcPath, dstPath) + } +} + +func (d *Local) copySymlink(srcPath, dstPath string) error { + linkOrig, err := os.Readlink(srcPath) + if err != nil { + return err + } + dstDir := filepath.Dir(dstPath) + if !filepath.IsAbs(linkOrig) { + srcDir := filepath.Dir(srcPath) + rel, err := filepath.Rel(dstDir, srcDir) + if err != nil { + rel, err = filepath.Abs(srcDir) + } + if err != nil { + return err + } + linkOrig = filepath.Clean(filepath.Join(rel, linkOrig)) + } + err = os.MkdirAll(dstDir, os.FileMode(d.mkdirPerm)) + if err != nil { + return err + } + return os.Symlink(linkOrig, dstPath) +} + +func (d *Local) recurAndTryCopy(srcPath, dstPath string) error { + err := os.MkdirAll(dstPath, os.FileMode(d.mkdirPerm)) + if err != nil { + return err + } + files, err := readDir(srcPath) + if err != nil { + return err + } + for _, f := range files { + if !f.IsDir() { + sp := filepath.Join(srcPath, f.Name()) + dp := filepath.Join(dstPath, f.Name()) + if err = d.tryCopy(sp, dp, f); err != nil { + return err + } + } + } + for _, f := range files { + if f.IsDir() { + sp := filepath.Join(srcPath, f.Name()) + dp := filepath.Join(dstPath, f.Name()) + if err = d.recurAndTryCopy(sp, dp); err != nil { + return err + } + } + } + return nil +} + +func tryReflinkCopy(srcPath, dstPath string) error { + err := reflink.Always(srcPath, dstPath) + if errors.Is(err, reflink.ErrReflinkUnsupported) || errors.Is(err, reflink.ErrReflinkFailed) || isCrossDeviceError(err) { + return errs.NotImplement + } + return err +} diff --git a/drivers/local/util_unix.go b/drivers/local/util_unix.go index 3362df34..be0fca41 100644 --- a/drivers/local/util_unix.go +++ b/drivers/local/util_unix.go @@ -3,11 +3,13 @@ package local import ( + "errors" "io/fs" "strings" "syscall" "github.com/OpenListTeam/OpenList/v4/internal/model" + "golang.org/x/sys/unix" ) func isHidden(f fs.FileInfo, _ string) bool { @@ -27,3 +29,7 @@ func getDiskUsage(path string) (model.DiskUsage, error) { FreeSpace: free, }, nil } + +func isCrossDeviceError(err error) bool { + return errors.Is(err, unix.EXDEV) +} diff --git a/drivers/local/util_windows.go b/drivers/local/util_windows.go index 37064009..38387fa1 100644 --- a/drivers/local/util_windows.go +++ b/drivers/local/util_windows.go @@ -49,3 +49,7 @@ func getDiskUsage(path string) (model.DiskUsage, error) { FreeSpace: freeBytes, }, nil } + +func isCrossDeviceError(err error) bool { + return errors.Is(err, windows.ERROR_NOT_SAME_DEVICE) +} diff --git a/go.mod b/go.mod index c34fc6f3..df1207a2 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.4 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 + github.com/KarpelesLab/reflink v1.0.2 github.com/KirCute/zip v1.0.1 github.com/OpenListTeam/go-cache v0.1.0 github.com/OpenListTeam/sftpd-openlist v1.0.1 @@ -114,7 +115,6 @@ require ( github.com/minio/minlz v1.0.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/otiai10/mint v1.6.3 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -256,7 +256,6 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/otiai10/copy v1.14.1 github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/go.sum b/go.sum index 7d45ace2..377e3186 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= +github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= @@ -585,10 +587,6 @@ github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= github.com/nwaples/rardecode/v2 v2.1.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= -github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8= -github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I= -github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= -github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=