From 39dcf9bd1983c4f545aa3822502b4f5ebd2988c0 Mon Sep 17 00:00:00 2001 From: ASLant <77436463+Y-ASLant@users.noreply.github.com> Date: Thu, 6 Nov 2025 23:22:02 +0800 Subject: [PATCH] feat(onedrive): support frontend direct upload (#1532) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * OneDrive添加直连上传 * refactor * fix: duplicate root path join --------- Co-authored-by: KirCute <951206789@qq.com> --- drivers/onedrive/driver.go | 15 +++++++++ drivers/onedrive/meta.go | 1 + drivers/onedrive/util.go | 33 ++++++++++++++++++-- drivers/onedrive_app/driver.go | 14 +++++++++ drivers/onedrive_app/meta.go | 17 ++++++----- drivers/onedrive_app/util.go | 33 ++++++++++++++++++-- internal/driver/driver.go | 9 ++++++ internal/errs/object.go | 7 +++-- internal/fs/fs.go | 16 ++++++++++ internal/fs/put.go | 8 +++++ internal/model/direct_upload.go | 8 +++++ internal/op/fs.go | 48 ++++++++++++++++++++++++++--- server/ftp/afero.go | 4 +-- server/handles/direct_upload.go | 54 +++++++++++++++++++++++++++++++++ server/handles/fsread.go | 34 ++++++++++++--------- server/router.go | 2 ++ 16 files changed, 265 insertions(+), 38 deletions(-) create mode 100644 internal/model/direct_upload.go create mode 100644 server/handles/direct_upload.go diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index c3a3020b..92e07003 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -236,4 +236,19 @@ func (d *Onedrive) GetDetails(ctx context.Context) (*model.StorageDetails, error }, nil } +func (d *Onedrive) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +// GetDirectUploadInfo returns the direct upload info for OneDrive +func (d *Onedrive) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) +} + var _ driver.Driver = (*Onedrive)(nil) diff --git a/drivers/onedrive/meta.go b/drivers/onedrive/meta.go index 239911f5..18b461ea 100644 --- a/drivers/onedrive/meta.go +++ b/drivers/onedrive/meta.go @@ -19,6 +19,7 @@ type Addition struct { ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 919e0308..820f740d 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -133,7 +133,7 @@ func (d *Onedrive) _refreshToken() error { return nil } -func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { if d.ref != nil { return d.ref.Request(url, method, callback, resp) } @@ -152,7 +152,7 @@ func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, return nil, err } if e.Error.Code != "" { - if e.Error.Code == "InvalidAuthenticationToken" { + if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.refreshToken() if err != nil { return nil, err @@ -310,9 +310,36 @@ func (d *Onedrive) getDrive(ctx context.Context) (*DriveResp, error) { var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, &resp) + }, &resp, true) if err != nil { return nil, err } return &resp, nil } + +func (d *Onedrive) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { + // Create upload session + url := d.GetMetaUrl(false, path) + "/createUploadSession" + metadata := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) + if err != nil { + return nil, err + } + + uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() + if uploadUrl == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: uploadUrl, + ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes + Method: "PUT", + }, nil +} diff --git a/drivers/onedrive_app/driver.go b/drivers/onedrive_app/driver.go index f28adde0..de6b2b7f 100644 --- a/drivers/onedrive_app/driver.go +++ b/drivers/onedrive_app/driver.go @@ -222,4 +222,18 @@ func (d *OnedriveAPP) GetDetails(ctx context.Context) (*model.StorageDetails, er }, nil } +func (d *OnedriveAPP) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *OnedriveAPP) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) +} + var _ driver.Driver = (*OnedriveAPP)(nil) diff --git a/drivers/onedrive_app/meta.go b/drivers/onedrive_app/meta.go index 32694e6d..c3f0c02e 100644 --- a/drivers/onedrive_app/meta.go +++ b/drivers/onedrive_app/meta.go @@ -7,14 +7,15 @@ import ( type Addition struct { driver.RootPath - Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` - ClientID string `json:"client_id" required:"true"` - ClientSecret string `json:"client_secret" required:"true"` - TenantID string `json:"tenant_id"` - Email string `json:"email"` - ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` - CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` - DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` + ClientID string `json:"client_id" required:"true"` + ClientSecret string `json:"client_secret" required:"true"` + TenantID string `json:"tenant_id"` + Email string `json:"email"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` + CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 4cc6b9c3..7a7dd0bf 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -88,7 +88,7 @@ func (d *OnedriveAPP) _accessToken() error { return nil } -func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { @@ -104,7 +104,7 @@ func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallba return nil, err } if e.Error.Code != "" { - if e.Error.Code == "InvalidAuthenticationToken" { + if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.accessToken() if err != nil { return nil, err @@ -216,9 +216,36 @@ func (d *OnedriveAPP) getDrive(ctx context.Context) (*DriveResp, error) { var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, &resp) + }, &resp, true) if err != nil { return nil, err } return &resp, nil } + +func (d *OnedriveAPP) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { + // Create upload session + url := d.GetMetaUrl(false, path) + "/createUploadSession" + metadata := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) + if err != nil { + return nil, err + } + + uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() + if uploadUrl == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: uploadUrl, + ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes + Method: "PUT", + }, nil +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 1ce1e451..1f73d35b 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -218,3 +218,12 @@ type LinkCacheModeResolver interface { // ResolveLinkCacheMode returns the LinkCacheMode for the given path. ResolveLinkCacheMode(path string) LinkCacheMode } + +type DirectUploader interface { + // GetDirectUploadTools returns available frontend-direct upload tools + GetDirectUploadTools() []string + // GetDirectUploadInfo returns the information needed for direct upload from client to storage + // actualPath is the path relative to the storage root (after removing mount path prefix) + // return errs.NotImplement if the driver does not support the given direct upload tool + GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) +} diff --git a/internal/errs/object.go b/internal/errs/object.go index 00e8232f..bddc78ab 100644 --- a/internal/errs/object.go +++ b/internal/errs/object.go @@ -7,9 +7,10 @@ import ( ) var ( - ObjectNotFound = errors.New("object not found") - NotFolder = errors.New("not a folder") - NotFile = errors.New("not a file") + ObjectNotFound = errors.New("object not found") + ObjectAlreadyExists = errors.New("object already exists") + NotFolder = errors.New("not a folder") + NotFile = errors.New("not a file") ) func IsObjectNotFound(err error) bool { diff --git a/internal/fs/fs.go b/internal/fs/fs.go index ca199ed4..9e02f629 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -167,6 +167,14 @@ func GetStorage(path string, args *GetStoragesArgs) (driver.Driver, error) { return storageDriver, nil } +func GetStorageAndActualPath(path string) (driver.Driver, string, error) { + return op.GetStorageAndActualPath(path) +} + +func GetByActualPath(ctx context.Context, storage driver.Driver, actualPath string) (model.Obj, error) { + return op.Get(ctx, storage, actualPath) +} + func Other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { res, err := other(ctx, args) if err != nil { @@ -190,3 +198,11 @@ func PutURL(ctx context.Context, path, dstName, urlStr string) error { } return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) } + +func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64) (any, error) { + info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize) + if err != nil { + log.Errorf("failed get %s direct upload info for %s(%d bytes): %+v", path, dstName, fileSize, err) + } + return info, err +} diff --git a/internal/fs/put.go b/internal/fs/put.go index 881330b0..b4806efe 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -105,3 +105,11 @@ func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer } return op.Put(ctx, storage, dstDirActualPath, file, nil, lazyCache...) } + +func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64) (any, error) { + storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize) +} diff --git a/internal/model/direct_upload.go b/internal/model/direct_upload.go new file mode 100644 index 00000000..89bbfeb5 --- /dev/null +++ b/internal/model/direct_upload.go @@ -0,0 +1,8 @@ +package model + +type HttpDirectUploadInfo struct { + UploadURL string `json:"upload_url"` // The URL to upload the file + ChunkSize int64 `json:"chunk_size"` // The chunk size for uploading, 0 means no chunking required + Headers map[string]string `json:"headers,omitempty"` // Optional headers to include in the upload request + Method string `json:"method,omitempty"` // HTTP method, default is PUT +} diff --git a/internal/op/fs.go b/internal/op/fs.go index d9118047..98df7039 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -568,15 +568,15 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url dstPath := stdpath.Join(dstDirPath, dstName) _, err := GetUnwrap(ctx, storage, dstPath) if err == nil { - return errors.New("obj already exists") + return errors.WithStack(errs.ObjectAlreadyExists) } err = MakeDir(ctx, storage, dstDirPath) if err != nil { - return errors.WithMessagef(err, "failed to put url") + return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { - return errors.WithMessagef(err, "failed to put url") + return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } switch s := storage.(type) { case driver.PutURLResult: @@ -599,8 +599,48 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url } } default: - return errs.NotImplement + return errors.WithStack(errs.NotImplement) } log.Debugf("put url [%s](%s) done", dstName, url) return errors.WithStack(err) } + +func GetDirectUploadTools(storage driver.Driver) []string { + du, ok := storage.(driver.DirectUploader) + if !ok { + return nil + } + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil + } + return du.GetDirectUploadTools() +} + +func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64) (any, error) { + du, ok := storage.(driver.DirectUploader) + if !ok { + return nil, errors.WithStack(errs.NotImplement) + } + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } + dstDirPath = utils.FixAndCleanPath(dstDirPath) + dstPath := stdpath.Join(dstDirPath, dstName) + _, err := GetUnwrap(ctx, storage, dstPath) + if err == nil { + return nil, errors.WithStack(errs.ObjectAlreadyExists) + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) + } + info, err := du.GetDirectUploadInfo(ctx, tool, dstDir, dstName, fileSize) + if err != nil { + return nil, errors.WithStack(err) + } + return info, nil +} diff --git a/server/ftp/afero.go b/server/ftp/afero.go index 8b7b467f..02c956a5 100644 --- a/server/ftp/afero.go +++ b/server/ftp/afero.go @@ -104,7 +104,7 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return nil, err } if (flags & os.O_EXCL) != 0 { - return nil, errors.New("file already exists") + return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { return nil, errors.New("cannot write to uploading file") @@ -122,7 +122,7 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return nil, errs.ObjectNotFound } if (flags&os.O_EXCL) != 0 && exists { - return nil, errors.New("file already exists") + return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { if offset != 0 { diff --git a/server/handles/direct_upload.go b/server/handles/direct_upload.go new file mode 100644 index 00000000..69cfd2fa --- /dev/null +++ b/server/handles/direct_upload.go @@ -0,0 +1,54 @@ +package handles + +import ( + "net/url" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +type FsGetDirectUploadInfoReq struct { + Path string `json:"path" form:"path"` + FileName string `json:"file_name" form:"file_name"` + FileSize int64 `json:"file_size" form:"file_size"` + Tool string `json:"tool" form:"tool"` +} + +// FsGetDirectUploadInfo returns the direct upload info if supported by the driver +// If the driver does not support direct upload, returns null for upload_info +func FsGetDirectUploadInfo(c *gin.Context) { + var req FsGetDirectUploadInfoReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + // Decode path + path, err := url.PathUnescape(req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + // Get user and join path + user := c.Request.Context().Value(conf.UserKey).(*model.User) + path, err = user.JoinPath(path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + overwrite := c.GetHeader("Overwrite") != "false" + if !overwrite { + if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, directUploadInfo) +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 6665094c..d5aab75e 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -49,12 +49,13 @@ type ObjResp struct { } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` + DirectUploadTools []string `json:"direct_upload_tools,omitempty"` } func FsListSplit(c *gin.Context) { @@ -109,17 +110,20 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { } total, objs := pagination(objs, &req.PageReq) provider := "unknown" - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err == nil { - provider = storage.GetStorage().Driver + var directUploadTools []string + if user.CanWrite() { + if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + directUploadTools = op.GetDirectUploadTools(storage) + } } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), - Provider: provider, + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), + Total: int64(total), + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: user.CanWrite() || common.CanWrite(meta, reqPath), + Provider: provider, + DirectUploadTools: directUploadTools, }) } diff --git a/server/router.go b/server/router.go index 0975fe69..fbf3a918 100644 --- a/server/router.go +++ b/server/router.go @@ -211,6 +211,8 @@ func _fs(g *gin.RouterGroup) { // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) g.POST("/archive/decompress", handles.FsArchiveDecompress) + // Direct upload (client-side upload to storage) + g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) } func _task(g *gin.RouterGroup) {