Compare commits

...

10 Commits

Author SHA1 Message Date
renovate[bot]
54ab52ca00 fix(deps): update github.com/city404/v6-public-rpc-proto/go digest to acca598 2025-11-24 01:00:48 +00:00
VXTLS
3989d35abd fix(misskey): folderId format validation and root directory handling (#1647)
fix(misskey): Fix folderId format validation and root directory handling
2025-11-21 12:18:54 +08:00
KirCute
72e2ae1f14 feat(fs): support manually trigger objs update hook (#1620)
* feat(fs): support manually trigger objs update hook

* fix: support driver internal copy & move case

* fix

* fix: apply suggestions of Copilot
2025-11-21 12:18:20 +08:00
Seven
3e37f575d8 fix(openlist_driver): ensure UA is correctly propagated (#1679) 2025-11-21 12:13:41 +08:00
MoYan
c0d480366d fix(driver/123): initialize Platform field (#1644)
* fix(driver/123): initialization the Platform field

Signed-off-by: MoYan <1561515308@qq.com>

* Fix formatting of Platform field in Pan123

Signed-off-by: MoYan <1561515308@qq.com>

---------

Signed-off-by: MoYan <1561515308@qq.com>
2025-11-14 18:49:44 +08:00
Copilot
9de7561154 feat(upload): add optional system file filtering for uploads (#1634) 2025-11-14 14:45:39 +08:00
MadDogOwner
0866b9075f fix(link): correct link cache mode bitwise comparison (#1635)
* fix(link): correct link cache mode bitwise comparison

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

* refactor(link): use explicit flag equality for link cache mode bitmask checks

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

---------

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>
2025-11-13 13:52:33 +08:00
KirCute
055696f576 feat(s3): support frontend direct upload (#1631)
* feat(s3): support frontend direct upload

* feat(s3): support custom direct upload host

* fix: apply suggestions of Copilot
2025-11-13 13:22:17 +08:00
ShenLin
854415160c chore(issue templates): require logs (#1626) 2025-11-12 13:04:13 +08:00
varg1714
8f4f7d1291 feat(doubao): Add rate limiting (#1618) 2025-11-11 21:59:10 +08:00
33 changed files with 644 additions and 110 deletions

View File

@@ -13,7 +13,7 @@ body:
attributes:
label: 请确认以下事项
description: |
您必须勾选以下内容,否则您的问题可能会被直接关闭。
您必须确认、同意并勾选以下内容,否则您的问题一定会被直接关闭。
或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。
options:
- label: |
@@ -59,6 +59,14 @@ body:
label: 问题描述(必填)
validations:
required: true
- type: textarea
id: logs
attributes:
label: 日志(必填)
description: |
请复制粘贴错误日志,或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug)
validations:
required: true
- type: textarea
id: config
attributes:
@@ -67,12 +75,6 @@ body:
请提供您的`OpenList`应用的配置文件,并截图相关存储配置。(可隐藏隐私字段)
validations:
required: true
- type: textarea
id: logs
attributes:
label: 日志(可选)
description: |
请复制粘贴错误日志,或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug)
- type: textarea
id: reproduction
attributes:

View File

@@ -13,7 +13,7 @@ body:
attributes:
label: Please confirm the following
description: |
You must check all the following, otherwise your issue may be closed directly.
You must confirm, agree, and check all the following, otherwise your issue will definitely be closed directly.
Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions).
options:
- label: |
@@ -59,6 +59,14 @@ body:
label: Bug Description (required)
validations:
required: true
- type: textarea
id: logs
attributes:
label: Logs (required)
description: |
Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs)
validations:
required: true
- type: textarea
id: config
attributes:
@@ -67,12 +75,6 @@ body:
Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields)
validations:
required: true
- type: textarea
id: logs
attributes:
label: Logs (optional)
description: |
Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs)
- type: textarea
id: reproduction
attributes:

View File

@@ -28,6 +28,7 @@ func init() {
return &Pan123{
Addition: Addition{
UploadThread: 3,
Platform: "web",
},
}
})

View File

@@ -5,7 +5,6 @@ import (
"errors"
stdpath "path"
"strings"
"sync"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
@@ -17,9 +16,15 @@ import (
log "github.com/sirupsen/logrus"
)
type detailWithIndex struct {
idx int
val *model.StorageDetails
}
func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj {
var objs []model.Obj
var wg sync.WaitGroup
detailsChan := make(chan detailWithIndex, len(d.pathMap))
workerCount := 0
for _, k := range d.rootOrder {
obj := model.Object{
Name: k,
@@ -47,22 +52,26 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model
DriverName: remoteDriver.Config().Name,
},
}
wg.Add(1)
go func() {
defer wg.Done()
c, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
details, e := op.GetStorageDetails(c, remoteDriver, refresh)
workerCount++
go func(dri driver.Driver, i int) {
details, e := op.GetStorageDetails(ctx, dri, refresh)
if e != nil {
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
log.Errorf("failed get %s storage details: %+v", remoteDriver.GetStorage().MountPath, e)
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e)
}
return
}
objs[idx].(*model.ObjStorageDetails).StorageDetails = details
}()
detailsChan <- detailWithIndex{idx: i, val: details}
}(remoteDriver, idx)
}
for workerCount > 0 {
select {
case r := <-detailsChan:
objs[r.idx].(*model.ObjStorageDetails).StorageDetails = r.val
workerCount--
case <-time.After(time.Second):
workerCount = 0
}
}
wg.Wait()
return objs
}

View File

@@ -15,6 +15,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
"github.com/google/uuid"
"golang.org/x/time/rate"
)
type Doubao struct {
@@ -23,6 +24,7 @@ type Doubao struct {
*UploadToken
UserId string
uploadThread int
limiter *rate.Limiter
}
func (d *Doubao) Config() driver.Config {
@@ -61,6 +63,17 @@ func (d *Doubao) Init(ctx context.Context) error {
d.UploadToken = uploadToken
}
if d.LimitRate > 0 {
d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1)
}
return nil
}
func (d *Doubao) WaitLimit(ctx context.Context) error {
if d.limiter != nil {
return d.limiter.Wait(ctx)
}
return nil
}
@@ -69,6 +82,10 @@ func (d *Doubao) Drop(ctx context.Context) error {
}
func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
if err := d.WaitLimit(ctx); err != nil {
return nil, err
}
var files []model.Obj
fileList, err := d.getFiles(dir.GetID(), "")
if err != nil {
@@ -95,6 +112,10 @@ func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) (
}
func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
if err := d.WaitLimit(ctx); err != nil {
return nil, err
}
var downloadUrl string
if u, ok := file.(*Object); ok {
@@ -160,6 +181,10 @@ func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
}
func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
if err := d.WaitLimit(ctx); err != nil {
return err
}
var r UploadNodeResp
_, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
@@ -177,6 +202,10 @@ func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin
}
func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
if err := d.WaitLimit(ctx); err != nil {
return err
}
var r UploadNodeResp
_, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
@@ -191,6 +220,10 @@ func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
}
func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
if err := d.WaitLimit(ctx); err != nil {
return err
}
var r BaseResp
_, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
@@ -207,6 +240,10 @@ func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj,
}
func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error {
if err := d.WaitLimit(ctx); err != nil {
return err
}
var r BaseResp
_, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}})
@@ -215,6 +252,10 @@ func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error {
}
func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
if err := d.WaitLimit(ctx); err != nil {
return nil, err
}
// 根据MIME类型确定数据类型
mimetype := file.GetMimetype()
dataType := FileDataType

View File

@@ -13,6 +13,7 @@ type Addition struct {
Cookie string `json:"cookie" type:"text"`
UploadThread string `json:"upload_thread" default:"3"`
DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"`
LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"`
}
var config = driver.Config{
@@ -23,6 +24,10 @@ var config = driver.Config{
func init() {
op.RegisterDriver(func() driver.Driver {
return &Doubao{}
return &Doubao{
Addition: Addition{
LimitRate: 2,
},
}
})
}

View File

@@ -57,18 +57,22 @@ func setBody(body interface{}) base.ReqCallback {
}
func handleFolderId(dir model.Obj) interface{} {
if dir.GetID() == "" {
return nil
if isRootFolder(dir) {
return nil // Root folder doesn't need folderId
}
return dir.GetID()
}
func isRootFolder(dir model.Obj) bool {
return dir.GetID() == ""
}
// API layer methods
func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) {
var files []MFile
var body map[string]string
if dir.GetPath() != "/" {
if !isRootFolder(dir) {
body = map[string]string{"folderId": dir.GetID()}
} else {
body = map[string]string{}
@@ -85,7 +89,7 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) {
func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) {
var folders []MFolder
var body map[string]string
if dir.GetPath() != "/" {
if !isRootFolder(dir) {
body = map[string]string{"folderId": dir.GetID()}
} else {
body = map[string]string{}
@@ -197,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt
Reader: stream,
UpdateProgress: up,
})
req := base.RestyClient.R().
SetContext(ctx).
SetFileReader("file", stream.GetName(), reader).
SetFormData(map[string]string{
"folderId": handleFolderId(dstDir).(string),
// Build form data, only add folderId if not root folder
formData := map[string]string{
"name": stream.GetName(),
"comment": "",
"isSensitive": "false",
"force": "false",
}).
}
folderId := handleFolderId(dstDir)
if folderId != nil {
formData["folderId"] = folderId.(string)
}
req := base.RestyClient.R().
SetContext(ctx).
SetFileReader("file", stream.GetName(), reader).
SetFormData(formData).
SetResult(&file).
SetAuthToken(d.AccessToken)

View File

@@ -117,7 +117,7 @@ func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs
if d.PassUAToUpsteam {
userAgent := args.Header.Get("user-agent")
if userAgent != "" {
headers["User-Agent"] = base.UserAgent
headers["User-Agent"] = userAgent
}
}
// if PassIPToUpsteam is true, then pass the ip address to the upstream

View File

@@ -217,11 +217,10 @@ func (d *QuarkOrUC) GetDetails(ctx context.Context) (*model.StorageDetails, erro
if err != nil {
return nil, err
}
used := memberInfo.Data.UseCapacity
total := memberInfo.Data.TotalCapacity
return &model.StorageDetails{
DiskUsage: model.DiskUsage{
TotalSpace: memberInfo.Data.TotalCapacity,
FreeSpace: memberInfo.Data.TotalCapacity - memberInfo.Data.UseCapacity,
},
DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
}, nil
}

View File

@@ -10,6 +10,7 @@ import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/cron"
@@ -27,6 +28,7 @@ type S3 struct {
Session *session.Session
client *s3.S3
linkClient *s3.S3
directUploadClient *s3.S3
config driver.Config
cron *cron.Cron
@@ -52,16 +54,18 @@ func (d *S3) Init(ctx context.Context) error {
if err != nil {
log.Errorln("Doge init session error:", err)
}
d.client = d.getClient(false)
d.linkClient = d.getClient(true)
d.client = d.getClient(ClientTypeNormal)
d.linkClient = d.getClient(ClientTypeLink)
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
})
}
err := d.initSession()
if err != nil {
return err
}
d.client = d.getClient(false)
d.linkClient = d.getClient(true)
d.client = d.getClient(ClientTypeNormal)
d.linkClient = d.getClient(ClientTypeLink)
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
return nil
}
@@ -210,4 +214,33 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up
return err
}
func (d *S3) GetDirectUploadTools() []string {
if !d.EnableDirectUpload {
return nil
}
return []string{"HttpDirect"}
}
func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) {
if !d.EnableDirectUpload {
return nil, errs.NotImplement
}
path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false)
req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{
Bucket: &d.Bucket,
Key: &path,
})
if req == nil {
return nil, fmt.Errorf("failed to create PutObject request")
}
link, err := req.Presign(time.Hour * time.Duration(d.SignURLExpire))
if err != nil {
return nil, err
}
return &model.HttpDirectUploadInfo{
UploadURL: link,
Method: "PUT",
}, nil
}
var _ driver.Driver = (*S3)(nil)

View File

@@ -21,6 +21,8 @@ type Addition struct {
ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."`
EnableDirectUpload bool `json:"enable_direct_upload" default:"false"`
DirectUploadHost string `json:"direct_upload_host" required:"false"`
}
func init() {

View File

@@ -41,9 +41,15 @@ func (d *S3) initSession() error {
return err
}
func (d *S3) getClient(link bool) *s3.S3 {
const (
ClientTypeNormal = iota
ClientTypeLink
ClientTypeDirectUpload
)
func (d *S3) getClient(clientType int) *s3.S3 {
client := s3.New(d.Session)
if link && d.CustomHost != "" {
if clientType == ClientTypeLink && d.CustomHost != "" {
client.Handlers.Build.PushBack(func(r *request.Request) {
if r.HTTPRequest.Method != http.MethodGet {
return
@@ -58,6 +64,20 @@ func (d *S3) getClient(link bool) *s3.S3 {
}
})
}
if clientType == ClientTypeDirectUpload && d.DirectUploadHost != "" {
client.Handlers.Build.PushBack(func(r *request.Request) {
if r.HTTPRequest.Method != http.MethodPut {
return
}
split := strings.SplitN(d.DirectUploadHost, "://", 2)
if utils.SliceContains([]string{"http", "https"}, split[0]) {
r.HTTPRequest.URL.Scheme = split[0]
r.HTTPRequest.URL.Host = split[1]
} else {
r.HTTPRequest.URL.Host = d.DirectUploadHost
}
})
}
return client
}

2
go.mod
View File

@@ -22,7 +22,7 @@ require (
github.com/charmbracelet/bubbles v0.21.0
github.com/charmbracelet/bubbletea v1.3.6
github.com/charmbracelet/lipgloss v1.1.0
github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e
github.com/city404/v6-public-rpc-proto/go acca598c387c
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc
github.com/coreos/go-oidc v2.3.0+incompatible
github.com/deckarep/golang-set/v2 v2.8.0

View File

@@ -177,6 +177,9 @@ func InitialSettings() []model.SettingItem {
{Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
{Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC},
{Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`},
// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},

View File

@@ -56,6 +56,9 @@ const (
ShareArchivePreview = "share_archive_preview"
ShareForceProxy = "share_force_proxy"
ShareSummaryContent = "share_summary_content"
HandleHookAfterWriting = "handle_hook_after_writing"
HandleHookRateLimit = "handle_hook_rate_limit"
IgnoreSystemFiles = "ignore_system_files"
// index
SearchIndex = "search_index"

View File

@@ -11,6 +11,7 @@ var (
ObjectAlreadyExists = errors.New("object already exists")
NotFolder = errors.New("not a folder")
NotFile = errors.New("not a file")
IgnoredSystemFile = errors.New("system file upload ignored")
)
func IsObjectNotFound(err error) bool {

View File

@@ -11,6 +11,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/archive/tool"
"github.com/OpenListTeam/OpenList/v4/internal/cache"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -20,10 +21,13 @@ import (
gocache "github.com/OpenListTeam/go-cache"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
var archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
var (
archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
)
func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -196,8 +200,10 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg
return obj, archiveMetaProvider, err
}
var archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
var archiveListG singleflight.Group[[]model.Obj]
var (
archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
archiveListG singleflight.Group[[]model.Obj]
)
func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -397,8 +403,10 @@ type objWithLink struct {
obj model.Obj
}
var extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
var extractG = singleflight.Group[*objWithLink]{}
var (
extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
extractG = singleflight.Group[*objWithLink]{}
)
func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -506,9 +514,9 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
return errors.WithMessage(err, "failed to get dst dir")
}
var newObjs []model.Obj
switch s := storage.(type) {
case driver.ArchiveDecompressResult:
var newObjs []model.Obj
newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
if err == nil {
if len(newObjs) > 0 {
@@ -527,5 +535,31 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
default:
return errs.NotImplement
}
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
onlyList := false
targetPath := dstDirPath
if newObjs != nil && len(newObjs) == 1 && newObjs[0].IsDir() {
targetPath = stdpath.Join(dstDirPath, newObjs[0].GetName())
} else if newObjs != nil && len(newObjs) == 1 && !newObjs[0].IsDir() {
onlyList = true
} else if args.PutIntoNewDir {
targetPath = stdpath.Join(dstDirPath, strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName())))
} else if innerBase := stdpath.Base(args.InnerPath); innerBase != "." && innerBase != "/" {
targetPath = stdpath.Join(dstDirPath, innerBase)
dstObj, e := GetUnwrap(ctx, storage, targetPath)
onlyList = e != nil || !dstObj.IsDir()
}
if onlyList {
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
} else {
var limiter *rate.Limiter
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
limiter = rate.NewLimiter(rate.Limit(f), 1)
}
}
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
}
}
return errors.WithStack(err)
}

View File

@@ -2,10 +2,11 @@ package op
import (
"context"
stderrors "errors"
stdpath "path"
"strconv"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -14,6 +15,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
var listG singleflight.Group[[]model.Obj]
@@ -173,10 +175,10 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li
mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path)
}
typeKey := args.Type
if mode&driver.LinkCacheIP == 1 {
if mode&driver.LinkCacheIP == driver.LinkCacheIP {
typeKey += "/" + args.IP
}
if mode&driver.LinkCacheUA == 1 {
if mode&driver.LinkCacheUA == driver.LinkCacheUA {
typeKey += "/" + args.Header.Get("User-Agent")
}
key := Key(storage, path)
@@ -310,7 +312,7 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
srcDirPath := stdpath.Dir(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
if dstDirPath == srcDirPath {
return stderrors.New("move in place")
return errors.New("move in place")
}
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
@@ -343,8 +345,24 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
}
}
default:
return errs.NotImplement
err = errs.NotImplement
}
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
if !srcObj.IsDir() {
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
} else {
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
var limiter *rate.Limiter
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
limiter = rate.NewLimiter(rate.Limit(f), 1)
}
}
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
}
}
return errors.WithStack(err)
}
@@ -397,7 +415,7 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
if dstDirPath == stdpath.Dir(srcPath) {
return stderrors.New("copy in place")
return errors.New("copy in place")
}
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
@@ -428,8 +446,24 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
}
}
default:
return errs.NotImplement
err = errs.NotImplement
}
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
if !srcObj.IsDir() {
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
} else {
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
var limiter *rate.Limiter
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
limiter = rate.NewLimiter(rate.Limit(f), 1)
}
}
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
}
}
return errors.WithStack(err)
}
@@ -557,6 +591,9 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
err = Remove(ctx, storage, tempPath)
}
}
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
}
return errors.WithStack(err)
}
@@ -601,6 +638,9 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url
default:
return errors.WithStack(errs.NotImplement)
}
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
}
log.Debugf("put url [%s](%s) done", dstName, url)
return errors.WithStack(err)
}
@@ -644,3 +684,8 @@ func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver
}
return info, nil
}
func needHandleObjsUpdateHook() bool {
needHandle, _ := GetSettingItemByKey(conf.HandleHookAfterWriting)
return needHandle != nil && (needHandle.Value == "true" || needHandle.Value == "1")
}

View File

@@ -0,0 +1,125 @@
package op
import (
"context"
stdpath "path"
"sync"
"sync/atomic"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
var (
ManualScanCancel = atomic.Pointer[context.CancelFunc]{}
ScannedCount = atomic.Uint64{}
)
func ManualScanRunning() bool {
return ManualScanCancel.Load() != nil
}
func BeginManualScan(rawPath string, limit float64) error {
rawPath = utils.FixAndCleanPath(rawPath)
ctx, cancel := context.WithCancel(context.Background())
if !ManualScanCancel.CompareAndSwap(nil, &cancel) {
cancel()
return errors.New("manual scan is running, please try later")
}
ScannedCount.Store(0)
go func() {
defer func() { (*ManualScanCancel.Swap(nil))() }()
err := RecursivelyList(ctx, rawPath, rate.Limit(limit), &ScannedCount)
if err != nil {
log.Errorf("failed recursively list: %v", err)
}
}()
return nil
}
func StopManualScan() {
c := ManualScanCancel.Load()
if c != nil {
(*c)()
}
}
func RecursivelyList(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64) error {
storage, actualPath, err := GetStorageAndActualPath(rawPath)
if err != nil && !errors.Is(err, errs.StorageNotFound) {
return err
} else if err == nil {
var limiter *rate.Limiter
if limit > .0 {
limiter = rate.NewLimiter(limit, 1)
}
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
} else {
var wg sync.WaitGroup
recursivelyListVirtual(ctx, rawPath, limit, counter, &wg)
wg.Wait()
}
return nil
}
func recursivelyListVirtual(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64, wg *sync.WaitGroup) {
objs := GetStorageVirtualFilesByPath(rawPath)
if counter != nil {
counter.Add(uint64(len(objs)))
}
for _, obj := range objs {
if utils.IsCanceled(ctx) {
return
}
nextPath := stdpath.Join(rawPath, obj.GetName())
storage, actualPath, err := GetStorageAndActualPath(nextPath)
if err != nil && !errors.Is(err, errs.StorageNotFound) {
log.Errorf("error recursively list: failed get storage [%s]: %v", nextPath, err)
} else if err == nil {
var limiter *rate.Limiter
if limit > .0 {
limiter = rate.NewLimiter(limit, 1)
}
wg.Add(1)
go func() {
defer wg.Done()
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
}()
} else {
recursivelyListVirtual(ctx, nextPath, limit, counter, wg)
}
}
}
func RecursivelyListStorage(ctx context.Context, storage driver.Driver, actualPath string, limiter *rate.Limiter, counter *atomic.Uint64) {
objs, err := List(ctx, storage, actualPath, model.ListArgs{Refresh: true})
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Errorf("error recursively list: failed list (%s)[%s]: %v", storage.GetStorage().MountPath, actualPath, err)
}
return
}
if counter != nil {
counter.Add(uint64(len(objs)))
}
for _, obj := range objs {
if utils.IsCanceled(ctx) {
return
}
if !obj.IsDir() {
continue
}
if limiter != nil {
if err = limiter.Wait(ctx); err != nil {
return
}
}
nextPath := stdpath.Join(actualPath, obj.GetName())
RecursivelyListStorage(ctx, storage, nextPath, limiter, counter)
}
}

View File

@@ -358,16 +358,21 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string,
DriverName: d.Config().Name,
},
}
timeoutCtx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
details, err := GetStorageDetails(timeoutCtx, d, refresh)
resultChan := make(chan *model.StorageDetails, 1)
go func(dri driver.Driver) {
details, err := GetStorageDetails(ctx, dri, refresh)
if err != nil {
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
log.Errorf("failed get %s storage details: %+v", d.GetStorage().MountPath, err)
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err)
}
return ret
}
ret.StorageDetails = details
resultChan <- details
}(d)
select {
case r := <-resultChan:
ret.StorageDetails = r
case <-time.After(time.Second):
}
return ret
})
}

View File

@@ -28,3 +28,11 @@ func GetInt(key string, defaultVal int) int {
func GetBool(key string) bool {
return GetStr(key) == "true" || GetStr(key) == "1"
}
func GetFloat(key string, defaultVal float64) float64 {
f, err := strconv.ParseFloat(GetStr(key), 64)
if err != nil {
return defaultVal
}
return f
}

View File

@@ -5,11 +5,14 @@ import (
"fmt"
"path"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
type SrcPathToRemove string
@@ -27,13 +30,32 @@ func RefreshAndRemove(dstPath string, payloads ...any) {
if dstNeedRefresh {
op.Cache.DeleteDirectory(dstStorage, dstActualPath)
}
dstNeedHandleHook := setting.GetBool(conf.HandleHookAfterWriting)
dstHandleHookLimit := setting.GetFloat(conf.HandleHookRateLimit, .0)
var listLimiter *rate.Limiter
if dstNeedRefresh && dstNeedHandleHook && dstHandleHookLimit > .0 {
listLimiter = rate.NewLimiter(rate.Limit(dstHandleHookLimit), 1)
}
var ctx context.Context
for _, payload := range payloads {
switch p := payload.(type) {
case DstPathToRefresh:
if dstNeedRefresh {
if dstNeedHandleHook {
if ctx == nil {
ctx = context.Background()
}
if listLimiter != nil {
_ = listLimiter.Wait(ctx)
}
_, e := op.List(ctx, dstStorage, string(p), model.ListArgs{Refresh: true})
if e != nil {
log.Errorf("failed handle objs update hook: %v", e)
}
} else {
op.Cache.DeleteDirectory(dstStorage, string(p))
}
}
case SrcPathToRemove:
if ctx == nil {
ctx = context.Background()

View File

@@ -185,3 +185,20 @@ const (
GB
TB
)
// IsSystemFile checks if a filename is a common system file that should be ignored
// Returns true for files like .DS_Store, desktop.ini, Thumbs.db, and Apple Double files (._*)
func IsSystemFile(filename string) bool {
// Common system files
switch filename {
case ".DS_Store", "desktop.ini", "Thumbs.db":
return true
}
// Apple Double files (._*)
if strings.HasPrefix(filename, "._") {
return true
}
return false
}

42
pkg/utils/file_test.go Normal file
View File

@@ -0,0 +1,42 @@
package utils
import (
"testing"
)
func TestIsSystemFile(t *testing.T) {
testCases := []struct {
filename string
expected bool
}{
// System files that should be filtered
{".DS_Store", true},
{"desktop.ini", true},
{"Thumbs.db", true},
{"._test.txt", true},
{"._", true},
{"._somefile", true},
{"._folder_name", true},
// Regular files that should not be filtered
{"test.txt", false},
{"file.pdf", false},
{"document.docx", false},
{".gitignore", false},
{".env", false},
{"_underscore.txt", false},
{"normal_file.txt", false},
{"", false},
{".hidden", false},
{"..special", false},
}
for _, tc := range testCases {
t.Run(tc.filename, func(t *testing.T) {
result := IsSystemFile(tc.filename)
if result != tc.expected {
t.Errorf("IsSystemFile(%q) = %v, want %v", tc.filename, result, tc.expected)
}
})
}
}

View File

@@ -15,7 +15,9 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server/common"
ftpserver "github.com/fclairamb/ftpserverlib"
"github.com/pkg/errors"
@@ -49,6 +51,11 @@ func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy,
if err != nil {
return nil, err
}
// Check if system file should be ignored
_, name := stdpath.Split(path)
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
return nil, errs.IgnoredSystemFile
}
tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*")
if err != nil {
return nil, err
@@ -150,6 +157,11 @@ func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length i
if err != nil {
return nil, err
}
// Check if system file should be ignored
_, name := stdpath.Split(path)
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
return nil, errs.IgnoredSystemFile
}
if trunc {
_ = fs.Remove(ctx, path)
}

View File

@@ -8,8 +8,10 @@ import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/internal/task"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@@ -28,6 +30,14 @@ func getLastModified(c *gin.Context) time.Time {
return lastModified
}
// shouldIgnoreSystemFile checks if the filename should be ignored based on settings
func shouldIgnoreSystemFile(filename string) bool {
if setting.GetBool(conf.IgnoreSystemFiles) {
return utils.IsSystemFile(filename)
}
return false
}
func FsStream(c *gin.Context) {
defer func() {
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
@@ -56,6 +66,11 @@ func FsStream(c *gin.Context) {
}
}
dir, name := stdpath.Split(path)
// Check if system file should be ignored
if shouldIgnoreSystemFile(name) {
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
return
}
// 如果请求头 Content-Length 和 X-File-Size 都没有,则 size=-1表示未知大小的流式上传
size := c.Request.ContentLength
if size < 0 {
@@ -160,6 +175,11 @@ func FsForm(c *gin.Context) {
}
defer f.Close()
dir, name := stdpath.Split(path)
// Check if system file should be ignored
if shouldIgnoreSystemFile(name) {
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
return
}
h := make(map[*utils.HashType]string)
if md5 := c.GetHeader("X-File-Md5"); md5 != "" {
h[utils.MD5] = md5

47
server/handles/scan.go Normal file
View File

@@ -0,0 +1,47 @@
package handles
import (
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/server/common"
"github.com/gin-gonic/gin"
)
type ManualScanReq struct {
Path string `json:"path"`
Limit float64 `json:"limit"`
}
func StartManualScan(c *gin.Context) {
var req ManualScanReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
if err := op.BeginManualScan(req.Path, req.Limit); err != nil {
common.ErrorResp(c, err, 400)
return
}
common.SuccessResp(c)
}
func StopManualScan(c *gin.Context) {
if !op.ManualScanRunning() {
common.ErrorStrResp(c, "manual scan is not running", 400)
return
}
op.StopManualScan()
common.SuccessResp(c)
}
type ManualScanResp struct {
ObjCount uint64 `json:"obj_count"`
IsDone bool `json:"is_done"`
}
func GetManualScanProgress(c *gin.Context) {
ret := ManualScanResp{
ObjCount: op.ScannedCount.Load(),
IsDone: !op.ManualScanRunning(),
}
common.SuccessResp(c, ret)
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"errors"
"strconv"
"sync"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
@@ -24,9 +23,15 @@ type StorageResp struct {
MountDetails *model.StorageDetails `json:"mount_details,omitempty"`
}
func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
type detailWithIndex struct {
idx int
val *model.StorageDetails
}
func makeStorageResp(ctx *gin.Context, storages []model.Storage) []*StorageResp {
ret := make([]*StorageResp, len(storages))
var wg sync.WaitGroup
detailsChan := make(chan detailWithIndex, len(storages))
workerCount := 0
for i, s := range storages {
ret[i] = &StorageResp{
Storage: s,
@@ -43,22 +48,26 @@ func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
if !ok {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
ctx, cancel := context.WithTimeout(c, time.Second*3)
defer cancel()
details, err := op.GetStorageDetails(ctx, d)
if err != nil {
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
log.Errorf("failed get %s details: %+v", s.MountPath, err)
workerCount++
go func(dri driver.Driver, idx int) {
details, e := op.GetStorageDetails(ctx, dri)
if e != nil {
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
log.Errorf("failed get %s details: %+v", dri.GetStorage().MountPath, e)
}
return
}
ret[i].MountDetails = details
}()
detailsChan <- detailWithIndex{idx: idx, val: details}
}(d, i)
}
for workerCount > 0 {
select {
case r := <-detailsChan:
ret[r.idx].MountDetails = r.val
workerCount--
case <-time.After(time.Second * 3):
workerCount = 0
}
}
wg.Wait()
return ret
}

View File

@@ -11,7 +11,7 @@ import (
func SearchIndex(c *gin.Context) {
mode := setting.GetStr(conf.SearchIndex)
if mode == "none" {
common.ErrorResp(c, errs.SearchNotAvailable, 500)
common.ErrorResp(c, errs.SearchNotAvailable, 404)
c.Abort()
} else {
c.Next()

View File

@@ -179,6 +179,11 @@ func admin(g *gin.RouterGroup) {
index.POST("/stop", middlewares.SearchIndex, handles.StopIndex)
index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex)
index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
scan := g.Group("/scan")
scan.POST("/start", handles.StartManualScan)
scan.POST("/stop", handles.StopManualScan)
scan.GET("/progress", handles.GetManualScanProgress)
}
func fsAndShare(g *gin.RouterGroup) {

View File

@@ -19,6 +19,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@@ -286,6 +287,10 @@ func (b *s3Backend) PutObject(
Modified: ti,
Ctime: time.Now(),
}
// Check if system file should be ignored
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
return result, errs.IgnoredSystemFile
}
stream := &stream.FileStream{
Obj: &obj,
Reader: input,

View File

@@ -20,6 +20,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/net"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
@@ -358,6 +359,10 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
Modified: h.getModTime(r),
Ctime: h.getCreateTime(r),
}
// Check if system file should be ignored
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
return http.StatusForbidden, errs.IgnoredSystemFile
}
fsStream := &stream.FileStream{
Obj: &obj,
Reader: r.Body,