mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2025-11-25 11:29:29 +08:00
Compare commits
10 Commits
v4.1.7
...
renovate/g
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54ab52ca00 | ||
|
|
3989d35abd | ||
|
|
72e2ae1f14 | ||
|
|
3e37f575d8 | ||
|
|
c0d480366d | ||
|
|
9de7561154 | ||
|
|
0866b9075f | ||
|
|
055696f576 | ||
|
|
854415160c | ||
|
|
8f4f7d1291 |
16
.github/ISSUE_TEMPLATE/00-bug_report_zh.yml
vendored
16
.github/ISSUE_TEMPLATE/00-bug_report_zh.yml
vendored
@@ -13,7 +13,7 @@ body:
|
||||
attributes:
|
||||
label: 请确认以下事项
|
||||
description: |
|
||||
您必须勾选以下内容,否则您的问题可能会被直接关闭。
|
||||
您必须确认、同意并勾选以下内容,否则您的问题一定会被直接关闭。
|
||||
或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。
|
||||
options:
|
||||
- label: |
|
||||
@@ -59,6 +59,14 @@ body:
|
||||
label: 问题描述(必填)
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: 日志(必填)
|
||||
description: |
|
||||
请复制粘贴错误日志,或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug)
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: config
|
||||
attributes:
|
||||
@@ -67,12 +75,6 @@ body:
|
||||
请提供您的`OpenList`应用的配置文件,并截图相关存储配置。(可隐藏隐私字段)
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: 日志(可选)
|
||||
description: |
|
||||
请复制粘贴错误日志,或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug)
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
|
||||
16
.github/ISSUE_TEMPLATE/01-bug_report_en.yml
vendored
16
.github/ISSUE_TEMPLATE/01-bug_report_en.yml
vendored
@@ -13,7 +13,7 @@ body:
|
||||
attributes:
|
||||
label: Please confirm the following
|
||||
description: |
|
||||
You must check all the following, otherwise your issue may be closed directly.
|
||||
You must confirm, agree, and check all the following, otherwise your issue will definitely be closed directly.
|
||||
Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions).
|
||||
options:
|
||||
- label: |
|
||||
@@ -59,6 +59,14 @@ body:
|
||||
label: Bug Description (required)
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs (required)
|
||||
description: |
|
||||
Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs)
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: config
|
||||
attributes:
|
||||
@@ -67,12 +75,6 @@ body:
|
||||
Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields)
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs (optional)
|
||||
description: |
|
||||
Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs)
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
|
||||
@@ -28,6 +28,7 @@ func init() {
|
||||
return &Pan123{
|
||||
Addition: Addition{
|
||||
UploadThread: 3,
|
||||
Platform: "web",
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
stdpath "path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
@@ -17,9 +16,15 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type detailWithIndex struct {
|
||||
idx int
|
||||
val *model.StorageDetails
|
||||
}
|
||||
|
||||
func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj {
|
||||
var objs []model.Obj
|
||||
var wg sync.WaitGroup
|
||||
detailsChan := make(chan detailWithIndex, len(d.pathMap))
|
||||
workerCount := 0
|
||||
for _, k := range d.rootOrder {
|
||||
obj := model.Object{
|
||||
Name: k,
|
||||
@@ -47,22 +52,26 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model
|
||||
DriverName: remoteDriver.Config().Name,
|
||||
},
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
c, cancel := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel()
|
||||
details, e := op.GetStorageDetails(c, remoteDriver, refresh)
|
||||
workerCount++
|
||||
go func(dri driver.Driver, i int) {
|
||||
details, e := op.GetStorageDetails(ctx, dri, refresh)
|
||||
if e != nil {
|
||||
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
|
||||
log.Errorf("failed get %s storage details: %+v", remoteDriver.GetStorage().MountPath, e)
|
||||
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e)
|
||||
}
|
||||
return
|
||||
}
|
||||
objs[idx].(*model.ObjStorageDetails).StorageDetails = details
|
||||
}()
|
||||
detailsChan <- detailWithIndex{idx: i, val: details}
|
||||
}(remoteDriver, idx)
|
||||
}
|
||||
for workerCount > 0 {
|
||||
select {
|
||||
case r := <-detailsChan:
|
||||
objs[r.idx].(*model.ObjStorageDetails).StorageDetails = r.val
|
||||
workerCount--
|
||||
case <-time.After(time.Second):
|
||||
workerCount = 0
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
return objs
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type Doubao struct {
|
||||
@@ -23,6 +24,7 @@ type Doubao struct {
|
||||
*UploadToken
|
||||
UserId string
|
||||
uploadThread int
|
||||
limiter *rate.Limiter
|
||||
}
|
||||
|
||||
func (d *Doubao) Config() driver.Config {
|
||||
@@ -61,6 +63,17 @@ func (d *Doubao) Init(ctx context.Context) error {
|
||||
d.UploadToken = uploadToken
|
||||
}
|
||||
|
||||
if d.LimitRate > 0 {
|
||||
d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Doubao) WaitLimit(ctx context.Context) error {
|
||||
if d.limiter != nil {
|
||||
return d.limiter.Wait(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -69,6 +82,10 @@ func (d *Doubao) Drop(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
|
||||
if err := d.WaitLimit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var files []model.Obj
|
||||
fileList, err := d.getFiles(dir.GetID(), "")
|
||||
if err != nil {
|
||||
@@ -95,6 +112,10 @@ func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) (
|
||||
}
|
||||
|
||||
func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
|
||||
if err := d.WaitLimit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var downloadUrl string
|
||||
|
||||
if u, ok := file.(*Object); ok {
|
||||
@@ -160,6 +181,10 @@ func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
|
||||
}
|
||||
|
||||
func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
|
||||
if err := d.WaitLimit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var r UploadNodeResp
|
||||
_, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(base.Json{
|
||||
@@ -177,6 +202,10 @@ func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin
|
||||
}
|
||||
|
||||
func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
|
||||
if err := d.WaitLimit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var r UploadNodeResp
|
||||
_, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(base.Json{
|
||||
@@ -191,6 +220,10 @@ func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
|
||||
}
|
||||
|
||||
func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
|
||||
if err := d.WaitLimit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var r BaseResp
|
||||
_, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(base.Json{
|
||||
@@ -207,6 +240,10 @@ func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj,
|
||||
}
|
||||
|
||||
func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error {
|
||||
if err := d.WaitLimit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var r BaseResp
|
||||
_, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}})
|
||||
@@ -215,6 +252,10 @@ func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error {
|
||||
}
|
||||
|
||||
func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
|
||||
if err := d.WaitLimit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 根据MIME类型确定数据类型
|
||||
mimetype := file.GetMimetype()
|
||||
dataType := FileDataType
|
||||
|
||||
@@ -13,6 +13,7 @@ type Addition struct {
|
||||
Cookie string `json:"cookie" type:"text"`
|
||||
UploadThread string `json:"upload_thread" default:"3"`
|
||||
DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"`
|
||||
LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"`
|
||||
}
|
||||
|
||||
var config = driver.Config{
|
||||
@@ -23,6 +24,10 @@ var config = driver.Config{
|
||||
|
||||
func init() {
|
||||
op.RegisterDriver(func() driver.Driver {
|
||||
return &Doubao{}
|
||||
return &Doubao{
|
||||
Addition: Addition{
|
||||
LimitRate: 2,
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -57,18 +57,22 @@ func setBody(body interface{}) base.ReqCallback {
|
||||
}
|
||||
|
||||
func handleFolderId(dir model.Obj) interface{} {
|
||||
if dir.GetID() == "" {
|
||||
return nil
|
||||
if isRootFolder(dir) {
|
||||
return nil // Root folder doesn't need folderId
|
||||
}
|
||||
return dir.GetID()
|
||||
}
|
||||
|
||||
func isRootFolder(dir model.Obj) bool {
|
||||
return dir.GetID() == ""
|
||||
}
|
||||
|
||||
// API layer methods
|
||||
|
||||
func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) {
|
||||
var files []MFile
|
||||
var body map[string]string
|
||||
if dir.GetPath() != "/" {
|
||||
if !isRootFolder(dir) {
|
||||
body = map[string]string{"folderId": dir.GetID()}
|
||||
} else {
|
||||
body = map[string]string{}
|
||||
@@ -85,7 +89,7 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) {
|
||||
func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) {
|
||||
var folders []MFolder
|
||||
var body map[string]string
|
||||
if dir.GetPath() != "/" {
|
||||
if !isRootFolder(dir) {
|
||||
body = map[string]string{"folderId": dir.GetID()}
|
||||
} else {
|
||||
body = map[string]string{}
|
||||
@@ -197,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt
|
||||
Reader: stream,
|
||||
UpdateProgress: up,
|
||||
})
|
||||
req := base.RestyClient.R().
|
||||
SetContext(ctx).
|
||||
SetFileReader("file", stream.GetName(), reader).
|
||||
SetFormData(map[string]string{
|
||||
"folderId": handleFolderId(dstDir).(string),
|
||||
|
||||
// Build form data, only add folderId if not root folder
|
||||
formData := map[string]string{
|
||||
"name": stream.GetName(),
|
||||
"comment": "",
|
||||
"isSensitive": "false",
|
||||
"force": "false",
|
||||
}).
|
||||
}
|
||||
|
||||
folderId := handleFolderId(dstDir)
|
||||
if folderId != nil {
|
||||
formData["folderId"] = folderId.(string)
|
||||
}
|
||||
|
||||
req := base.RestyClient.R().
|
||||
SetContext(ctx).
|
||||
SetFileReader("file", stream.GetName(), reader).
|
||||
SetFormData(formData).
|
||||
SetResult(&file).
|
||||
SetAuthToken(d.AccessToken)
|
||||
|
||||
|
||||
@@ -117,7 +117,7 @@ func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs
|
||||
if d.PassUAToUpsteam {
|
||||
userAgent := args.Header.Get("user-agent")
|
||||
if userAgent != "" {
|
||||
headers["User-Agent"] = base.UserAgent
|
||||
headers["User-Agent"] = userAgent
|
||||
}
|
||||
}
|
||||
// if PassIPToUpsteam is true, then pass the ip address to the upstream
|
||||
|
||||
@@ -217,11 +217,10 @@ func (d *QuarkOrUC) GetDetails(ctx context.Context) (*model.StorageDetails, erro
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
used := memberInfo.Data.UseCapacity
|
||||
total := memberInfo.Data.TotalCapacity
|
||||
return &model.StorageDetails{
|
||||
DiskUsage: model.DiskUsage{
|
||||
TotalSpace: memberInfo.Data.TotalCapacity,
|
||||
FreeSpace: memberInfo.Data.TotalCapacity - memberInfo.Data.UseCapacity,
|
||||
},
|
||||
DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/cron"
|
||||
@@ -27,6 +28,7 @@ type S3 struct {
|
||||
Session *session.Session
|
||||
client *s3.S3
|
||||
linkClient *s3.S3
|
||||
directUploadClient *s3.S3
|
||||
|
||||
config driver.Config
|
||||
cron *cron.Cron
|
||||
@@ -52,16 +54,18 @@ func (d *S3) Init(ctx context.Context) error {
|
||||
if err != nil {
|
||||
log.Errorln("Doge init session error:", err)
|
||||
}
|
||||
d.client = d.getClient(false)
|
||||
d.linkClient = d.getClient(true)
|
||||
d.client = d.getClient(ClientTypeNormal)
|
||||
d.linkClient = d.getClient(ClientTypeLink)
|
||||
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
|
||||
})
|
||||
}
|
||||
err := d.initSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.client = d.getClient(false)
|
||||
d.linkClient = d.getClient(true)
|
||||
d.client = d.getClient(ClientTypeNormal)
|
||||
d.linkClient = d.getClient(ClientTypeLink)
|
||||
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -210,4 +214,33 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *S3) GetDirectUploadTools() []string {
|
||||
if !d.EnableDirectUpload {
|
||||
return nil
|
||||
}
|
||||
return []string{"HttpDirect"}
|
||||
}
|
||||
|
||||
func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) {
|
||||
if !d.EnableDirectUpload {
|
||||
return nil, errs.NotImplement
|
||||
}
|
||||
path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false)
|
||||
req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{
|
||||
Bucket: &d.Bucket,
|
||||
Key: &path,
|
||||
})
|
||||
if req == nil {
|
||||
return nil, fmt.Errorf("failed to create PutObject request")
|
||||
}
|
||||
link, err := req.Presign(time.Hour * time.Duration(d.SignURLExpire))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.HttpDirectUploadInfo{
|
||||
UploadURL: link,
|
||||
Method: "PUT",
|
||||
}, nil
|
||||
}
|
||||
|
||||
var _ driver.Driver = (*S3)(nil)
|
||||
|
||||
@@ -21,6 +21,8 @@ type Addition struct {
|
||||
ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
|
||||
RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
|
||||
AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."`
|
||||
EnableDirectUpload bool `json:"enable_direct_upload" default:"false"`
|
||||
DirectUploadHost string `json:"direct_upload_host" required:"false"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -41,9 +41,15 @@ func (d *S3) initSession() error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *S3) getClient(link bool) *s3.S3 {
|
||||
const (
|
||||
ClientTypeNormal = iota
|
||||
ClientTypeLink
|
||||
ClientTypeDirectUpload
|
||||
)
|
||||
|
||||
func (d *S3) getClient(clientType int) *s3.S3 {
|
||||
client := s3.New(d.Session)
|
||||
if link && d.CustomHost != "" {
|
||||
if clientType == ClientTypeLink && d.CustomHost != "" {
|
||||
client.Handlers.Build.PushBack(func(r *request.Request) {
|
||||
if r.HTTPRequest.Method != http.MethodGet {
|
||||
return
|
||||
@@ -58,6 +64,20 @@ func (d *S3) getClient(link bool) *s3.S3 {
|
||||
}
|
||||
})
|
||||
}
|
||||
if clientType == ClientTypeDirectUpload && d.DirectUploadHost != "" {
|
||||
client.Handlers.Build.PushBack(func(r *request.Request) {
|
||||
if r.HTTPRequest.Method != http.MethodPut {
|
||||
return
|
||||
}
|
||||
split := strings.SplitN(d.DirectUploadHost, "://", 2)
|
||||
if utils.SliceContains([]string{"http", "https"}, split[0]) {
|
||||
r.HTTPRequest.URL.Scheme = split[0]
|
||||
r.HTTPRequest.URL.Host = split[1]
|
||||
} else {
|
||||
r.HTTPRequest.URL.Host = d.DirectUploadHost
|
||||
}
|
||||
})
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
|
||||
2
go.mod
2
go.mod
@@ -22,7 +22,7 @@ require (
|
||||
github.com/charmbracelet/bubbles v0.21.0
|
||||
github.com/charmbracelet/bubbletea v1.3.6
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e
|
||||
github.com/city404/v6-public-rpc-proto/go acca598c387c
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc
|
||||
github.com/coreos/go-oidc v2.3.0+incompatible
|
||||
github.com/deckarep/golang-set/v2 v2.8.0
|
||||
|
||||
@@ -177,6 +177,9 @@ func InitialSettings() []model.SettingItem {
|
||||
{Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
|
||||
{Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||
{Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC},
|
||||
{Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||
{Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||
{Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`},
|
||||
|
||||
// single settings
|
||||
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
|
||||
|
||||
@@ -56,6 +56,9 @@ const (
|
||||
ShareArchivePreview = "share_archive_preview"
|
||||
ShareForceProxy = "share_force_proxy"
|
||||
ShareSummaryContent = "share_summary_content"
|
||||
HandleHookAfterWriting = "handle_hook_after_writing"
|
||||
HandleHookRateLimit = "handle_hook_rate_limit"
|
||||
IgnoreSystemFiles = "ignore_system_files"
|
||||
|
||||
// index
|
||||
SearchIndex = "search_index"
|
||||
|
||||
@@ -11,6 +11,7 @@ var (
|
||||
ObjectAlreadyExists = errors.New("object already exists")
|
||||
NotFolder = errors.New("not a folder")
|
||||
NotFile = errors.New("not a file")
|
||||
IgnoredSystemFile = errors.New("system file upload ignored")
|
||||
)
|
||||
|
||||
func IsObjectNotFound(err error) bool {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/archive/tool"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/cache"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
@@ -20,10 +21,13 @@ import (
|
||||
gocache "github.com/OpenListTeam/go-cache"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
var archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
|
||||
var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
|
||||
var (
|
||||
archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
|
||||
archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
|
||||
)
|
||||
|
||||
func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
@@ -196,8 +200,10 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg
|
||||
return obj, archiveMetaProvider, err
|
||||
}
|
||||
|
||||
var archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
|
||||
var archiveListG singleflight.Group[[]model.Obj]
|
||||
var (
|
||||
archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
|
||||
archiveListG singleflight.Group[[]model.Obj]
|
||||
)
|
||||
|
||||
func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
@@ -397,8 +403,10 @@ type objWithLink struct {
|
||||
obj model.Obj
|
||||
}
|
||||
|
||||
var extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
|
||||
var extractG = singleflight.Group[*objWithLink]{}
|
||||
var (
|
||||
extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
|
||||
extractG = singleflight.Group[*objWithLink]{}
|
||||
)
|
||||
|
||||
func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
@@ -506,9 +514,9 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
|
||||
return errors.WithMessage(err, "failed to get dst dir")
|
||||
}
|
||||
|
||||
var newObjs []model.Obj
|
||||
switch s := storage.(type) {
|
||||
case driver.ArchiveDecompressResult:
|
||||
var newObjs []model.Obj
|
||||
newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
|
||||
if err == nil {
|
||||
if len(newObjs) > 0 {
|
||||
@@ -527,5 +535,31 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
|
||||
default:
|
||||
return errs.NotImplement
|
||||
}
|
||||
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||
onlyList := false
|
||||
targetPath := dstDirPath
|
||||
if newObjs != nil && len(newObjs) == 1 && newObjs[0].IsDir() {
|
||||
targetPath = stdpath.Join(dstDirPath, newObjs[0].GetName())
|
||||
} else if newObjs != nil && len(newObjs) == 1 && !newObjs[0].IsDir() {
|
||||
onlyList = true
|
||||
} else if args.PutIntoNewDir {
|
||||
targetPath = stdpath.Join(dstDirPath, strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName())))
|
||||
} else if innerBase := stdpath.Base(args.InnerPath); innerBase != "." && innerBase != "/" {
|
||||
targetPath = stdpath.Join(dstDirPath, innerBase)
|
||||
dstObj, e := GetUnwrap(ctx, storage, targetPath)
|
||||
onlyList = e != nil || !dstObj.IsDir()
|
||||
}
|
||||
if onlyList {
|
||||
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||
} else {
|
||||
var limiter *rate.Limiter
|
||||
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
|
||||
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
|
||||
limiter = rate.NewLimiter(rate.Limit(f), 1)
|
||||
}
|
||||
}
|
||||
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
|
||||
}
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
@@ -2,10 +2,11 @@ package op
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
stdpath "path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
var listG singleflight.Group[[]model.Obj]
|
||||
@@ -173,10 +175,10 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li
|
||||
mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path)
|
||||
}
|
||||
typeKey := args.Type
|
||||
if mode&driver.LinkCacheIP == 1 {
|
||||
if mode&driver.LinkCacheIP == driver.LinkCacheIP {
|
||||
typeKey += "/" + args.IP
|
||||
}
|
||||
if mode&driver.LinkCacheUA == 1 {
|
||||
if mode&driver.LinkCacheUA == driver.LinkCacheUA {
|
||||
typeKey += "/" + args.Header.Get("User-Agent")
|
||||
}
|
||||
key := Key(storage, path)
|
||||
@@ -310,7 +312,7 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
||||
srcDirPath := stdpath.Dir(srcPath)
|
||||
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
||||
if dstDirPath == srcDirPath {
|
||||
return stderrors.New("move in place")
|
||||
return errors.New("move in place")
|
||||
}
|
||||
srcRawObj, err := Get(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
@@ -343,8 +345,24 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
||||
}
|
||||
}
|
||||
default:
|
||||
return errs.NotImplement
|
||||
err = errs.NotImplement
|
||||
}
|
||||
|
||||
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||
if !srcObj.IsDir() {
|
||||
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||
} else {
|
||||
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
|
||||
var limiter *rate.Limiter
|
||||
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
|
||||
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
|
||||
limiter = rate.NewLimiter(rate.Limit(f), 1)
|
||||
}
|
||||
}
|
||||
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
@@ -397,7 +415,7 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
||||
srcPath = utils.FixAndCleanPath(srcPath)
|
||||
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
||||
if dstDirPath == stdpath.Dir(srcPath) {
|
||||
return stderrors.New("copy in place")
|
||||
return errors.New("copy in place")
|
||||
}
|
||||
srcRawObj, err := Get(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
@@ -428,8 +446,24 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
||||
}
|
||||
}
|
||||
default:
|
||||
return errs.NotImplement
|
||||
err = errs.NotImplement
|
||||
}
|
||||
|
||||
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||
if !srcObj.IsDir() {
|
||||
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||
} else {
|
||||
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
|
||||
var limiter *rate.Limiter
|
||||
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
|
||||
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
|
||||
limiter = rate.NewLimiter(rate.Limit(f), 1)
|
||||
}
|
||||
}
|
||||
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
@@ -557,6 +591,9 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
|
||||
err = Remove(ctx, storage, tempPath)
|
||||
}
|
||||
}
|
||||
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
@@ -601,6 +638,9 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url
|
||||
default:
|
||||
return errors.WithStack(errs.NotImplement)
|
||||
}
|
||||
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||
}
|
||||
log.Debugf("put url [%s](%s) done", dstName, url)
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
@@ -644,3 +684,8 @@ func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func needHandleObjsUpdateHook() bool {
|
||||
needHandle, _ := GetSettingItemByKey(conf.HandleHookAfterWriting)
|
||||
return needHandle != nil && (needHandle.Value == "true" || needHandle.Value == "1")
|
||||
}
|
||||
|
||||
125
internal/op/recursive_list.go
Normal file
125
internal/op/recursive_list.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package op
|
||||
|
||||
import (
|
||||
"context"
|
||||
stdpath "path"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
var (
|
||||
ManualScanCancel = atomic.Pointer[context.CancelFunc]{}
|
||||
ScannedCount = atomic.Uint64{}
|
||||
)
|
||||
|
||||
func ManualScanRunning() bool {
|
||||
return ManualScanCancel.Load() != nil
|
||||
}
|
||||
|
||||
func BeginManualScan(rawPath string, limit float64) error {
|
||||
rawPath = utils.FixAndCleanPath(rawPath)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
if !ManualScanCancel.CompareAndSwap(nil, &cancel) {
|
||||
cancel()
|
||||
return errors.New("manual scan is running, please try later")
|
||||
}
|
||||
ScannedCount.Store(0)
|
||||
go func() {
|
||||
defer func() { (*ManualScanCancel.Swap(nil))() }()
|
||||
err := RecursivelyList(ctx, rawPath, rate.Limit(limit), &ScannedCount)
|
||||
if err != nil {
|
||||
log.Errorf("failed recursively list: %v", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func StopManualScan() {
|
||||
c := ManualScanCancel.Load()
|
||||
if c != nil {
|
||||
(*c)()
|
||||
}
|
||||
}
|
||||
|
||||
func RecursivelyList(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64) error {
|
||||
storage, actualPath, err := GetStorageAndActualPath(rawPath)
|
||||
if err != nil && !errors.Is(err, errs.StorageNotFound) {
|
||||
return err
|
||||
} else if err == nil {
|
||||
var limiter *rate.Limiter
|
||||
if limit > .0 {
|
||||
limiter = rate.NewLimiter(limit, 1)
|
||||
}
|
||||
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
|
||||
} else {
|
||||
var wg sync.WaitGroup
|
||||
recursivelyListVirtual(ctx, rawPath, limit, counter, &wg)
|
||||
wg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func recursivelyListVirtual(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64, wg *sync.WaitGroup) {
|
||||
objs := GetStorageVirtualFilesByPath(rawPath)
|
||||
if counter != nil {
|
||||
counter.Add(uint64(len(objs)))
|
||||
}
|
||||
for _, obj := range objs {
|
||||
if utils.IsCanceled(ctx) {
|
||||
return
|
||||
}
|
||||
nextPath := stdpath.Join(rawPath, obj.GetName())
|
||||
storage, actualPath, err := GetStorageAndActualPath(nextPath)
|
||||
if err != nil && !errors.Is(err, errs.StorageNotFound) {
|
||||
log.Errorf("error recursively list: failed get storage [%s]: %v", nextPath, err)
|
||||
} else if err == nil {
|
||||
var limiter *rate.Limiter
|
||||
if limit > .0 {
|
||||
limiter = rate.NewLimiter(limit, 1)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
|
||||
}()
|
||||
} else {
|
||||
recursivelyListVirtual(ctx, nextPath, limit, counter, wg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func RecursivelyListStorage(ctx context.Context, storage driver.Driver, actualPath string, limiter *rate.Limiter, counter *atomic.Uint64) {
|
||||
objs, err := List(ctx, storage, actualPath, model.ListArgs{Refresh: true})
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Errorf("error recursively list: failed list (%s)[%s]: %v", storage.GetStorage().MountPath, actualPath, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if counter != nil {
|
||||
counter.Add(uint64(len(objs)))
|
||||
}
|
||||
for _, obj := range objs {
|
||||
if utils.IsCanceled(ctx) {
|
||||
return
|
||||
}
|
||||
if !obj.IsDir() {
|
||||
continue
|
||||
}
|
||||
if limiter != nil {
|
||||
if err = limiter.Wait(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
nextPath := stdpath.Join(actualPath, obj.GetName())
|
||||
RecursivelyListStorage(ctx, storage, nextPath, limiter, counter)
|
||||
}
|
||||
}
|
||||
@@ -358,16 +358,21 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string,
|
||||
DriverName: d.Config().Name,
|
||||
},
|
||||
}
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel()
|
||||
details, err := GetStorageDetails(timeoutCtx, d, refresh)
|
||||
resultChan := make(chan *model.StorageDetails, 1)
|
||||
go func(dri driver.Driver) {
|
||||
details, err := GetStorageDetails(ctx, dri, refresh)
|
||||
if err != nil {
|
||||
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
|
||||
log.Errorf("failed get %s storage details: %+v", d.GetStorage().MountPath, err)
|
||||
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
ret.StorageDetails = details
|
||||
resultChan <- details
|
||||
}(d)
|
||||
select {
|
||||
case r := <-resultChan:
|
||||
ret.StorageDetails = r
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
return ret
|
||||
})
|
||||
}
|
||||
|
||||
@@ -28,3 +28,11 @@ func GetInt(key string, defaultVal int) int {
|
||||
func GetBool(key string) bool {
|
||||
return GetStr(key) == "true" || GetStr(key) == "1"
|
||||
}
|
||||
|
||||
func GetFloat(key string, defaultVal float64) float64 {
|
||||
f, err := strconv.ParseFloat(GetStr(key), 64)
|
||||
if err != nil {
|
||||
return defaultVal
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
@@ -5,11 +5,14 @@ import (
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type SrcPathToRemove string
|
||||
@@ -27,13 +30,32 @@ func RefreshAndRemove(dstPath string, payloads ...any) {
|
||||
if dstNeedRefresh {
|
||||
op.Cache.DeleteDirectory(dstStorage, dstActualPath)
|
||||
}
|
||||
dstNeedHandleHook := setting.GetBool(conf.HandleHookAfterWriting)
|
||||
dstHandleHookLimit := setting.GetFloat(conf.HandleHookRateLimit, .0)
|
||||
var listLimiter *rate.Limiter
|
||||
if dstNeedRefresh && dstNeedHandleHook && dstHandleHookLimit > .0 {
|
||||
listLimiter = rate.NewLimiter(rate.Limit(dstHandleHookLimit), 1)
|
||||
}
|
||||
var ctx context.Context
|
||||
for _, payload := range payloads {
|
||||
switch p := payload.(type) {
|
||||
case DstPathToRefresh:
|
||||
if dstNeedRefresh {
|
||||
if dstNeedHandleHook {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if listLimiter != nil {
|
||||
_ = listLimiter.Wait(ctx)
|
||||
}
|
||||
_, e := op.List(ctx, dstStorage, string(p), model.ListArgs{Refresh: true})
|
||||
if e != nil {
|
||||
log.Errorf("failed handle objs update hook: %v", e)
|
||||
}
|
||||
} else {
|
||||
op.Cache.DeleteDirectory(dstStorage, string(p))
|
||||
}
|
||||
}
|
||||
case SrcPathToRemove:
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
|
||||
@@ -185,3 +185,20 @@ const (
|
||||
GB
|
||||
TB
|
||||
)
|
||||
|
||||
// IsSystemFile checks if a filename is a common system file that should be ignored
|
||||
// Returns true for files like .DS_Store, desktop.ini, Thumbs.db, and Apple Double files (._*)
|
||||
func IsSystemFile(filename string) bool {
|
||||
// Common system files
|
||||
switch filename {
|
||||
case ".DS_Store", "desktop.ini", "Thumbs.db":
|
||||
return true
|
||||
}
|
||||
|
||||
// Apple Double files (._*)
|
||||
if strings.HasPrefix(filename, "._") {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
42
pkg/utils/file_test.go
Normal file
42
pkg/utils/file_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsSystemFile(t *testing.T) {
|
||||
testCases := []struct {
|
||||
filename string
|
||||
expected bool
|
||||
}{
|
||||
// System files that should be filtered
|
||||
{".DS_Store", true},
|
||||
{"desktop.ini", true},
|
||||
{"Thumbs.db", true},
|
||||
{"._test.txt", true},
|
||||
{"._", true},
|
||||
{"._somefile", true},
|
||||
{"._folder_name", true},
|
||||
|
||||
// Regular files that should not be filtered
|
||||
{"test.txt", false},
|
||||
{"file.pdf", false},
|
||||
{"document.docx", false},
|
||||
{".gitignore", false},
|
||||
{".env", false},
|
||||
{"_underscore.txt", false},
|
||||
{"normal_file.txt", false},
|
||||
{"", false},
|
||||
{".hidden", false},
|
||||
{"..special", false},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.filename, func(t *testing.T) {
|
||||
result := IsSystemFile(tc.filename)
|
||||
if result != tc.expected {
|
||||
t.Errorf("IsSystemFile(%q) = %v, want %v", tc.filename, result, tc.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,9 @@ import (
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||
ftpserver "github.com/fclairamb/ftpserverlib"
|
||||
"github.com/pkg/errors"
|
||||
@@ -49,6 +51,11 @@ func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if system file should be ignored
|
||||
_, name := stdpath.Split(path)
|
||||
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
|
||||
return nil, errs.IgnoredSystemFile
|
||||
}
|
||||
tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -150,6 +157,11 @@ func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length i
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if system file should be ignored
|
||||
_, name := stdpath.Split(path)
|
||||
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
|
||||
return nil, errs.IgnoredSystemFile
|
||||
}
|
||||
if trunc {
|
||||
_ = fs.Remove(ctx, path)
|
||||
}
|
||||
|
||||
@@ -8,8 +8,10 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/task"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
@@ -28,6 +30,14 @@ func getLastModified(c *gin.Context) time.Time {
|
||||
return lastModified
|
||||
}
|
||||
|
||||
// shouldIgnoreSystemFile checks if the filename should be ignored based on settings
|
||||
func shouldIgnoreSystemFile(filename string) bool {
|
||||
if setting.GetBool(conf.IgnoreSystemFiles) {
|
||||
return utils.IsSystemFile(filename)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func FsStream(c *gin.Context) {
|
||||
defer func() {
|
||||
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
|
||||
@@ -56,6 +66,11 @@ func FsStream(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
dir, name := stdpath.Split(path)
|
||||
// Check if system file should be ignored
|
||||
if shouldIgnoreSystemFile(name) {
|
||||
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
|
||||
return
|
||||
}
|
||||
// 如果请求头 Content-Length 和 X-File-Size 都没有,则 size=-1,表示未知大小的流式上传
|
||||
size := c.Request.ContentLength
|
||||
if size < 0 {
|
||||
@@ -160,6 +175,11 @@ func FsForm(c *gin.Context) {
|
||||
}
|
||||
defer f.Close()
|
||||
dir, name := stdpath.Split(path)
|
||||
// Check if system file should be ignored
|
||||
if shouldIgnoreSystemFile(name) {
|
||||
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
|
||||
return
|
||||
}
|
||||
h := make(map[*utils.HashType]string)
|
||||
if md5 := c.GetHeader("X-File-Md5"); md5 != "" {
|
||||
h[utils.MD5] = md5
|
||||
|
||||
47
server/handles/scan.go
Normal file
47
server/handles/scan.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package handles
|
||||
|
||||
import (
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type ManualScanReq struct {
|
||||
Path string `json:"path"`
|
||||
Limit float64 `json:"limit"`
|
||||
}
|
||||
|
||||
func StartManualScan(c *gin.Context) {
|
||||
var req ManualScanReq
|
||||
if err := c.ShouldBind(&req); err != nil {
|
||||
common.ErrorResp(c, err, 400)
|
||||
return
|
||||
}
|
||||
if err := op.BeginManualScan(req.Path, req.Limit); err != nil {
|
||||
common.ErrorResp(c, err, 400)
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c)
|
||||
}
|
||||
|
||||
func StopManualScan(c *gin.Context) {
|
||||
if !op.ManualScanRunning() {
|
||||
common.ErrorStrResp(c, "manual scan is not running", 400)
|
||||
return
|
||||
}
|
||||
op.StopManualScan()
|
||||
common.SuccessResp(c)
|
||||
}
|
||||
|
||||
type ManualScanResp struct {
|
||||
ObjCount uint64 `json:"obj_count"`
|
||||
IsDone bool `json:"is_done"`
|
||||
}
|
||||
|
||||
func GetManualScanProgress(c *gin.Context) {
|
||||
ret := ManualScanResp{
|
||||
ObjCount: op.ScannedCount.Load(),
|
||||
IsDone: !op.ManualScanRunning(),
|
||||
}
|
||||
common.SuccessResp(c, ret)
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
@@ -24,9 +23,15 @@ type StorageResp struct {
|
||||
MountDetails *model.StorageDetails `json:"mount_details,omitempty"`
|
||||
}
|
||||
|
||||
func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
|
||||
type detailWithIndex struct {
|
||||
idx int
|
||||
val *model.StorageDetails
|
||||
}
|
||||
|
||||
func makeStorageResp(ctx *gin.Context, storages []model.Storage) []*StorageResp {
|
||||
ret := make([]*StorageResp, len(storages))
|
||||
var wg sync.WaitGroup
|
||||
detailsChan := make(chan detailWithIndex, len(storages))
|
||||
workerCount := 0
|
||||
for i, s := range storages {
|
||||
ret[i] = &StorageResp{
|
||||
Storage: s,
|
||||
@@ -43,22 +48,26 @@ func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ctx, cancel := context.WithTimeout(c, time.Second*3)
|
||||
defer cancel()
|
||||
details, err := op.GetStorageDetails(ctx, d)
|
||||
if err != nil {
|
||||
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
|
||||
log.Errorf("failed get %s details: %+v", s.MountPath, err)
|
||||
workerCount++
|
||||
go func(dri driver.Driver, idx int) {
|
||||
details, e := op.GetStorageDetails(ctx, dri)
|
||||
if e != nil {
|
||||
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
|
||||
log.Errorf("failed get %s details: %+v", dri.GetStorage().MountPath, e)
|
||||
}
|
||||
return
|
||||
}
|
||||
ret[i].MountDetails = details
|
||||
}()
|
||||
detailsChan <- detailWithIndex{idx: idx, val: details}
|
||||
}(d, i)
|
||||
}
|
||||
for workerCount > 0 {
|
||||
select {
|
||||
case r := <-detailsChan:
|
||||
ret[r.idx].MountDetails = r.val
|
||||
workerCount--
|
||||
case <-time.After(time.Second * 3):
|
||||
workerCount = 0
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
return ret
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
func SearchIndex(c *gin.Context) {
|
||||
mode := setting.GetStr(conf.SearchIndex)
|
||||
if mode == "none" {
|
||||
common.ErrorResp(c, errs.SearchNotAvailable, 500)
|
||||
common.ErrorResp(c, errs.SearchNotAvailable, 404)
|
||||
c.Abort()
|
||||
} else {
|
||||
c.Next()
|
||||
|
||||
@@ -179,6 +179,11 @@ func admin(g *gin.RouterGroup) {
|
||||
index.POST("/stop", middlewares.SearchIndex, handles.StopIndex)
|
||||
index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex)
|
||||
index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
|
||||
|
||||
scan := g.Group("/scan")
|
||||
scan.POST("/start", handles.StartManualScan)
|
||||
scan.POST("/stop", handles.StopManualScan)
|
||||
scan.GET("/progress", handles.GetManualScanProgress)
|
||||
}
|
||||
|
||||
func fsAndShare(g *gin.RouterGroup) {
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
@@ -286,6 +287,10 @@ func (b *s3Backend) PutObject(
|
||||
Modified: ti,
|
||||
Ctime: time.Now(),
|
||||
}
|
||||
// Check if system file should be ignored
|
||||
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
|
||||
return result, errs.IgnoredSystemFile
|
||||
}
|
||||
stream := &stream.FileStream{
|
||||
Obj: &obj,
|
||||
Reader: input,
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/net"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
@@ -358,6 +359,10 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
|
||||
Modified: h.getModTime(r),
|
||||
Ctime: h.getCreateTime(r),
|
||||
}
|
||||
// Check if system file should be ignored
|
||||
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
|
||||
return http.StatusForbidden, errs.IgnoredSystemFile
|
||||
}
|
||||
fsStream := &stream.FileStream{
|
||||
Obj: &obj,
|
||||
Reader: r.Body,
|
||||
|
||||
Reference in New Issue
Block a user