mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2025-11-25 19:37:41 +08:00
Compare commits
10 Commits
v4.1.7
...
renovate/g
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54ab52ca00 | ||
|
|
3989d35abd | ||
|
|
72e2ae1f14 | ||
|
|
3e37f575d8 | ||
|
|
c0d480366d | ||
|
|
9de7561154 | ||
|
|
0866b9075f | ||
|
|
055696f576 | ||
|
|
854415160c | ||
|
|
8f4f7d1291 |
16
.github/ISSUE_TEMPLATE/00-bug_report_zh.yml
vendored
16
.github/ISSUE_TEMPLATE/00-bug_report_zh.yml
vendored
@@ -13,7 +13,7 @@ body:
|
|||||||
attributes:
|
attributes:
|
||||||
label: 请确认以下事项
|
label: 请确认以下事项
|
||||||
description: |
|
description: |
|
||||||
您必须勾选以下内容,否则您的问题可能会被直接关闭。
|
您必须确认、同意并勾选以下内容,否则您的问题一定会被直接关闭。
|
||||||
或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。
|
或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。
|
||||||
options:
|
options:
|
||||||
- label: |
|
- label: |
|
||||||
@@ -59,6 +59,14 @@ body:
|
|||||||
label: 问题描述(必填)
|
label: 问题描述(必填)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
|
- type: textarea
|
||||||
|
id: logs
|
||||||
|
attributes:
|
||||||
|
label: 日志(必填)
|
||||||
|
description: |
|
||||||
|
请复制粘贴错误日志,或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug)
|
||||||
|
validations:
|
||||||
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
id: config
|
id: config
|
||||||
attributes:
|
attributes:
|
||||||
@@ -67,12 +75,6 @@ body:
|
|||||||
请提供您的`OpenList`应用的配置文件,并截图相关存储配置。(可隐藏隐私字段)
|
请提供您的`OpenList`应用的配置文件,并截图相关存储配置。(可隐藏隐私字段)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
|
||||||
id: logs
|
|
||||||
attributes:
|
|
||||||
label: 日志(可选)
|
|
||||||
description: |
|
|
||||||
请复制粘贴错误日志,或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug)
|
|
||||||
- type: textarea
|
- type: textarea
|
||||||
id: reproduction
|
id: reproduction
|
||||||
attributes:
|
attributes:
|
||||||
|
|||||||
16
.github/ISSUE_TEMPLATE/01-bug_report_en.yml
vendored
16
.github/ISSUE_TEMPLATE/01-bug_report_en.yml
vendored
@@ -13,7 +13,7 @@ body:
|
|||||||
attributes:
|
attributes:
|
||||||
label: Please confirm the following
|
label: Please confirm the following
|
||||||
description: |
|
description: |
|
||||||
You must check all the following, otherwise your issue may be closed directly.
|
You must confirm, agree, and check all the following, otherwise your issue will definitely be closed directly.
|
||||||
Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions).
|
Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions).
|
||||||
options:
|
options:
|
||||||
- label: |
|
- label: |
|
||||||
@@ -59,6 +59,14 @@ body:
|
|||||||
label: Bug Description (required)
|
label: Bug Description (required)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
|
- type: textarea
|
||||||
|
id: logs
|
||||||
|
attributes:
|
||||||
|
label: Logs (required)
|
||||||
|
description: |
|
||||||
|
Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs)
|
||||||
|
validations:
|
||||||
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
id: config
|
id: config
|
||||||
attributes:
|
attributes:
|
||||||
@@ -67,12 +75,6 @@ body:
|
|||||||
Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields)
|
Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
|
||||||
id: logs
|
|
||||||
attributes:
|
|
||||||
label: Logs (optional)
|
|
||||||
description: |
|
|
||||||
Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs)
|
|
||||||
- type: textarea
|
- type: textarea
|
||||||
id: reproduction
|
id: reproduction
|
||||||
attributes:
|
attributes:
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ func init() {
|
|||||||
return &Pan123{
|
return &Pan123{
|
||||||
Addition: Addition{
|
Addition: Addition{
|
||||||
UploadThread: 3,
|
UploadThread: 3,
|
||||||
|
Platform: "web",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
stdpath "path"
|
stdpath "path"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||||
@@ -17,9 +16,15 @@ import (
|
|||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type detailWithIndex struct {
|
||||||
|
idx int
|
||||||
|
val *model.StorageDetails
|
||||||
|
}
|
||||||
|
|
||||||
func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj {
|
func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj {
|
||||||
var objs []model.Obj
|
var objs []model.Obj
|
||||||
var wg sync.WaitGroup
|
detailsChan := make(chan detailWithIndex, len(d.pathMap))
|
||||||
|
workerCount := 0
|
||||||
for _, k := range d.rootOrder {
|
for _, k := range d.rootOrder {
|
||||||
obj := model.Object{
|
obj := model.Object{
|
||||||
Name: k,
|
Name: k,
|
||||||
@@ -47,22 +52,26 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model
|
|||||||
DriverName: remoteDriver.Config().Name,
|
DriverName: remoteDriver.Config().Name,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
wg.Add(1)
|
workerCount++
|
||||||
go func() {
|
go func(dri driver.Driver, i int) {
|
||||||
defer wg.Done()
|
details, e := op.GetStorageDetails(ctx, dri, refresh)
|
||||||
c, cancel := context.WithTimeout(ctx, time.Second)
|
|
||||||
defer cancel()
|
|
||||||
details, e := op.GetStorageDetails(c, remoteDriver, refresh)
|
|
||||||
if e != nil {
|
if e != nil {
|
||||||
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
|
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
|
||||||
log.Errorf("failed get %s storage details: %+v", remoteDriver.GetStorage().MountPath, e)
|
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e)
|
||||||
}
|
}
|
||||||
return
|
|
||||||
}
|
}
|
||||||
objs[idx].(*model.ObjStorageDetails).StorageDetails = details
|
detailsChan <- detailWithIndex{idx: i, val: details}
|
||||||
}()
|
}(remoteDriver, idx)
|
||||||
|
}
|
||||||
|
for workerCount > 0 {
|
||||||
|
select {
|
||||||
|
case r := <-detailsChan:
|
||||||
|
objs[r.idx].(*model.ObjStorageDetails).StorageDetails = r.val
|
||||||
|
workerCount--
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
workerCount = 0
|
||||||
|
}
|
||||||
}
|
}
|
||||||
wg.Wait()
|
|
||||||
return objs
|
return objs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||||
"github.com/go-resty/resty/v2"
|
"github.com/go-resty/resty/v2"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Doubao struct {
|
type Doubao struct {
|
||||||
@@ -23,6 +24,7 @@ type Doubao struct {
|
|||||||
*UploadToken
|
*UploadToken
|
||||||
UserId string
|
UserId string
|
||||||
uploadThread int
|
uploadThread int
|
||||||
|
limiter *rate.Limiter
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) Config() driver.Config {
|
func (d *Doubao) Config() driver.Config {
|
||||||
@@ -61,6 +63,17 @@ func (d *Doubao) Init(ctx context.Context) error {
|
|||||||
d.UploadToken = uploadToken
|
d.UploadToken = uploadToken
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if d.LimitRate > 0 {
|
||||||
|
d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Doubao) WaitLimit(ctx context.Context) error {
|
||||||
|
if d.limiter != nil {
|
||||||
|
return d.limiter.Wait(ctx)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,6 +82,10 @@ func (d *Doubao) Drop(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
|
func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
|
||||||
|
if err := d.WaitLimit(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
var files []model.Obj
|
var files []model.Obj
|
||||||
fileList, err := d.getFiles(dir.GetID(), "")
|
fileList, err := d.getFiles(dir.GetID(), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -95,6 +112,10 @@ func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
|
func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
|
||||||
|
if err := d.WaitLimit(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
var downloadUrl string
|
var downloadUrl string
|
||||||
|
|
||||||
if u, ok := file.(*Object); ok {
|
if u, ok := file.(*Object); ok {
|
||||||
@@ -160,6 +181,10 @@ func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
|
func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
|
||||||
|
if err := d.WaitLimit(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
var r UploadNodeResp
|
var r UploadNodeResp
|
||||||
_, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) {
|
_, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) {
|
||||||
req.SetBody(base.Json{
|
req.SetBody(base.Json{
|
||||||
@@ -177,6 +202,10 @@ func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
|
func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
|
||||||
|
if err := d.WaitLimit(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
var r UploadNodeResp
|
var r UploadNodeResp
|
||||||
_, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) {
|
_, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) {
|
||||||
req.SetBody(base.Json{
|
req.SetBody(base.Json{
|
||||||
@@ -191,6 +220,10 @@ func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
|
func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
|
||||||
|
if err := d.WaitLimit(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
var r BaseResp
|
var r BaseResp
|
||||||
_, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) {
|
_, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) {
|
||||||
req.SetBody(base.Json{
|
req.SetBody(base.Json{
|
||||||
@@ -207,6 +240,10 @@ func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error {
|
func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error {
|
||||||
|
if err := d.WaitLimit(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
var r BaseResp
|
var r BaseResp
|
||||||
_, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) {
|
_, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) {
|
||||||
req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}})
|
req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}})
|
||||||
@@ -215,6 +252,10 @@ func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
|
func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
|
||||||
|
if err := d.WaitLimit(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// 根据MIME类型确定数据类型
|
// 根据MIME类型确定数据类型
|
||||||
mimetype := file.GetMimetype()
|
mimetype := file.GetMimetype()
|
||||||
dataType := FileDataType
|
dataType := FileDataType
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ type Addition struct {
|
|||||||
Cookie string `json:"cookie" type:"text"`
|
Cookie string `json:"cookie" type:"text"`
|
||||||
UploadThread string `json:"upload_thread" default:"3"`
|
UploadThread string `json:"upload_thread" default:"3"`
|
||||||
DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"`
|
DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"`
|
||||||
|
LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var config = driver.Config{
|
var config = driver.Config{
|
||||||
@@ -23,6 +24,10 @@ var config = driver.Config{
|
|||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
op.RegisterDriver(func() driver.Driver {
|
op.RegisterDriver(func() driver.Driver {
|
||||||
return &Doubao{}
|
return &Doubao{
|
||||||
|
Addition: Addition{
|
||||||
|
LimitRate: 2,
|
||||||
|
},
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,18 +57,22 @@ func setBody(body interface{}) base.ReqCallback {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func handleFolderId(dir model.Obj) interface{} {
|
func handleFolderId(dir model.Obj) interface{} {
|
||||||
if dir.GetID() == "" {
|
if isRootFolder(dir) {
|
||||||
return nil
|
return nil // Root folder doesn't need folderId
|
||||||
}
|
}
|
||||||
return dir.GetID()
|
return dir.GetID()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isRootFolder(dir model.Obj) bool {
|
||||||
|
return dir.GetID() == ""
|
||||||
|
}
|
||||||
|
|
||||||
// API layer methods
|
// API layer methods
|
||||||
|
|
||||||
func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) {
|
func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) {
|
||||||
var files []MFile
|
var files []MFile
|
||||||
var body map[string]string
|
var body map[string]string
|
||||||
if dir.GetPath() != "/" {
|
if !isRootFolder(dir) {
|
||||||
body = map[string]string{"folderId": dir.GetID()}
|
body = map[string]string{"folderId": dir.GetID()}
|
||||||
} else {
|
} else {
|
||||||
body = map[string]string{}
|
body = map[string]string{}
|
||||||
@@ -85,7 +89,7 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) {
|
|||||||
func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) {
|
func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) {
|
||||||
var folders []MFolder
|
var folders []MFolder
|
||||||
var body map[string]string
|
var body map[string]string
|
||||||
if dir.GetPath() != "/" {
|
if !isRootFolder(dir) {
|
||||||
body = map[string]string{"folderId": dir.GetID()}
|
body = map[string]string{"folderId": dir.GetID()}
|
||||||
} else {
|
} else {
|
||||||
body = map[string]string{}
|
body = map[string]string{}
|
||||||
@@ -197,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt
|
|||||||
Reader: stream,
|
Reader: stream,
|
||||||
UpdateProgress: up,
|
UpdateProgress: up,
|
||||||
})
|
})
|
||||||
req := base.RestyClient.R().
|
|
||||||
SetContext(ctx).
|
// Build form data, only add folderId if not root folder
|
||||||
SetFileReader("file", stream.GetName(), reader).
|
formData := map[string]string{
|
||||||
SetFormData(map[string]string{
|
|
||||||
"folderId": handleFolderId(dstDir).(string),
|
|
||||||
"name": stream.GetName(),
|
"name": stream.GetName(),
|
||||||
"comment": "",
|
"comment": "",
|
||||||
"isSensitive": "false",
|
"isSensitive": "false",
|
||||||
"force": "false",
|
"force": "false",
|
||||||
}).
|
}
|
||||||
|
|
||||||
|
folderId := handleFolderId(dstDir)
|
||||||
|
if folderId != nil {
|
||||||
|
formData["folderId"] = folderId.(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := base.RestyClient.R().
|
||||||
|
SetContext(ctx).
|
||||||
|
SetFileReader("file", stream.GetName(), reader).
|
||||||
|
SetFormData(formData).
|
||||||
SetResult(&file).
|
SetResult(&file).
|
||||||
SetAuthToken(d.AccessToken)
|
SetAuthToken(d.AccessToken)
|
||||||
|
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs
|
|||||||
if d.PassUAToUpsteam {
|
if d.PassUAToUpsteam {
|
||||||
userAgent := args.Header.Get("user-agent")
|
userAgent := args.Header.Get("user-agent")
|
||||||
if userAgent != "" {
|
if userAgent != "" {
|
||||||
headers["User-Agent"] = base.UserAgent
|
headers["User-Agent"] = userAgent
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// if PassIPToUpsteam is true, then pass the ip address to the upstream
|
// if PassIPToUpsteam is true, then pass the ip address to the upstream
|
||||||
|
|||||||
@@ -217,11 +217,10 @@ func (d *QuarkOrUC) GetDetails(ctx context.Context) (*model.StorageDetails, erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
used := memberInfo.Data.UseCapacity
|
||||||
|
total := memberInfo.Data.TotalCapacity
|
||||||
return &model.StorageDetails{
|
return &model.StorageDetails{
|
||||||
DiskUsage: model.DiskUsage{
|
DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
|
||||||
TotalSpace: memberInfo.Data.TotalCapacity,
|
|
||||||
FreeSpace: memberInfo.Data.TotalCapacity - memberInfo.Data.UseCapacity,
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||||
"github.com/OpenListTeam/OpenList/v4/pkg/cron"
|
"github.com/OpenListTeam/OpenList/v4/pkg/cron"
|
||||||
@@ -27,6 +28,7 @@ type S3 struct {
|
|||||||
Session *session.Session
|
Session *session.Session
|
||||||
client *s3.S3
|
client *s3.S3
|
||||||
linkClient *s3.S3
|
linkClient *s3.S3
|
||||||
|
directUploadClient *s3.S3
|
||||||
|
|
||||||
config driver.Config
|
config driver.Config
|
||||||
cron *cron.Cron
|
cron *cron.Cron
|
||||||
@@ -52,16 +54,18 @@ func (d *S3) Init(ctx context.Context) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorln("Doge init session error:", err)
|
log.Errorln("Doge init session error:", err)
|
||||||
}
|
}
|
||||||
d.client = d.getClient(false)
|
d.client = d.getClient(ClientTypeNormal)
|
||||||
d.linkClient = d.getClient(true)
|
d.linkClient = d.getClient(ClientTypeLink)
|
||||||
|
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
err := d.initSession()
|
err := d.initSession()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
d.client = d.getClient(false)
|
d.client = d.getClient(ClientTypeNormal)
|
||||||
d.linkClient = d.getClient(true)
|
d.linkClient = d.getClient(ClientTypeLink)
|
||||||
|
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -210,4 +214,33 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (d *S3) GetDirectUploadTools() []string {
|
||||||
|
if !d.EnableDirectUpload {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []string{"HttpDirect"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) {
|
||||||
|
if !d.EnableDirectUpload {
|
||||||
|
return nil, errs.NotImplement
|
||||||
|
}
|
||||||
|
path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false)
|
||||||
|
req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{
|
||||||
|
Bucket: &d.Bucket,
|
||||||
|
Key: &path,
|
||||||
|
})
|
||||||
|
if req == nil {
|
||||||
|
return nil, fmt.Errorf("failed to create PutObject request")
|
||||||
|
}
|
||||||
|
link, err := req.Presign(time.Hour * time.Duration(d.SignURLExpire))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &model.HttpDirectUploadInfo{
|
||||||
|
UploadURL: link,
|
||||||
|
Method: "PUT",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
var _ driver.Driver = (*S3)(nil)
|
var _ driver.Driver = (*S3)(nil)
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ type Addition struct {
|
|||||||
ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
|
ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
|
||||||
RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
|
RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
|
||||||
AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."`
|
AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."`
|
||||||
|
EnableDirectUpload bool `json:"enable_direct_upload" default:"false"`
|
||||||
|
DirectUploadHost string `json:"direct_upload_host" required:"false"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
|||||||
@@ -41,9 +41,15 @@ func (d *S3) initSession() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *S3) getClient(link bool) *s3.S3 {
|
const (
|
||||||
|
ClientTypeNormal = iota
|
||||||
|
ClientTypeLink
|
||||||
|
ClientTypeDirectUpload
|
||||||
|
)
|
||||||
|
|
||||||
|
func (d *S3) getClient(clientType int) *s3.S3 {
|
||||||
client := s3.New(d.Session)
|
client := s3.New(d.Session)
|
||||||
if link && d.CustomHost != "" {
|
if clientType == ClientTypeLink && d.CustomHost != "" {
|
||||||
client.Handlers.Build.PushBack(func(r *request.Request) {
|
client.Handlers.Build.PushBack(func(r *request.Request) {
|
||||||
if r.HTTPRequest.Method != http.MethodGet {
|
if r.HTTPRequest.Method != http.MethodGet {
|
||||||
return
|
return
|
||||||
@@ -58,6 +64,20 @@ func (d *S3) getClient(link bool) *s3.S3 {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
if clientType == ClientTypeDirectUpload && d.DirectUploadHost != "" {
|
||||||
|
client.Handlers.Build.PushBack(func(r *request.Request) {
|
||||||
|
if r.HTTPRequest.Method != http.MethodPut {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
split := strings.SplitN(d.DirectUploadHost, "://", 2)
|
||||||
|
if utils.SliceContains([]string{"http", "https"}, split[0]) {
|
||||||
|
r.HTTPRequest.URL.Scheme = split[0]
|
||||||
|
r.HTTPRequest.URL.Host = split[1]
|
||||||
|
} else {
|
||||||
|
r.HTTPRequest.URL.Host = d.DirectUploadHost
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
return client
|
return client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
2
go.mod
2
go.mod
@@ -22,7 +22,7 @@ require (
|
|||||||
github.com/charmbracelet/bubbles v0.21.0
|
github.com/charmbracelet/bubbles v0.21.0
|
||||||
github.com/charmbracelet/bubbletea v1.3.6
|
github.com/charmbracelet/bubbletea v1.3.6
|
||||||
github.com/charmbracelet/lipgloss v1.1.0
|
github.com/charmbracelet/lipgloss v1.1.0
|
||||||
github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e
|
github.com/city404/v6-public-rpc-proto/go acca598c387c
|
||||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc
|
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc
|
||||||
github.com/coreos/go-oidc v2.3.0+incompatible
|
github.com/coreos/go-oidc v2.3.0+incompatible
|
||||||
github.com/deckarep/golang-set/v2 v2.8.0
|
github.com/deckarep/golang-set/v2 v2.8.0
|
||||||
|
|||||||
@@ -177,6 +177,9 @@ func InitialSettings() []model.SettingItem {
|
|||||||
{Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
|
{Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
|
||||||
{Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
|
{Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||||
{Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC},
|
{Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC},
|
||||||
|
{Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||||
|
{Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||||
|
{Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`},
|
||||||
|
|
||||||
// single settings
|
// single settings
|
||||||
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
|
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
|
||||||
|
|||||||
@@ -56,6 +56,9 @@ const (
|
|||||||
ShareArchivePreview = "share_archive_preview"
|
ShareArchivePreview = "share_archive_preview"
|
||||||
ShareForceProxy = "share_force_proxy"
|
ShareForceProxy = "share_force_proxy"
|
||||||
ShareSummaryContent = "share_summary_content"
|
ShareSummaryContent = "share_summary_content"
|
||||||
|
HandleHookAfterWriting = "handle_hook_after_writing"
|
||||||
|
HandleHookRateLimit = "handle_hook_rate_limit"
|
||||||
|
IgnoreSystemFiles = "ignore_system_files"
|
||||||
|
|
||||||
// index
|
// index
|
||||||
SearchIndex = "search_index"
|
SearchIndex = "search_index"
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ var (
|
|||||||
ObjectAlreadyExists = errors.New("object already exists")
|
ObjectAlreadyExists = errors.New("object already exists")
|
||||||
NotFolder = errors.New("not a folder")
|
NotFolder = errors.New("not a folder")
|
||||||
NotFile = errors.New("not a file")
|
NotFile = errors.New("not a file")
|
||||||
|
IgnoredSystemFile = errors.New("system file upload ignored")
|
||||||
)
|
)
|
||||||
|
|
||||||
func IsObjectNotFound(err error) bool {
|
func IsObjectNotFound(err error) bool {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/archive/tool"
|
"github.com/OpenListTeam/OpenList/v4/internal/archive/tool"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/cache"
|
"github.com/OpenListTeam/OpenList/v4/internal/cache"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
@@ -20,10 +21,13 @@ import (
|
|||||||
gocache "github.com/OpenListTeam/go-cache"
|
gocache "github.com/OpenListTeam/go-cache"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
var archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
|
var (
|
||||||
var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
|
archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
|
||||||
|
archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
|
||||||
|
)
|
||||||
|
|
||||||
func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) {
|
func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) {
|
||||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||||
@@ -196,8 +200,10 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg
|
|||||||
return obj, archiveMetaProvider, err
|
return obj, archiveMetaProvider, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
|
var (
|
||||||
var archiveListG singleflight.Group[[]model.Obj]
|
archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
|
||||||
|
archiveListG singleflight.Group[[]model.Obj]
|
||||||
|
)
|
||||||
|
|
||||||
func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) {
|
func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) {
|
||||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||||
@@ -397,8 +403,10 @@ type objWithLink struct {
|
|||||||
obj model.Obj
|
obj model.Obj
|
||||||
}
|
}
|
||||||
|
|
||||||
var extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
|
var (
|
||||||
var extractG = singleflight.Group[*objWithLink]{}
|
extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
|
||||||
|
extractG = singleflight.Group[*objWithLink]{}
|
||||||
|
)
|
||||||
|
|
||||||
func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) {
|
func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) {
|
||||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||||
@@ -506,9 +514,9 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
|
|||||||
return errors.WithMessage(err, "failed to get dst dir")
|
return errors.WithMessage(err, "failed to get dst dir")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var newObjs []model.Obj
|
||||||
switch s := storage.(type) {
|
switch s := storage.(type) {
|
||||||
case driver.ArchiveDecompressResult:
|
case driver.ArchiveDecompressResult:
|
||||||
var newObjs []model.Obj
|
|
||||||
newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
|
newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if len(newObjs) > 0 {
|
if len(newObjs) > 0 {
|
||||||
@@ -527,5 +535,31 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
|
|||||||
default:
|
default:
|
||||||
return errs.NotImplement
|
return errs.NotImplement
|
||||||
}
|
}
|
||||||
|
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||||
|
onlyList := false
|
||||||
|
targetPath := dstDirPath
|
||||||
|
if newObjs != nil && len(newObjs) == 1 && newObjs[0].IsDir() {
|
||||||
|
targetPath = stdpath.Join(dstDirPath, newObjs[0].GetName())
|
||||||
|
} else if newObjs != nil && len(newObjs) == 1 && !newObjs[0].IsDir() {
|
||||||
|
onlyList = true
|
||||||
|
} else if args.PutIntoNewDir {
|
||||||
|
targetPath = stdpath.Join(dstDirPath, strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName())))
|
||||||
|
} else if innerBase := stdpath.Base(args.InnerPath); innerBase != "." && innerBase != "/" {
|
||||||
|
targetPath = stdpath.Join(dstDirPath, innerBase)
|
||||||
|
dstObj, e := GetUnwrap(ctx, storage, targetPath)
|
||||||
|
onlyList = e != nil || !dstObj.IsDir()
|
||||||
|
}
|
||||||
|
if onlyList {
|
||||||
|
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||||
|
} else {
|
||||||
|
var limiter *rate.Limiter
|
||||||
|
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
|
||||||
|
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
|
||||||
|
limiter = rate.NewLimiter(rate.Limit(f), 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,11 @@ package op
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
stderrors "errors"
|
|
||||||
stdpath "path"
|
stdpath "path"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
@@ -14,6 +15,7 @@ import (
|
|||||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
var listG singleflight.Group[[]model.Obj]
|
var listG singleflight.Group[[]model.Obj]
|
||||||
@@ -173,10 +175,10 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li
|
|||||||
mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path)
|
mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path)
|
||||||
}
|
}
|
||||||
typeKey := args.Type
|
typeKey := args.Type
|
||||||
if mode&driver.LinkCacheIP == 1 {
|
if mode&driver.LinkCacheIP == driver.LinkCacheIP {
|
||||||
typeKey += "/" + args.IP
|
typeKey += "/" + args.IP
|
||||||
}
|
}
|
||||||
if mode&driver.LinkCacheUA == 1 {
|
if mode&driver.LinkCacheUA == driver.LinkCacheUA {
|
||||||
typeKey += "/" + args.Header.Get("User-Agent")
|
typeKey += "/" + args.Header.Get("User-Agent")
|
||||||
}
|
}
|
||||||
key := Key(storage, path)
|
key := Key(storage, path)
|
||||||
@@ -310,7 +312,7 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
|||||||
srcDirPath := stdpath.Dir(srcPath)
|
srcDirPath := stdpath.Dir(srcPath)
|
||||||
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
||||||
if dstDirPath == srcDirPath {
|
if dstDirPath == srcDirPath {
|
||||||
return stderrors.New("move in place")
|
return errors.New("move in place")
|
||||||
}
|
}
|
||||||
srcRawObj, err := Get(ctx, storage, srcPath)
|
srcRawObj, err := Get(ctx, storage, srcPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -343,8 +345,24 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return errs.NotImplement
|
err = errs.NotImplement
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||||
|
if !srcObj.IsDir() {
|
||||||
|
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||||
|
} else {
|
||||||
|
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
|
||||||
|
var limiter *rate.Limiter
|
||||||
|
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
|
||||||
|
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
|
||||||
|
limiter = rate.NewLimiter(rate.Limit(f), 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -397,7 +415,7 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
|||||||
srcPath = utils.FixAndCleanPath(srcPath)
|
srcPath = utils.FixAndCleanPath(srcPath)
|
||||||
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
||||||
if dstDirPath == stdpath.Dir(srcPath) {
|
if dstDirPath == stdpath.Dir(srcPath) {
|
||||||
return stderrors.New("copy in place")
|
return errors.New("copy in place")
|
||||||
}
|
}
|
||||||
srcRawObj, err := Get(ctx, storage, srcPath)
|
srcRawObj, err := Get(ctx, storage, srcPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -428,8 +446,24 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return errs.NotImplement
|
err = errs.NotImplement
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||||
|
if !srcObj.IsDir() {
|
||||||
|
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||||
|
} else {
|
||||||
|
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
|
||||||
|
var limiter *rate.Limiter
|
||||||
|
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
|
||||||
|
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
|
||||||
|
limiter = rate.NewLimiter(rate.Limit(f), 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -557,6 +591,9 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
|
|||||||
err = Remove(ctx, storage, tempPath)
|
err = Remove(ctx, storage, tempPath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||||
|
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||||
|
}
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,6 +638,9 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url
|
|||||||
default:
|
default:
|
||||||
return errors.WithStack(errs.NotImplement)
|
return errors.WithStack(errs.NotImplement)
|
||||||
}
|
}
|
||||||
|
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
|
||||||
|
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
|
||||||
|
}
|
||||||
log.Debugf("put url [%s](%s) done", dstName, url)
|
log.Debugf("put url [%s](%s) done", dstName, url)
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
@@ -644,3 +684,8 @@ func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver
|
|||||||
}
|
}
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func needHandleObjsUpdateHook() bool {
|
||||||
|
needHandle, _ := GetSettingItemByKey(conf.HandleHookAfterWriting)
|
||||||
|
return needHandle != nil && (needHandle.Value == "true" || needHandle.Value == "1")
|
||||||
|
}
|
||||||
|
|||||||
125
internal/op/recursive_list.go
Normal file
125
internal/op/recursive_list.go
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
package op
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
stdpath "path"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ManualScanCancel = atomic.Pointer[context.CancelFunc]{}
|
||||||
|
ScannedCount = atomic.Uint64{}
|
||||||
|
)
|
||||||
|
|
||||||
|
func ManualScanRunning() bool {
|
||||||
|
return ManualScanCancel.Load() != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func BeginManualScan(rawPath string, limit float64) error {
|
||||||
|
rawPath = utils.FixAndCleanPath(rawPath)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
if !ManualScanCancel.CompareAndSwap(nil, &cancel) {
|
||||||
|
cancel()
|
||||||
|
return errors.New("manual scan is running, please try later")
|
||||||
|
}
|
||||||
|
ScannedCount.Store(0)
|
||||||
|
go func() {
|
||||||
|
defer func() { (*ManualScanCancel.Swap(nil))() }()
|
||||||
|
err := RecursivelyList(ctx, rawPath, rate.Limit(limit), &ScannedCount)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed recursively list: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func StopManualScan() {
|
||||||
|
c := ManualScanCancel.Load()
|
||||||
|
if c != nil {
|
||||||
|
(*c)()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func RecursivelyList(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64) error {
|
||||||
|
storage, actualPath, err := GetStorageAndActualPath(rawPath)
|
||||||
|
if err != nil && !errors.Is(err, errs.StorageNotFound) {
|
||||||
|
return err
|
||||||
|
} else if err == nil {
|
||||||
|
var limiter *rate.Limiter
|
||||||
|
if limit > .0 {
|
||||||
|
limiter = rate.NewLimiter(limit, 1)
|
||||||
|
}
|
||||||
|
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
|
||||||
|
} else {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
recursivelyListVirtual(ctx, rawPath, limit, counter, &wg)
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recursivelyListVirtual(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64, wg *sync.WaitGroup) {
|
||||||
|
objs := GetStorageVirtualFilesByPath(rawPath)
|
||||||
|
if counter != nil {
|
||||||
|
counter.Add(uint64(len(objs)))
|
||||||
|
}
|
||||||
|
for _, obj := range objs {
|
||||||
|
if utils.IsCanceled(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
nextPath := stdpath.Join(rawPath, obj.GetName())
|
||||||
|
storage, actualPath, err := GetStorageAndActualPath(nextPath)
|
||||||
|
if err != nil && !errors.Is(err, errs.StorageNotFound) {
|
||||||
|
log.Errorf("error recursively list: failed get storage [%s]: %v", nextPath, err)
|
||||||
|
} else if err == nil {
|
||||||
|
var limiter *rate.Limiter
|
||||||
|
if limit > .0 {
|
||||||
|
limiter = rate.NewLimiter(limit, 1)
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
|
||||||
|
}()
|
||||||
|
} else {
|
||||||
|
recursivelyListVirtual(ctx, nextPath, limit, counter, wg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func RecursivelyListStorage(ctx context.Context, storage driver.Driver, actualPath string, limiter *rate.Limiter, counter *atomic.Uint64) {
|
||||||
|
objs, err := List(ctx, storage, actualPath, model.ListArgs{Refresh: true})
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
log.Errorf("error recursively list: failed list (%s)[%s]: %v", storage.GetStorage().MountPath, actualPath, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if counter != nil {
|
||||||
|
counter.Add(uint64(len(objs)))
|
||||||
|
}
|
||||||
|
for _, obj := range objs {
|
||||||
|
if utils.IsCanceled(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !obj.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if limiter != nil {
|
||||||
|
if err = limiter.Wait(ctx); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nextPath := stdpath.Join(actualPath, obj.GetName())
|
||||||
|
RecursivelyListStorage(ctx, storage, nextPath, limiter, counter)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -358,16 +358,21 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string,
|
|||||||
DriverName: d.Config().Name,
|
DriverName: d.Config().Name,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
timeoutCtx, cancel := context.WithTimeout(ctx, time.Second)
|
resultChan := make(chan *model.StorageDetails, 1)
|
||||||
defer cancel()
|
go func(dri driver.Driver) {
|
||||||
details, err := GetStorageDetails(timeoutCtx, d, refresh)
|
details, err := GetStorageDetails(ctx, dri, refresh)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
|
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
|
||||||
log.Errorf("failed get %s storage details: %+v", d.GetStorage().MountPath, err)
|
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err)
|
||||||
}
|
}
|
||||||
return ret
|
|
||||||
}
|
}
|
||||||
ret.StorageDetails = details
|
resultChan <- details
|
||||||
|
}(d)
|
||||||
|
select {
|
||||||
|
case r := <-resultChan:
|
||||||
|
ret.StorageDetails = r
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
}
|
||||||
return ret
|
return ret
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,3 +28,11 @@ func GetInt(key string, defaultVal int) int {
|
|||||||
func GetBool(key string) bool {
|
func GetBool(key string) bool {
|
||||||
return GetStr(key) == "true" || GetStr(key) == "1"
|
return GetStr(key) == "true" || GetStr(key) == "1"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func GetFloat(key string, defaultVal float64) float64 {
|
||||||
|
f, err := strconv.ParseFloat(GetStr(key), 64)
|
||||||
|
if err != nil {
|
||||||
|
return defaultVal
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,11 +5,14 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
"path"
|
||||||
|
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
type SrcPathToRemove string
|
type SrcPathToRemove string
|
||||||
@@ -27,13 +30,32 @@ func RefreshAndRemove(dstPath string, payloads ...any) {
|
|||||||
if dstNeedRefresh {
|
if dstNeedRefresh {
|
||||||
op.Cache.DeleteDirectory(dstStorage, dstActualPath)
|
op.Cache.DeleteDirectory(dstStorage, dstActualPath)
|
||||||
}
|
}
|
||||||
|
dstNeedHandleHook := setting.GetBool(conf.HandleHookAfterWriting)
|
||||||
|
dstHandleHookLimit := setting.GetFloat(conf.HandleHookRateLimit, .0)
|
||||||
|
var listLimiter *rate.Limiter
|
||||||
|
if dstNeedRefresh && dstNeedHandleHook && dstHandleHookLimit > .0 {
|
||||||
|
listLimiter = rate.NewLimiter(rate.Limit(dstHandleHookLimit), 1)
|
||||||
|
}
|
||||||
var ctx context.Context
|
var ctx context.Context
|
||||||
for _, payload := range payloads {
|
for _, payload := range payloads {
|
||||||
switch p := payload.(type) {
|
switch p := payload.(type) {
|
||||||
case DstPathToRefresh:
|
case DstPathToRefresh:
|
||||||
if dstNeedRefresh {
|
if dstNeedRefresh {
|
||||||
|
if dstNeedHandleHook {
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
|
}
|
||||||
|
if listLimiter != nil {
|
||||||
|
_ = listLimiter.Wait(ctx)
|
||||||
|
}
|
||||||
|
_, e := op.List(ctx, dstStorage, string(p), model.ListArgs{Refresh: true})
|
||||||
|
if e != nil {
|
||||||
|
log.Errorf("failed handle objs update hook: %v", e)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
op.Cache.DeleteDirectory(dstStorage, string(p))
|
op.Cache.DeleteDirectory(dstStorage, string(p))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
case SrcPathToRemove:
|
case SrcPathToRemove:
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
ctx = context.Background()
|
ctx = context.Background()
|
||||||
|
|||||||
@@ -185,3 +185,20 @@ const (
|
|||||||
GB
|
GB
|
||||||
TB
|
TB
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// IsSystemFile checks if a filename is a common system file that should be ignored
|
||||||
|
// Returns true for files like .DS_Store, desktop.ini, Thumbs.db, and Apple Double files (._*)
|
||||||
|
func IsSystemFile(filename string) bool {
|
||||||
|
// Common system files
|
||||||
|
switch filename {
|
||||||
|
case ".DS_Store", "desktop.ini", "Thumbs.db":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apple Double files (._*)
|
||||||
|
if strings.HasPrefix(filename, "._") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
42
pkg/utils/file_test.go
Normal file
42
pkg/utils/file_test.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsSystemFile(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
filename string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
// System files that should be filtered
|
||||||
|
{".DS_Store", true},
|
||||||
|
{"desktop.ini", true},
|
||||||
|
{"Thumbs.db", true},
|
||||||
|
{"._test.txt", true},
|
||||||
|
{"._", true},
|
||||||
|
{"._somefile", true},
|
||||||
|
{"._folder_name", true},
|
||||||
|
|
||||||
|
// Regular files that should not be filtered
|
||||||
|
{"test.txt", false},
|
||||||
|
{"file.pdf", false},
|
||||||
|
{"document.docx", false},
|
||||||
|
{".gitignore", false},
|
||||||
|
{".env", false},
|
||||||
|
{"_underscore.txt", false},
|
||||||
|
{"normal_file.txt", false},
|
||||||
|
{"", false},
|
||||||
|
{".hidden", false},
|
||||||
|
{"..special", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.filename, func(t *testing.T) {
|
||||||
|
result := IsSystemFile(tc.filename)
|
||||||
|
if result != tc.expected {
|
||||||
|
t.Errorf("IsSystemFile(%q) = %v, want %v", tc.filename, result, tc.expected)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,7 +15,9 @@ import (
|
|||||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||||
"github.com/OpenListTeam/OpenList/v4/server/common"
|
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||||
ftpserver "github.com/fclairamb/ftpserverlib"
|
ftpserver "github.com/fclairamb/ftpserverlib"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
@@ -49,6 +51,11 @@ func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// Check if system file should be ignored
|
||||||
|
_, name := stdpath.Split(path)
|
||||||
|
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
|
||||||
|
return nil, errs.IgnoredSystemFile
|
||||||
|
}
|
||||||
tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*")
|
tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -150,6 +157,11 @@ func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length i
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// Check if system file should be ignored
|
||||||
|
_, name := stdpath.Split(path)
|
||||||
|
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
|
||||||
|
return nil, errs.IgnoredSystemFile
|
||||||
|
}
|
||||||
if trunc {
|
if trunc {
|
||||||
_ = fs.Remove(ctx, path)
|
_ = fs.Remove(ctx, path)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,10 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/task"
|
"github.com/OpenListTeam/OpenList/v4/internal/task"
|
||||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||||
@@ -28,6 +30,14 @@ func getLastModified(c *gin.Context) time.Time {
|
|||||||
return lastModified
|
return lastModified
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shouldIgnoreSystemFile checks if the filename should be ignored based on settings
|
||||||
|
func shouldIgnoreSystemFile(filename string) bool {
|
||||||
|
if setting.GetBool(conf.IgnoreSystemFiles) {
|
||||||
|
return utils.IsSystemFile(filename)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func FsStream(c *gin.Context) {
|
func FsStream(c *gin.Context) {
|
||||||
defer func() {
|
defer func() {
|
||||||
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
|
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
|
||||||
@@ -56,6 +66,11 @@ func FsStream(c *gin.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
dir, name := stdpath.Split(path)
|
dir, name := stdpath.Split(path)
|
||||||
|
// Check if system file should be ignored
|
||||||
|
if shouldIgnoreSystemFile(name) {
|
||||||
|
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
|
||||||
|
return
|
||||||
|
}
|
||||||
// 如果请求头 Content-Length 和 X-File-Size 都没有,则 size=-1,表示未知大小的流式上传
|
// 如果请求头 Content-Length 和 X-File-Size 都没有,则 size=-1,表示未知大小的流式上传
|
||||||
size := c.Request.ContentLength
|
size := c.Request.ContentLength
|
||||||
if size < 0 {
|
if size < 0 {
|
||||||
@@ -160,6 +175,11 @@ func FsForm(c *gin.Context) {
|
|||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
dir, name := stdpath.Split(path)
|
dir, name := stdpath.Split(path)
|
||||||
|
// Check if system file should be ignored
|
||||||
|
if shouldIgnoreSystemFile(name) {
|
||||||
|
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
|
||||||
|
return
|
||||||
|
}
|
||||||
h := make(map[*utils.HashType]string)
|
h := make(map[*utils.HashType]string)
|
||||||
if md5 := c.GetHeader("X-File-Md5"); md5 != "" {
|
if md5 := c.GetHeader("X-File-Md5"); md5 != "" {
|
||||||
h[utils.MD5] = md5
|
h[utils.MD5] = md5
|
||||||
|
|||||||
47
server/handles/scan.go
Normal file
47
server/handles/scan.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package handles
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ManualScanReq struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Limit float64 `json:"limit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func StartManualScan(c *gin.Context) {
|
||||||
|
var req ManualScanReq
|
||||||
|
if err := c.ShouldBind(&req); err != nil {
|
||||||
|
common.ErrorResp(c, err, 400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := op.BeginManualScan(req.Path, req.Limit); err != nil {
|
||||||
|
common.ErrorResp(c, err, 400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
common.SuccessResp(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func StopManualScan(c *gin.Context) {
|
||||||
|
if !op.ManualScanRunning() {
|
||||||
|
common.ErrorStrResp(c, "manual scan is not running", 400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
op.StopManualScan()
|
||||||
|
common.SuccessResp(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ManualScanResp struct {
|
||||||
|
ObjCount uint64 `json:"obj_count"`
|
||||||
|
IsDone bool `json:"is_done"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetManualScanProgress(c *gin.Context) {
|
||||||
|
ret := ManualScanResp{
|
||||||
|
ObjCount: op.ScannedCount.Load(),
|
||||||
|
IsDone: !op.ManualScanRunning(),
|
||||||
|
}
|
||||||
|
common.SuccessResp(c, ret)
|
||||||
|
}
|
||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||||
@@ -24,9 +23,15 @@ type StorageResp struct {
|
|||||||
MountDetails *model.StorageDetails `json:"mount_details,omitempty"`
|
MountDetails *model.StorageDetails `json:"mount_details,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
|
type detailWithIndex struct {
|
||||||
|
idx int
|
||||||
|
val *model.StorageDetails
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeStorageResp(ctx *gin.Context, storages []model.Storage) []*StorageResp {
|
||||||
ret := make([]*StorageResp, len(storages))
|
ret := make([]*StorageResp, len(storages))
|
||||||
var wg sync.WaitGroup
|
detailsChan := make(chan detailWithIndex, len(storages))
|
||||||
|
workerCount := 0
|
||||||
for i, s := range storages {
|
for i, s := range storages {
|
||||||
ret[i] = &StorageResp{
|
ret[i] = &StorageResp{
|
||||||
Storage: s,
|
Storage: s,
|
||||||
@@ -43,22 +48,26 @@ func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
|
|||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
wg.Add(1)
|
workerCount++
|
||||||
go func() {
|
go func(dri driver.Driver, idx int) {
|
||||||
defer wg.Done()
|
details, e := op.GetStorageDetails(ctx, dri)
|
||||||
ctx, cancel := context.WithTimeout(c, time.Second*3)
|
if e != nil {
|
||||||
defer cancel()
|
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
|
||||||
details, err := op.GetStorageDetails(ctx, d)
|
log.Errorf("failed get %s details: %+v", dri.GetStorage().MountPath, e)
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
|
|
||||||
log.Errorf("failed get %s details: %+v", s.MountPath, err)
|
|
||||||
}
|
}
|
||||||
return
|
|
||||||
}
|
}
|
||||||
ret[i].MountDetails = details
|
detailsChan <- detailWithIndex{idx: idx, val: details}
|
||||||
}()
|
}(d, i)
|
||||||
|
}
|
||||||
|
for workerCount > 0 {
|
||||||
|
select {
|
||||||
|
case r := <-detailsChan:
|
||||||
|
ret[r.idx].MountDetails = r.val
|
||||||
|
workerCount--
|
||||||
|
case <-time.After(time.Second * 3):
|
||||||
|
workerCount = 0
|
||||||
|
}
|
||||||
}
|
}
|
||||||
wg.Wait()
|
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func SearchIndex(c *gin.Context) {
|
func SearchIndex(c *gin.Context) {
|
||||||
mode := setting.GetStr(conf.SearchIndex)
|
mode := setting.GetStr(conf.SearchIndex)
|
||||||
if mode == "none" {
|
if mode == "none" {
|
||||||
common.ErrorResp(c, errs.SearchNotAvailable, 500)
|
common.ErrorResp(c, errs.SearchNotAvailable, 404)
|
||||||
c.Abort()
|
c.Abort()
|
||||||
} else {
|
} else {
|
||||||
c.Next()
|
c.Next()
|
||||||
|
|||||||
@@ -179,6 +179,11 @@ func admin(g *gin.RouterGroup) {
|
|||||||
index.POST("/stop", middlewares.SearchIndex, handles.StopIndex)
|
index.POST("/stop", middlewares.SearchIndex, handles.StopIndex)
|
||||||
index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex)
|
index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex)
|
||||||
index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
|
index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
|
||||||
|
|
||||||
|
scan := g.Group("/scan")
|
||||||
|
scan.POST("/start", handles.StartManualScan)
|
||||||
|
scan.POST("/stop", handles.StopManualScan)
|
||||||
|
scan.GET("/progress", handles.GetManualScanProgress)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fsAndShare(g *gin.RouterGroup) {
|
func fsAndShare(g *gin.RouterGroup) {
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||||
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
||||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||||
@@ -286,6 +287,10 @@ func (b *s3Backend) PutObject(
|
|||||||
Modified: ti,
|
Modified: ti,
|
||||||
Ctime: time.Now(),
|
Ctime: time.Now(),
|
||||||
}
|
}
|
||||||
|
// Check if system file should be ignored
|
||||||
|
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
|
||||||
|
return result, errs.IgnoredSystemFile
|
||||||
|
}
|
||||||
stream := &stream.FileStream{
|
stream := &stream.FileStream{
|
||||||
Obj: &obj,
|
Obj: &obj,
|
||||||
Reader: input,
|
Reader: input,
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import (
|
|||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/net"
|
"github.com/OpenListTeam/OpenList/v4/internal/net"
|
||||||
|
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||||
@@ -358,6 +359,10 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
|
|||||||
Modified: h.getModTime(r),
|
Modified: h.getModTime(r),
|
||||||
Ctime: h.getCreateTime(r),
|
Ctime: h.getCreateTime(r),
|
||||||
}
|
}
|
||||||
|
// Check if system file should be ignored
|
||||||
|
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
|
||||||
|
return http.StatusForbidden, errs.IgnoredSystemFile
|
||||||
|
}
|
||||||
fsStream := &stream.FileStream{
|
fsStream := &stream.FileStream{
|
||||||
Obj: &obj,
|
Obj: &obj,
|
||||||
Reader: r.Body,
|
Reader: r.Body,
|
||||||
|
|||||||
Reference in New Issue
Block a user