Compare commits

...

5 Commits

Author SHA1 Message Date
renovate[bot]
35d3266ed3 fix(deps): update module google.golang.org/grpc to v1.77.0 2025-11-18 03:09:43 +00:00
MoYan
c0d480366d fix(driver/123): initialize Platform field (#1644)
* fix(driver/123): initialization the Platform field

Signed-off-by: MoYan <1561515308@qq.com>

* Fix formatting of Platform field in Pan123

Signed-off-by: MoYan <1561515308@qq.com>

---------

Signed-off-by: MoYan <1561515308@qq.com>
2025-11-14 18:49:44 +08:00
Copilot
9de7561154 feat(upload): add optional system file filtering for uploads (#1634) 2025-11-14 14:45:39 +08:00
MadDogOwner
0866b9075f fix(link): correct link cache mode bitwise comparison (#1635)
* fix(link): correct link cache mode bitwise comparison

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

* refactor(link): use explicit flag equality for link cache mode bitmask checks

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

---------

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>
2025-11-13 13:52:33 +08:00
KirCute
055696f576 feat(s3): support frontend direct upload (#1631)
* feat(s3): support frontend direct upload

* feat(s3): support custom direct upload host

* fix: apply suggestions of Copilot
2025-11-13 13:22:17 +08:00
20 changed files with 248 additions and 66 deletions

View File

@@ -28,6 +28,7 @@ func init() {
return &Pan123{
Addition: Addition{
UploadThread: 3,
Platform: "web",
},
}
})

View File

@@ -5,7 +5,6 @@ import (
"errors"
stdpath "path"
"strings"
"sync"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
@@ -17,9 +16,15 @@ import (
log "github.com/sirupsen/logrus"
)
type detailWithIndex struct {
idx int
val *model.StorageDetails
}
func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj {
var objs []model.Obj
var wg sync.WaitGroup
detailsChan := make(chan detailWithIndex, len(d.pathMap))
workerCount := 0
for _, k := range d.rootOrder {
obj := model.Object{
Name: k,
@@ -47,22 +52,26 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model
DriverName: remoteDriver.Config().Name,
},
}
wg.Add(1)
go func() {
defer wg.Done()
c, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
details, e := op.GetStorageDetails(c, remoteDriver, refresh)
workerCount++
go func(dri driver.Driver, i int) {
details, e := op.GetStorageDetails(ctx, dri, refresh)
if e != nil {
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
log.Errorf("failed get %s storage details: %+v", remoteDriver.GetStorage().MountPath, e)
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e)
}
return
}
objs[idx].(*model.ObjStorageDetails).StorageDetails = details
}()
detailsChan <- detailWithIndex{idx: i, val: details}
}(remoteDriver, idx)
}
for workerCount > 0 {
select {
case r := <-detailsChan:
objs[r.idx].(*model.ObjStorageDetails).StorageDetails = r.val
workerCount--
case <-time.After(time.Second):
workerCount = 0
}
}
wg.Wait()
return objs
}

View File

@@ -7,19 +7,19 @@ import (
type Addition struct {
driver.RootPath
Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"`
IsSharepoint bool `json:"is_sharepoint"`
UseOnlineAPI bool `json:"use_online_api" default:"true"`
APIAddress string `json:"api_url_address" default:"https://api.oplist.org/onedrive/renewapi"`
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
RedirectUri string `json:"redirect_uri" required:"true" default:"https://api.oplist.org/onedrive/callback"`
RefreshToken string `json:"refresh_token" required:"true"`
SiteId string `json:"site_id"`
ChunkSize int64 `json:"chunk_size" type:"number" default:"5"`
CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"`
DisableDiskUsage bool `json:"disable_disk_usage" default:"false"`
EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"`
Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"`
IsSharepoint bool `json:"is_sharepoint"`
UseOnlineAPI bool `json:"use_online_api" default:"true"`
APIAddress string `json:"api_url_address" default:"https://api.oplist.org/onedrive/renewapi"`
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
RedirectUri string `json:"redirect_uri" required:"true" default:"https://api.oplist.org/onedrive/callback"`
RefreshToken string `json:"refresh_token" required:"true"`
SiteId string `json:"site_id"`
ChunkSize int64 `json:"chunk_size" type:"number" default:"5"`
CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"`
DisableDiskUsage bool `json:"disable_disk_usage" default:"false"`
EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"`
}
var config = driver.Config{

View File

@@ -217,11 +217,10 @@ func (d *QuarkOrUC) GetDetails(ctx context.Context) (*model.StorageDetails, erro
if err != nil {
return nil, err
}
used := memberInfo.Data.UseCapacity
total := memberInfo.Data.TotalCapacity
return &model.StorageDetails{
DiskUsage: model.DiskUsage{
TotalSpace: memberInfo.Data.TotalCapacity,
FreeSpace: memberInfo.Data.TotalCapacity - memberInfo.Data.UseCapacity,
},
DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
}, nil
}

View File

@@ -10,6 +10,7 @@ import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/cron"
@@ -24,9 +25,10 @@ import (
type S3 struct {
model.Storage
Addition
Session *session.Session
client *s3.S3
linkClient *s3.S3
Session *session.Session
client *s3.S3
linkClient *s3.S3
directUploadClient *s3.S3
config driver.Config
cron *cron.Cron
@@ -52,16 +54,18 @@ func (d *S3) Init(ctx context.Context) error {
if err != nil {
log.Errorln("Doge init session error:", err)
}
d.client = d.getClient(false)
d.linkClient = d.getClient(true)
d.client = d.getClient(ClientTypeNormal)
d.linkClient = d.getClient(ClientTypeLink)
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
})
}
err := d.initSession()
if err != nil {
return err
}
d.client = d.getClient(false)
d.linkClient = d.getClient(true)
d.client = d.getClient(ClientTypeNormal)
d.linkClient = d.getClient(ClientTypeLink)
d.directUploadClient = d.getClient(ClientTypeDirectUpload)
return nil
}
@@ -210,4 +214,33 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up
return err
}
func (d *S3) GetDirectUploadTools() []string {
if !d.EnableDirectUpload {
return nil
}
return []string{"HttpDirect"}
}
func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) {
if !d.EnableDirectUpload {
return nil, errs.NotImplement
}
path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false)
req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{
Bucket: &d.Bucket,
Key: &path,
})
if req == nil {
return nil, fmt.Errorf("failed to create PutObject request")
}
link, err := req.Presign(time.Hour * time.Duration(d.SignURLExpire))
if err != nil {
return nil, err
}
return &model.HttpDirectUploadInfo{
UploadURL: link,
Method: "PUT",
}, nil
}
var _ driver.Driver = (*S3)(nil)

View File

@@ -21,6 +21,8 @@ type Addition struct {
ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."`
EnableDirectUpload bool `json:"enable_direct_upload" default:"false"`
DirectUploadHost string `json:"direct_upload_host" required:"false"`
}
func init() {

View File

@@ -41,9 +41,15 @@ func (d *S3) initSession() error {
return err
}
func (d *S3) getClient(link bool) *s3.S3 {
const (
ClientTypeNormal = iota
ClientTypeLink
ClientTypeDirectUpload
)
func (d *S3) getClient(clientType int) *s3.S3 {
client := s3.New(d.Session)
if link && d.CustomHost != "" {
if clientType == ClientTypeLink && d.CustomHost != "" {
client.Handlers.Build.PushBack(func(r *request.Request) {
if r.HTTPRequest.Method != http.MethodGet {
return
@@ -58,6 +64,20 @@ func (d *S3) getClient(link bool) *s3.S3 {
}
})
}
if clientType == ClientTypeDirectUpload && d.DirectUploadHost != "" {
client.Handlers.Build.PushBack(func(r *request.Request) {
if r.HTTPRequest.Method != http.MethodPut {
return
}
split := strings.SplitN(d.DirectUploadHost, "://", 2)
if utils.SliceContains([]string{"http", "https"}, split[0]) {
r.HTTPRequest.URL.Scheme = split[0]
r.HTTPRequest.URL.Host = split[1]
} else {
r.HTTPRequest.URL.Host = d.DirectUploadHost
}
})
}
return client
}

2
go.mod
View File

@@ -288,7 +288,7 @@ require (
golang.org/x/text v0.27.0
golang.org/x/tools v0.35.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/grpc v1.73.0
google.golang.org/grpc v1.77.0
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect

View File

@@ -177,6 +177,7 @@ func InitialSettings() []model.SettingItem {
{Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
{Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC},
{Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`},
// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},

View File

@@ -56,6 +56,7 @@ const (
ShareArchivePreview = "share_archive_preview"
ShareForceProxy = "share_force_proxy"
ShareSummaryContent = "share_summary_content"
IgnoreSystemFiles = "ignore_system_files"
// index
SearchIndex = "search_index"

View File

@@ -11,6 +11,7 @@ var (
ObjectAlreadyExists = errors.New("object already exists")
NotFolder = errors.New("not a folder")
NotFile = errors.New("not a file")
IgnoredSystemFile = errors.New("system file upload ignored")
)
func IsObjectNotFound(err error) bool {

View File

@@ -173,10 +173,10 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li
mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path)
}
typeKey := args.Type
if mode&driver.LinkCacheIP == 1 {
if mode&driver.LinkCacheIP == driver.LinkCacheIP {
typeKey += "/" + args.IP
}
if mode&driver.LinkCacheUA == 1 {
if mode&driver.LinkCacheUA == driver.LinkCacheUA {
typeKey += "/" + args.Header.Get("User-Agent")
}
key := Key(storage, path)

View File

@@ -358,16 +358,21 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string,
DriverName: d.Config().Name,
},
}
timeoutCtx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
details, err := GetStorageDetails(timeoutCtx, d, refresh)
if err != nil {
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
log.Errorf("failed get %s storage details: %+v", d.GetStorage().MountPath, err)
resultChan := make(chan *model.StorageDetails, 1)
go func(dri driver.Driver) {
details, err := GetStorageDetails(ctx, dri, refresh)
if err != nil {
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err)
}
}
return ret
resultChan <- details
}(d)
select {
case r := <-resultChan:
ret.StorageDetails = r
case <-time.After(time.Second):
}
ret.StorageDetails = details
return ret
})
}

View File

@@ -185,3 +185,20 @@ const (
GB
TB
)
// IsSystemFile checks if a filename is a common system file that should be ignored
// Returns true for files like .DS_Store, desktop.ini, Thumbs.db, and Apple Double files (._*)
func IsSystemFile(filename string) bool {
// Common system files
switch filename {
case ".DS_Store", "desktop.ini", "Thumbs.db":
return true
}
// Apple Double files (._*)
if strings.HasPrefix(filename, "._") {
return true
}
return false
}

42
pkg/utils/file_test.go Normal file
View File

@@ -0,0 +1,42 @@
package utils
import (
"testing"
)
func TestIsSystemFile(t *testing.T) {
testCases := []struct {
filename string
expected bool
}{
// System files that should be filtered
{".DS_Store", true},
{"desktop.ini", true},
{"Thumbs.db", true},
{"._test.txt", true},
{"._", true},
{"._somefile", true},
{"._folder_name", true},
// Regular files that should not be filtered
{"test.txt", false},
{"file.pdf", false},
{"document.docx", false},
{".gitignore", false},
{".env", false},
{"_underscore.txt", false},
{"normal_file.txt", false},
{"", false},
{".hidden", false},
{"..special", false},
}
for _, tc := range testCases {
t.Run(tc.filename, func(t *testing.T) {
result := IsSystemFile(tc.filename)
if result != tc.expected {
t.Errorf("IsSystemFile(%q) = %v, want %v", tc.filename, result, tc.expected)
}
})
}
}

View File

@@ -15,7 +15,9 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server/common"
ftpserver "github.com/fclairamb/ftpserverlib"
"github.com/pkg/errors"
@@ -49,6 +51,11 @@ func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy,
if err != nil {
return nil, err
}
// Check if system file should be ignored
_, name := stdpath.Split(path)
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
return nil, errs.IgnoredSystemFile
}
tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*")
if err != nil {
return nil, err
@@ -150,6 +157,11 @@ func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length i
if err != nil {
return nil, err
}
// Check if system file should be ignored
_, name := stdpath.Split(path)
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) {
return nil, errs.IgnoredSystemFile
}
if trunc {
_ = fs.Remove(ctx, path)
}

View File

@@ -8,8 +8,10 @@ import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/internal/task"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@@ -28,6 +30,14 @@ func getLastModified(c *gin.Context) time.Time {
return lastModified
}
// shouldIgnoreSystemFile checks if the filename should be ignored based on settings
func shouldIgnoreSystemFile(filename string) bool {
if setting.GetBool(conf.IgnoreSystemFiles) {
return utils.IsSystemFile(filename)
}
return false
}
func FsStream(c *gin.Context) {
defer func() {
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
@@ -56,6 +66,11 @@ func FsStream(c *gin.Context) {
}
}
dir, name := stdpath.Split(path)
// Check if system file should be ignored
if shouldIgnoreSystemFile(name) {
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
return
}
// 如果请求头 Content-Length 和 X-File-Size 都没有,则 size=-1表示未知大小的流式上传
size := c.Request.ContentLength
if size < 0 {
@@ -160,6 +175,11 @@ func FsForm(c *gin.Context) {
}
defer f.Close()
dir, name := stdpath.Split(path)
// Check if system file should be ignored
if shouldIgnoreSystemFile(name) {
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
return
}
h := make(map[*utils.HashType]string)
if md5 := c.GetHeader("X-File-Md5"); md5 != "" {
h[utils.MD5] = md5

View File

@@ -4,7 +4,6 @@ import (
"context"
"errors"
"strconv"
"sync"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
@@ -24,9 +23,15 @@ type StorageResp struct {
MountDetails *model.StorageDetails `json:"mount_details,omitempty"`
}
func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
type detailWithIndex struct {
idx int
val *model.StorageDetails
}
func makeStorageResp(ctx *gin.Context, storages []model.Storage) []*StorageResp {
ret := make([]*StorageResp, len(storages))
var wg sync.WaitGroup
detailsChan := make(chan detailWithIndex, len(storages))
workerCount := 0
for i, s := range storages {
ret[i] = &StorageResp{
Storage: s,
@@ -43,22 +48,26 @@ func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp {
if !ok {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
ctx, cancel := context.WithTimeout(c, time.Second*3)
defer cancel()
details, err := op.GetStorageDetails(ctx, d)
if err != nil {
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
log.Errorf("failed get %s details: %+v", s.MountPath, err)
workerCount++
go func(dri driver.Driver, idx int) {
details, e := op.GetStorageDetails(ctx, dri)
if e != nil {
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
log.Errorf("failed get %s details: %+v", dri.GetStorage().MountPath, e)
}
return
}
ret[i].MountDetails = details
}()
detailsChan <- detailWithIndex{idx: idx, val: details}
}(d, i)
}
for workerCount > 0 {
select {
case r := <-detailsChan:
ret[r.idx].MountDetails = r.val
workerCount--
case <-time.After(time.Second * 3):
workerCount = 0
}
}
wg.Wait()
return ret
}

View File

@@ -19,6 +19,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@@ -286,6 +287,10 @@ func (b *s3Backend) PutObject(
Modified: ti,
Ctime: time.Now(),
}
// Check if system file should be ignored
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
return result, errs.IgnoredSystemFile
}
stream := &stream.FileStream{
Obj: &obj,
Reader: input,

View File

@@ -20,6 +20,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/net"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
@@ -358,6 +359,10 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
Modified: h.getModTime(r),
Ctime: h.getCreateTime(r),
}
// Check if system file should be ignored
if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
return http.StatusForbidden, errs.IgnoredSystemFile
}
fsStream := &stream.FileStream{
Obj: &obj,
Reader: r.Body,