Compare commits

...

21 Commits

Author SHA1 Message Date
jenfonro
2844797684 fix(baidu_netdisk): support resuming uploads when an error occurs (#1279)
support resuming uploads when an error occurs
2025-11-04 13:33:33 +08:00
Seven
9f4e439478 chore(strm): Built-in file types support modification (#1483) 2025-11-04 10:33:16 +08:00
jenfonro
9d09ee133d fix(google_driver): fix google link file display size (#1335)
* fix file link display size

* fix performance and field

* cn to en notes

---------

Co-authored-by: ShenLin <773933146@qq.com>
2025-11-04 09:26:20 +08:00
jenfonro
d88f0e8f3c feat(net): support proxy configuration via config file (#1359)
* support proxy

* debug

* debug2

* del debug

* add proxy configuration with env var fallback

* comments to en

* refactor(env): fallback env

---------

Co-authored-by: jyxjjj <773933146@qq.com>
2025-11-04 09:01:35 +08:00
ex-hentai
0857478516 feat(thunder): allow setting space (#1219)
allows access to files on remote devices via Thunder's tunneling service.
2025-11-03 10:53:38 +08:00
Seven
66d9809057 feat(strm): strm local file (#1127)
* feat(strm): strm local file

* feat: 代码优化

* feat: 访问被strm挂载路径时也更新

* fix: 路径最后带/判断缺失

* fix: 路径最后带/判断缺失

* refactor

* refactor

* fix: close seekable-stream in `generateStrm`

* refactor: lazy create local file

* 优化路径判断

---------

Co-authored-by: KirCute <kircute@foxmail.com>
2025-11-03 10:48:15 +08:00
MoYan
db8a7e8caf feat(123): allow modification of the platform header (#1542)
* feat(drivers/123): Allow modification of the platform field

* feat(drivers/123): Set login platfrom as web

* fix(drivers/123): update platform field help value
2025-11-03 10:02:52 +08:00
walloo
8f18e34da0 fix(alias): nil panic in ResolveLinkCacheMode (#1527)
* fix(alias): Check the driver path during initialization

* fix(alias): Don't check the driver path during initialization anymore.
2025-10-23 22:18:56 +08:00
ShenLin
525f26dc23 feat(command): add --config flag to set custom config path (#1479) 2025-10-22 19:35:34 +08:00
NewbieOrange
a0fcfa3ed2 fix(aliyundrive_open): use safe disk usage calculation (#1510) 2025-10-20 22:05:14 +08:00
ILoveScratch
15f276537c fix(share): remove share when user delete (#1493) 2025-10-19 22:48:21 +08:00
MadDogOwner
623a12050e feat(openlist): add PassIPToUpsteam to driver (#1498) 2025-10-19 22:45:40 +08:00
D@' 3z K!7
ae2d2d1021 feat(drivers): add ProtonDrive driver (#1368)
* feat(drivers): add ProtonDrive driver

- Implement complete ProtonDrive storage driver with end-to-end encryption support
- Add authentication via username/password with credential caching and reusable login
- Support all core operations: List, Link, Put, Copy, Move, Remove, Rename, MakeDir
- Include encrypted file operations with PGP key management and node passphrase handling
- Add temporary HTTP server for secure file downloads with range request support
- Support media streaming using temp server range requests
- Implement progress tracking for uploads and downloads
- Support directory operations with circular move detection
- Add proper error handling and panic recovery for external library integration
- Support buffered upload for specific sequential and encrypted, but optimized transmission.

* Update drivers/proton_drive/util.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: D@' 3z K!7 <99719341+Da3zKi7@users.noreply.github.com>

* chore

* feat(drivers): enhance ProtonDrive temp server

- Implement separate listen and public port configuration for complex network deployments
- Add intelligent port detection with 8080 as preferred default, fallback to auto-assignment
- Support Container/NAT/VM environments through configurable external host and port mapping
- Add port availability validation with graceful fallback to listen port
- Enable users to specify external domain/IP for client connections (e.g., 192.168.1.5)
- Follow FTP server configuration patterns for network flexibility
- Maintain localhost development simplicity while supporting production deployments

* feat(proton_drive): refactor directory handling and improve link retrieval

* fix(proton_drive): add NoLinkURL configuration option

* fix(proton_drive): update file size retrieval and enforce TwoFACode requirement

* feat(proton_drive): add expiration to link response

* fix(proton_drive): handle empty RootFolderID in Init method

* fix(proton_drive): update credential handling to use email and reusable login

* fix(proton_drive): update credential handling to use reusableCredential variable

* fix(proton_drive): update DirectRename to use GetLink for source object retrieval

* fix(proton_drive): refactor uploadFile to return model.Obj and handle errors correctly

* fix(proton_drive): refactor DirectMove to use getLink for source retrieval and simplify destination handling

* fix(proton_drive): simplify Copy method by removing temporary file creation and directly using FileStream

* refactor(proton_drive): remove unused temporary server and related code

* chore

* fix(proton_drive): fix driver

- Handle fresh login if ProtonDrive rejects AccessToken or RefreshToken
- Update stored credentials

* fix(proton_drive): simplify reusable login handling in Init method

* fix(proton_drive): fix driver

- Update stored credentials, now is failing

* feat(proton_drive): improve authentication handling and remove unused variables

* fix(proton_drive): fix driver

- Update stored credentials, now is failing

* fix(proton_drive): improve authentication handling

* refactor(proton_drive): move client initialization to initClient method

* feat(proton_drive): move addrs and addrKRs

* feat(proton_drive): optimize upload threads

- Change ConcurrentBlockUploadCount to user configured upload threads number
- Comment ConcurrentFileCryptoCount, default is runtime.GOMAXPROCS(0)

---------

Signed-off-by: D@' 3z K!7 <99719341+Da3zKi7@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: KirCute <951206789@qq.com>
Co-authored-by: j2rong4cn <j2rong@qq.com>
Co-authored-by: KirCute <kircute@foxmail.com>
2025-10-19 12:07:11 +08:00
jerrita
a109152a13 feat(onedrive): add ref support (#1435)
* feat(onedrive): add ref support

* fix(onedrive): remove redundant token assignment from reference

---------

Co-authored-by: j2rong4cn <j2rong@qq.com>
2025-10-18 21:48:26 +08:00
ILoveScratch
febbcd6027 feat(cache): improve cache management (#1339)
* feat(cache): improve cache management

* feat(disk-usage): add cache

* feat(disk-usage): add refresh

* fix(disk-usage): cache with ttl

* feat(cache): implement KeyedCache and TypedCache for improved caching mechanism

* fix(copy): update object retrieval to use Get instead of GetUnwrap

* refactor(cache): simplify DirectoryCache structure and improve object management

* fix(cache): correct cache entry initialization and key deletion logic in TypedCache

* refactor(driver): remove GetObjInfo interface and simplify Link function logic
https://github.com/OpenListTeam/OpenList/pull/888/files#r2430925783

* fix(link): optimize link retrieval and caching logic

* refactor(cache): consolidate cache management and improve directory cache handling

* fix(cache): add cache control based on storage configuration in List function

* .

* refactor: replace fmt.Sprintf with strconv for integer conversions

* refactor(cache): enhance cache entry management with Expirable interface

* fix(cache): improve link reference acquisition logic to handle expiration

* refactor: replace OnlyLinkMFile with NoLinkSF in driver configurations and logic

* refactor(link): enhance link caching logic with dynamic type keys based on IP and User-Agent

* feat(drivers): add LinkCacheType to driver configurations for enhanced caching

* refactor(cache): streamline directory object management in cache operations

* refactor(cache): remove unnecessary 'dirty' field from CacheEntry structure

* refactor(cache): replace 'dirty' field with bitwise flags

* refactor(io): 调高SyncClosers.AcquireReference的优先级

* refactor(link): 优化链接获取逻辑,增加重

* refactor(link): 添加RequireReference字段以增强链接管理

* refactor(link): 移除MFile字段,改用RangeReader

* refactor: 移除不必要的NoLinkSF字段

* refactor(cache): 修改目录缓存的脏标志定义和更新逻辑

* feat(cache): add expiration gc

---------

Co-authored-by: KirCute <951206789@qq.com>
Co-authored-by: KirCute <kircute@foxmail.com>
Co-authored-by: j2rong4cn <j2rong@qq.com>
2025-10-18 21:47:18 +08:00
jenfonro
549e60136b fix(fs):fix retry task after restart (#1467)
* fix retry task after restart

* fix: initialize SrcStorage and DstStorage in tasks to prevent nil pointer dereference

* feat: implement storage load signal mechanism for improved synchronization

* fix:  update StoragesLoaded logic

* refactor: reorganize storage loading logic and improve synchronization handling

---------

Co-authored-by: j2rong4cn <j2rong@qq.com>
2025-10-18 20:52:02 +08:00
Tursom K. Ulefits
14d2b8290a feat(local): move deleted files to corresponding locations (#1281) 2025-10-17 18:01:02 +08:00
ShenLin
cdc069d8e7 feat(http3|quic): Set Alt-Svc header only for HTTPS requests (#1469) 2025-10-14 18:07:29 +08:00
zzzhr1990
fb5094f688 feat(drivers): add halalcloud_open driver (#1430)
* 新增清真云Open驱动,支持最新的轻量SDK

* Change Go version in go.mod

Downgrade Go version from 1.24.2 to 1.23.4

Signed-off-by: zzzhr1990 <zzzhr@hotmail.com>

* Apply suggestions from code review

* Removed unnecessary comments
* Downgraded the Go version to 1.23.4.
* Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety.

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: zzzhr1990 <zzzhr@hotmail.com>

* feat(halalcloud_open): support disk usage

* Set useSingleUpload to true for upload safety

Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety.

Signed-off-by: zzzhr1990 <zzzhr@hotmail.com>

* Update meta.go

Change required for RefreshToken, If using a personal API approach, the RefreshToken is not required.

Signed-off-by: zzzhr1990 <zzzhr@hotmail.com>

* remove debug logs

* bump halalcloud SDK version

* fix unnecessary params

* Update drivers/halalcloud_open/driver_init.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: zzzhr1990 <zzzhr@hotmail.com>

* Fixed spelling errors; changed hardcoded retry parameters to constants.

* remove pointer in get link function in utils.go

---------

Signed-off-by: zzzhr1990 <zzzhr@hotmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: KirCute <951206789@qq.com>
2025-10-14 16:03:42 +08:00
KirCute
670e0bdc45 chore(frontend): optimize user operations (#1449) 2025-10-14 16:02:50 +08:00
ShenLin
89235012af feat(http3|quic): add http3|quic support (#1466)
* feat(http3|quic): add http3|quic support

* revert(ai): fix ai error

* fix(shutdown): shutdown was using close

* feat(http3|quic): add config if needs h3

* feat(http3|quic): add Alt-Svc to expose h3
2025-10-14 15:57:16 +08:00
115 changed files with 3837 additions and 716 deletions

View File

@@ -64,8 +64,9 @@ Thank you for your support and understanding of the OpenList project.
- [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage)
- [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV)
- [x] Teambition([China](https://www.teambition.com), [International](https://us.teambition.com))
- [x] [Mediatrack](https://www.mediatrack.cn)
- [x] [MediaFire](https://www.mediafire.com)
- [x] [Mediatrack](https://www.mediatrack.cn)
- [x] [ProtonDrive](https://proton.me/drive)
- [x] [139yun](https://yun.139.com) (Personal, Family, Group)
- [x] [YandexDisk](https://disk.yandex.com)
- [x] [BaiduNetdisk](http://pan.baidu.com)

View File

@@ -64,8 +64,9 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目,遵循 AGPL-3
- [x] [又拍云对象存储](https://www.upyun.com/products/file-storage)
- [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV)
- [x] Teambition([中国](https://www.teambition.com), [国际](https://us.teambition.com))
- [x] [分秒帧](https://www.mediatrack.cn)
- [x] [MediaFire](https://www.mediafire.com)
- [x] [分秒帧](https://www.mediatrack.cn)
- [x] [ProtonDrive](https://proton.me/drive)
- [x] [和彩云](https://yun.139.com)(个人、家庭、群组)
- [x] [YandexDisk](https://disk.yandex.com)
- [x] [百度网盘](http://pan.baidu.com)

View File

@@ -65,6 +65,7 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい
- [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV)
- [x] Teambition([中国](https://www.teambition.com), [国際](https://us.teambition.com))
- [x] [Mediatrack](https://www.mediatrack.cn)
- [x] [ProtonDrive](https://proton.me/drive)
- [x] [139yun](https://yun.139.com)(個人、家族、グループ)
- [x] [YandexDisk](https://disk.yandex.com)
- [x] [BaiduNetdisk](http://pan.baidu.com)

View File

@@ -66,6 +66,7 @@ Dank u voor uw ondersteuning en begrip
- [x] Teambition([China](https://www.teambition.com), [Internationaal](https://us.teambition.com))
- [x] [MediaFire](https://www.mediafire.com)
- [x] [Mediatrack](https://www.mediatrack.cn)
- [x] [ProtonDrive](https://proton.me/drive)
- [x] [139yun](https://yun.139.com) (Persoonlijk, Familie, Groep)
- [x] [YandexDisk](https://disk.yandex.com)
- [x] [BaiduNetdisk](http://pan.baidu.com)

View File

@@ -2,6 +2,7 @@ package flags
var (
DataDir string
ConfigPath string
Debug bool
NoPrefix bool
Dev bool

View File

@@ -27,7 +27,8 @@ func Execute() {
}
func init() {
RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data folder")
RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data directory (relative paths are resolved against the current working directory)")
RootCmd.PersistentFlags().StringVar(&flags.ConfigPath, "config", "", "path to config.json (relative to current working directory; defaults to [data directory]/config.json, where [data directory] is set by --data)")
RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode")
RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix")
RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode")

View File

@@ -27,6 +27,8 @@ import (
"github.com/spf13/cobra"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
"github.com/quic-go/quic-go/http3"
)
// ServerCmd represents the server command
@@ -63,6 +65,7 @@ the address is defined in config file`,
httpHandler = h2c.NewHandler(r, &http2.Server{})
}
var httpSrv, httpsSrv, unixSrv *http.Server
var quicSrv *http3.Server
if conf.Conf.Scheme.HttpPort != -1 {
httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort)
fmt.Printf("start HTTP server @ %s\n", httpBase)
@@ -86,6 +89,24 @@ the address is defined in config file`,
utils.Log.Fatalf("failed to start https: %s", err.Error())
}
}()
if conf.Conf.Scheme.EnableH3 {
fmt.Printf("start HTTP3 (quic) server @ %s\n", httpsBase)
utils.Log.Infof("start HTTP3 (quic) server @ %s", httpsBase)
r.Use(func(c *gin.Context) {
if c.Request.TLS != nil {
port := conf.Conf.Scheme.HttpsPort
c.Header("Alt-Svc", fmt.Sprintf("h3=\":%d\"; ma=86400", port))
}
c.Next()
})
quicSrv = &http3.Server{Addr: httpsBase, Handler: r}
go func() {
err := quicSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile)
if err != nil && !errors.Is(err, http.ErrServerClosed) {
utils.Log.Fatalf("failed to start http3 (quic): %s", err.Error())
}
}()
}
}
if conf.Conf.Scheme.UnixFile != "" {
fmt.Printf("start unix server @ %s\n", conf.Conf.Scheme.UnixFile)
@@ -203,6 +224,15 @@ the address is defined in config file`,
utils.Log.Fatal("HTTPS server shutdown err: ", err)
}
}()
if conf.Conf.Scheme.EnableH3 {
wg.Add(1)
go func() {
defer wg.Done()
if err := quicSrv.Shutdown(ctx); err != nil {
utils.Log.Fatal("HTTP3 (quic) server shutdown err: ", err)
}
}()
}
}
if conf.Conf.Scheme.UnixFile != "" {
wg.Add(1)

View File

@@ -15,10 +15,9 @@ type Addition struct {
}
var config = driver.Config{
Name: "115 Cloud",
DefaultRoot: "0",
// OnlyProxy: true,
// NoOverwriteUpload: true,
Name: "115 Cloud",
DefaultRoot: "0",
LinkCacheMode: driver.LinkCacheUA,
}
func init() {

View File

@@ -131,23 +131,6 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
}, nil
}
func (d *Open115) GetObjInfo(ctx context.Context, path string) (model.Obj, error) {
if err := d.WaitLimit(ctx); err != nil {
return nil, err
}
resp, err := d.client.GetFolderInfoByPath(ctx, path)
if err != nil {
return nil, err
}
return &Obj{
Fid: resp.FileID,
Fn: resp.FileName,
Fc: resp.FileCategory,
Sha1: resp.Sha1,
Pc: resp.PickCode,
}, nil
}
func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
if err := d.WaitLimit(ctx); err != nil {
return nil, err

View File

@@ -17,8 +17,9 @@ type Addition struct {
}
var config = driver.Config{
Name: "115 Open",
DefaultRoot: "0",
Name: "115 Open",
DefaultRoot: "0",
LinkCacheMode: driver.LinkCacheUA,
}
func init() {

View File

@@ -41,7 +41,9 @@ func (d *Pan123) GetAddition() driver.Additional {
}
func (d *Pan123) Init(ctx context.Context) error {
_, err := d.Request(UserInfo, http.MethodGet, nil, nil)
_, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) {
req.SetHeader("platform", "web")
}, nil)
return err
}

View File

@@ -12,7 +12,8 @@ type Addition struct {
//OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"`
//OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
AccessToken string
UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
Platform string `json:"platform" type:"string" default:"web" help:"the platform header value, sent with API requests"`
}
var config = driver.Config{

View File

@@ -203,7 +203,7 @@ do:
"referer": "https://www.123pan.com/",
"authorization": "Bearer " + d.AccessToken,
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) openlist-client",
"platform": "web",
"platform": d.Platform,
"app-version": "3",
//"user-agent": base.UserAgent,
})

View File

@@ -130,7 +130,7 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) {
func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
path := dir.GetPath()
if utils.PathEqual(path, "/") && !d.autoFlatten {
return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough), nil
return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil
}
root, sub := d.getRootAndPath(path)
dsts, ok := d.pathMap[root]
@@ -211,9 +211,6 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
if resultLink.ContentLength == 0 {
resultLink.ContentLength = fi.GetSize()
}
if resultLink.MFile != nil {
return &resultLink, nil
}
if d.DownloadConcurrency > 0 {
resultLink.Concurrency = d.DownloadConcurrency
}
@@ -527,4 +524,25 @@ func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj,
}
}
func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode {
root, sub := d.getRootAndPath(path)
dsts, ok := d.pathMap[root]
if !ok {
return 0
}
for _, dst := range dsts {
storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(dst, sub))
if err != nil {
continue
}
mode := storage.Config().LinkCacheMode
if mode == -1 {
return storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(actualPath)
} else {
return mode
}
}
return 0
}
var _ driver.Driver = (*Alias)(nil)

View File

@@ -26,6 +26,7 @@ var config = driver.Config{
NoUpload: false,
DefaultRoot: "/",
ProxyRangeOption: true,
LinkCacheMode: driver.LinkCacheAuto,
}
func init() {

View File

@@ -17,7 +17,7 @@ import (
log "github.com/sirupsen/logrus"
)
func (d *Alias) listRoot(ctx context.Context, withDetails bool) []model.Obj {
func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj {
var objs []model.Obj
var wg sync.WaitGroup
for _, k := range d.rootOrder {
@@ -52,7 +52,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails bool) []model.Obj {
defer wg.Done()
c, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
details, e := op.GetStorageDetails(c, remoteDriver)
details, e := op.GetStorageDetails(c, remoteDriver, refresh)
if e != nil {
if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) {
log.Errorf("failed get %s storage details: %+v", remoteDriver.GetStorage().MountPath, e)

View File

@@ -299,10 +299,7 @@ func (d *AliyundriveOpen) GetDetails(ctx context.Context) (*model.StorageDetails
total := utils.Json.Get(res, "personal_space_info", "total_size").ToUint64()
used := utils.Json.Get(res, "personal_space_info", "used_size").ToUint64()
return &model.StorageDetails{
DiskUsage: model.DiskUsage{
TotalSpace: total,
FreeSpace: total - used,
},
DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
}, nil
}

View File

@@ -36,6 +36,7 @@ import (
_ "github.com/OpenListTeam/OpenList/v4/drivers/google_drive"
_ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo"
_ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud"
_ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open"
_ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou"
_ "github.com/OpenListTeam/OpenList/v4/drivers/ipfs_api"
_ "github.com/OpenListTeam/OpenList/v4/drivers/kodbox"
@@ -55,6 +56,7 @@ import (
_ "github.com/OpenListTeam/OpenList/v4/drivers/openlist_share"
_ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak"
_ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak_share"
_ "github.com/OpenListTeam/OpenList/v4/drivers/proton_drive"
_ "github.com/OpenListTeam/OpenList/v4/drivers/quark_open"
_ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc"
_ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc_tv"

View File

@@ -5,11 +5,13 @@ import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"net/url"
"os"
stdpath "path"
"strconv"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
@@ -31,6 +33,8 @@ type BaiduNetdisk struct {
vipType int // 会员类型0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M)
}
var ErrUploadIDExpired = errors.New("uploadid expired")
func (d *BaiduNetdisk) Config() driver.Config {
return config
}
@@ -214,7 +218,6 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
// cal md5 for first 256k data
const SliceSize int64 = 256 * utils.KB
// cal md5
blockList := make([]string, 0, count)
byteSize := sliceSize
fileMd5H := md5.New()
@@ -244,7 +247,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
}
if tmpF != nil {
if written != streamSize {
return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize)
return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize)
}
_, err = tmpF.Seek(0, io.SeekStart)
if err != nil {
@@ -258,31 +261,14 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
mtime := stream.ModTime().Unix()
ctime := stream.CreateTime().Unix()
// step.1 预上传
// 尝试获取之前的进度
// step.1 尝试读取已保存进度
precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5)
if !ok {
params := map[string]string{
"method": "precreate",
}
form := map[string]string{
"path": path,
"size": strconv.FormatInt(streamSize, 10),
"isdir": "0",
"autoinit": "1",
"rtype": "3",
"block_list": blockListStr,
"content-md5": contentMd5,
"slice-md5": sliceMd5,
}
joinTime(form, ctime, mtime)
log.Debugf("[baidu_netdisk] precreate data: %s", form)
_, err = d.postForm("/xpan/file", params, form, &precreateResp)
// 没有进度,走预上传
precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime)
if err != nil {
return nil, err
}
log.Debugf("%+v", precreateResp)
if precreateResp.ReturnType == 2 {
// rapid upload, since got md5 match from baidu server
// 修复时间,具体原因见 Put 方法注释的 **注意**
@@ -291,45 +277,81 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
return fileToObj(precreateResp.File), nil
}
}
// step.2 上传分片
threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread,
retry.Attempts(1),
retry.Delay(time.Second),
retry.DelayType(retry.BackOffDelay))
uploadLoop:
for attempt := 0; attempt < 2; attempt++ {
threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread,
retry.Attempts(1),
retry.Delay(time.Second),
retry.DelayType(retry.BackOffDelay))
for i, partseq := range precreateResp.BlockList {
if utils.IsCanceled(upCtx) {
break
cacheReaderAt, okReaderAt := cache.(io.ReaderAt)
if !okReaderAt {
return nil, fmt.Errorf("cache object must implement io.ReaderAt interface for upload operations")
}
i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize
if partseq+1 == count {
byteSize = lastBlockSize
totalParts := len(precreateResp.BlockList)
doneParts := 0 // 新增计数器
for i, partseq := range precreateResp.BlockList {
if utils.IsCanceled(upCtx) || partseq < 0 {
continue
}
i, partseq := i, partseq
offset, size := int64(partseq)*sliceSize, sliceSize
if partseq+1 == count {
size = lastBlockSize
}
threadG.Go(func(ctx context.Context) error {
params := map[string]string{
"method": "upload",
"access_token": d.AccessToken,
"type": "tmpfile",
"path": path,
"uploadid": precreateResp.Uploadid,
"partseq": strconv.Itoa(partseq),
}
section := io.NewSectionReader(cacheReaderAt, offset, size)
err := d.uploadSlice(ctx, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section))
if err != nil {
return err
}
precreateResp.BlockList[i] = -1
doneParts++ // 直接递增计数器
if totalParts > 0 {
up(float64(doneParts) * 100.0 / float64(totalParts))
}
return nil
})
}
threadG.Go(func(ctx context.Context) error {
params := map[string]string{
"method": "upload",
"access_token": d.AccessToken,
"type": "tmpfile",
"path": path,
"uploadid": precreateResp.Uploadid,
"partseq": strconv.Itoa(partseq),
}
err := d.uploadSlice(ctx, params, stream.GetName(),
driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize)))
if err != nil {
return err
}
up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList)))
precreateResp.BlockList[i] = -1
return nil
})
}
if err = threadG.Wait(); err != nil {
// 如果属于用户主动取消,则保存上传进度
err = threadG.Wait()
if err == nil {
break uploadLoop
}
// 保存进度(所有错误都会保存)
precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 })
base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5)
if errors.Is(err, context.Canceled) {
precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 })
return nil, err
}
if errors.Is(err, ErrUploadIDExpired) {
log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch")
// 重新 precreate所有分片都要重传
newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime)
if err2 != nil {
return nil, err2
}
if newPre.ReturnType == 2 {
return fileToObj(newPre.File), nil
}
precreateResp = newPre
// 覆盖掉旧的进度
base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5)
continue uploadLoop
}
return nil, err
}
@@ -343,9 +365,46 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
// 修复时间,具体原因见 Put 方法注释的 **注意**
newFile.Ctime = ctime
newFile.Mtime = mtime
// 上传成功清理进度
base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5)
return fileToObj(newFile), nil
}
// precreate 执行预上传操作,支持首次上传和 uploadid 过期重试
func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) {
params := map[string]string{"method": "precreate"}
form := map[string]string{
"path": path,
"size": strconv.FormatInt(streamSize, 10),
"isdir": "0",
"autoinit": "1",
"rtype": "3",
"block_list": blockListStr,
}
// 只有在首次上传时才包含 content-md5 和 slice-md5
if contentMd5 != "" && sliceMd5 != "" {
form["content-md5"] = contentMd5
form["slice-md5"] = sliceMd5
}
joinTime(form, ctime, mtime)
var precreateResp PrecreateResp
_, err := d.postForm("/xpan/file", params, form, &precreateResp)
if err != nil {
return nil, err
}
// 修复时间,具体原因见 Put 方法注释的 **注意**
if precreateResp.ReturnType == 2 {
precreateResp.File.Ctime = ctime
precreateResp.File.Mtime = mtime
}
return &precreateResp, nil
}
func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error {
res, err := base.RestyClient.R().
SetContext(ctx).
@@ -358,8 +417,16 @@ func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string
log.Debugln(res.RawResponse.Status + res.String())
errCode := utils.Json.Get(res.Body(), "error_code").ToInt()
errNo := utils.Json.Get(res.Body(), "errno").ToInt()
respStr := res.String()
lower := strings.ToLower(respStr)
// 合并 uploadid 过期检测逻辑
if strings.Contains(lower, "uploadid") &&
(strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) {
return ErrUploadIDExpired
}
if errCode != 0 || errNo != 0 {
return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String())
return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String())
}
return nil
}

View File

@@ -18,8 +18,9 @@ type Addition struct {
}
var config = driver.Config{
Name: "BaiduPhoto",
LocalSort: true,
Name: "BaiduPhoto",
LocalSort: true,
LinkCacheMode: driver.LinkCacheUA,
}
func init() {

View File

@@ -25,6 +25,7 @@ func InitClient() {
}),
).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify})
NoRedirectClient.SetHeader("user-agent", UserAgent)
net.SetRestyProxyIfConfigured(NoRedirectClient)
RestyClient = NewRestyClient()
HttpClient = net.NewHttpClient()
@@ -37,5 +38,7 @@ func NewRestyClient() *resty.Client {
SetRetryResetReaders(true).
SetTimeout(DefaultTimeout).
SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify})
net.SetRestyProxyIfConfigured(client)
return client
}

View File

@@ -10,6 +10,7 @@ import (
"mime/multipart"
"net/http"
"net/url"
"strconv"
"strings"
"time"
@@ -239,7 +240,7 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStr
if err != nil {
return err
}
err = writer.WriteField("puid", fmt.Sprintf("%d", resp.Msg.Puid))
err = writer.WriteField("puid", strconv.Itoa(resp.Msg.Puid))
if err != nil {
fmt.Println("Error writing param2 to request body:", err)
return err
@@ -260,7 +261,7 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStr
return err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len()))
req.Header.Set("Content-Length", strconv.Itoa(body.Len()))
resps, err := http.DefaultClient.Do(req)
if err != nil {
return err

View File

@@ -258,7 +258,7 @@ type UploadDoneParam struct {
func fileToObj(f File) *model.Object {
if len(f.Content.FolderName) > 0 {
return &model.Object{
ID: fmt.Sprintf("%d", f.ID),
ID: strconv.Itoa(f.ID),
Name: f.Content.FolderName,
Size: 0,
Modified: time.UnixMilli(f.Inserttime),

View File

@@ -9,6 +9,7 @@ import (
"fmt"
"mime/multipart"
"net/http"
"strconv"
"strings"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
@@ -172,7 +173,7 @@ func (d *ChaoXing) Login() (string, error) {
return "", err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len()))
req.Header.Set("Content-Length", strconv.Itoa(body.Len()))
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err

View File

@@ -317,7 +317,8 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
}
return readSeeker, nil
}),
SyncClosers: utils.NewSyncClosers(remoteLink),
SyncClosers: utils.NewSyncClosers(remoteLink),
RequireReference: remoteLink.RequireReference,
}, nil
}

View File

@@ -486,7 +486,7 @@ func (d *Doubao) Upload(ctx context.Context, config *UploadConfig, dstDir model.
"Authorization": {storeInfo.Auth},
"Content-Type": {"application/octet-stream"},
"Content-Crc32": {crc32Value},
"Content-Length": {fmt.Sprintf("%d", file.GetSize())},
"Content-Length": {strconv.FormatInt(file.GetSize(), 10)},
"Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))},
}
res, err := base.HttpClient.Do(req)
@@ -612,7 +612,7 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi
"Authorization": {storeInfo.Auth},
"Content-Type": {"application/octet-stream"},
"Content-Crc32": {crc32Value},
"Content-Length": {fmt.Sprintf("%d", size)},
"Content-Length": {strconv.FormatInt(size, 10)},
"Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))},
}
res, err := base.HttpClient.Do(req)

View File

@@ -16,9 +16,10 @@ type Addition struct {
}
var config = driver.Config{
Name: "FebBox",
NoUpload: true,
DefaultRoot: "0",
Name: "FebBox",
NoUpload: true,
DefaultRoot: "0",
LinkCacheMode: driver.LinkCacheIP,
}
func init() {

View File

@@ -31,11 +31,11 @@ type Addition struct {
}
var config = driver.Config{
Name: "FTP",
LocalSort: true,
OnlyLinkMFile: false,
DefaultRoot: "/",
NoLinkURL: true,
Name: "FTP",
LocalSort: true,
OnlyProxy: true,
DefaultRoot: "/",
NoLinkURL: true,
}
func init() {

View File

@@ -27,6 +27,14 @@ import (
// do others that not defined in Driver interface
// Google Drive API field constants
const (
// File list query fields
FilesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
// Single file query fields
FileInfoFields = "id,name,mimeType,size,md5Checksum,sha1Checksum,sha256Checksum"
)
type googleDriveServiceAccount struct {
// Type string `json:"type"`
// ProjectID string `json:"project_id"`
@@ -235,7 +243,7 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) {
}
query := map[string]string{
"orderBy": orderBy,
"fields": "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken",
"fields": FilesListFields,
"pageSize": "1000",
"q": fmt.Sprintf("'%s' in parents and trashed = false", id),
//"includeItemsFromAllDrives": "true",
@@ -249,11 +257,82 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) {
return nil, err
}
pageToken = resp.NextPageToken
// Batch process shortcuts, API calls only for file shortcuts
shortcutTargetIds := make([]string, 0)
shortcutIndices := make([]int, 0)
// Collect target IDs of all file shortcuts (skip folder shortcuts)
for i := range resp.Files {
if resp.Files[i].MimeType == "application/vnd.google-apps.shortcut" &&
resp.Files[i].ShortcutDetails.TargetId != "" &&
resp.Files[i].ShortcutDetails.TargetMimeType != "application/vnd.google-apps.folder" {
shortcutTargetIds = append(shortcutTargetIds, resp.Files[i].ShortcutDetails.TargetId)
shortcutIndices = append(shortcutIndices, i)
}
}
// Batch get target file info (only for file shortcuts)
if len(shortcutTargetIds) > 0 {
targetFiles := d.batchGetTargetFilesInfo(shortcutTargetIds)
// Update shortcut file info
for j, targetId := range shortcutTargetIds {
if targetFile, exists := targetFiles[targetId]; exists {
fileIndex := shortcutIndices[j]
if targetFile.Size != "" {
resp.Files[fileIndex].Size = targetFile.Size
}
if targetFile.MD5Checksum != "" {
resp.Files[fileIndex].MD5Checksum = targetFile.MD5Checksum
}
if targetFile.SHA1Checksum != "" {
resp.Files[fileIndex].SHA1Checksum = targetFile.SHA1Checksum
}
if targetFile.SHA256Checksum != "" {
resp.Files[fileIndex].SHA256Checksum = targetFile.SHA256Checksum
}
}
}
}
res = append(res, resp.Files...)
}
return res, nil
}
// getTargetFileInfo gets target file details for shortcuts
func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) {
var targetFile File
url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s", targetId)
query := map[string]string{
"fields": FileInfoFields,
}
_, err := d.request(url, http.MethodGet, func(req *resty.Request) {
req.SetQueryParams(query)
}, &targetFile)
if err != nil {
return File{}, err
}
return targetFile, nil
}
// batchGetTargetFilesInfo batch gets target file info, sequential processing to avoid concurrency complexity
func (d *GoogleDrive) batchGetTargetFilesInfo(targetIds []string) map[string]File {
if len(targetIds) == 0 {
return make(map[string]File)
}
result := make(map[string]File)
// Sequential processing to avoid concurrency complexity
for _, targetId := range targetIds {
file, err := d.getTargetFileInfo(targetId)
if err == nil {
result[targetId] = file
}
}
return result
}
func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, url string, up driver.UpdateProgress) error {
defaultChunkSize := d.ChunkSize * 1024 * 1024
ss, err := stream.NewStreamSectionReader(file, int(defaultChunkSize), &up)

View File

@@ -0,0 +1,111 @@
package halalcloudopen
import (
"sync"
"time"
sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user"
)
var (
slicePostErrorRetryInterval = time.Second * 120
retryTimes = 5
)
type halalCommon struct {
// *AuthService // 登录信息
UserInfo *sdkUser.User // 用户信息
refreshTokenFunc func(token string) error
// serv *AuthService
configs sync.Map
}
func (m *halalCommon) GetAccessToken() (string, error) {
value, exists := m.configs.Load("access_token")
if !exists {
return "", nil // 如果不存在,返回空字符串
}
return value.(string), nil // 返回配置项的值
}
// GetRefreshToken implements ConfigStore.
func (m *halalCommon) GetRefreshToken() (string, error) {
value, exists := m.configs.Load("refresh_token")
if !exists {
return "", nil // 如果不存在,返回空字符串
}
return value.(string), nil // 返回配置项的值
}
// SetAccessToken implements ConfigStore.
func (m *halalCommon) SetAccessToken(token string) error {
m.configs.Store("access_token", token)
return nil
}
// SetRefreshToken implements ConfigStore.
func (m *halalCommon) SetRefreshToken(token string) error {
m.configs.Store("refresh_token", token)
if m.refreshTokenFunc != nil {
return m.refreshTokenFunc(token)
}
return nil
}
// SetToken implements ConfigStore.
func (m *halalCommon) SetToken(accessToken string, refreshToken string, expiresIn int64) error {
m.configs.Store("access_token", accessToken)
m.configs.Store("refresh_token", refreshToken)
m.configs.Store("expires_in", expiresIn)
if m.refreshTokenFunc != nil {
return m.refreshTokenFunc(refreshToken)
}
return nil
}
// ClearConfigs implements ConfigStore.
func (m *halalCommon) ClearConfigs() error {
m.configs = sync.Map{} // 清空map
return nil
}
// DeleteConfig implements ConfigStore.
func (m *halalCommon) DeleteConfig(key string) error {
_, exists := m.configs.Load(key)
if !exists {
return nil // 如果不存在,直接返回
}
m.configs.Delete(key) // 删除指定的配置项
return nil
}
// GetConfig implements ConfigStore.
func (m *halalCommon) GetConfig(key string) (string, error) {
value, exists := m.configs.Load(key)
if !exists {
return "", nil // 如果不存在,返回空字符串
}
return value.(string), nil // 返回配置项的值
}
// ListConfigs implements ConfigStore.
func (m *halalCommon) ListConfigs() (map[string]string, error) {
configs := make(map[string]string)
m.configs.Range(func(key, value interface{}) bool {
configs[key.(string)] = value.(string) // 将每个配置项添加到map中
return true // 继续遍历
})
return configs, nil // 返回所有配置项
}
// SetConfig implements ConfigStore.
func (m *halalCommon) SetConfig(key string, value string) error {
m.configs.Store(key, value) // 使用Store方法设置或更新配置项
return nil // 成功设置配置项后返回nil
}
func NewHalalCommon() *halalCommon {
return &halalCommon{
configs: sync.Map{},
}
}

View File

@@ -0,0 +1,29 @@
package halalcloudopen
import (
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
sdkClient "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient"
sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user"
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
)
type HalalCloudOpen struct {
*halalCommon
model.Storage
Addition
sdkClient *sdkClient.Client
sdkUserFileService *sdkUserFile.UserFileService
sdkUserService *sdkUser.UserService
uploadThread int
}
func (d *HalalCloudOpen) Config() driver.Config {
return config
}
func (d *HalalCloudOpen) GetAddition() driver.Additional {
return &d.Addition
}
var _ driver.Driver = (*HalalCloudOpen)(nil)

View File

@@ -0,0 +1,131 @@
package halalcloudopen
import (
"context"
"strconv"
"github.com/OpenListTeam/OpenList/v4/internal/model"
sdkModel "github.com/halalcloud/golang-sdk-lite/halalcloud/model"
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
)
func (d *HalalCloudOpen) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) {
files := make([]model.Obj, 0)
limit := int64(100)
token := ""
for {
result, err := d.sdkUserFileService.List(ctx, &sdkUserFile.FileListRequest{
Parent: &sdkUserFile.File{Path: dir.GetPath()},
ListInfo: &sdkModel.ScanListRequest{
Limit: strconv.FormatInt(limit, 10),
Token: token,
},
})
if err != nil {
return nil, err
}
for i := 0; len(result.Files) > i; i++ {
files = append(files, NewObjFile(result.Files[i]))
}
if result.ListInfo == nil || result.ListInfo.Token == "" {
break
}
token = result.ListInfo.Token
}
return files, nil
}
func (d *HalalCloudOpen) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) {
_, err := d.sdkUserFileService.Create(ctx, &sdkUserFile.File{
Path: dir.GetPath(),
Name: name,
})
return nil, err
}
func (d *HalalCloudOpen) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) {
oldDir := obj.GetPath()
newDir := dir.GetPath()
_, err := d.sdkUserFileService.Move(ctx, &sdkUserFile.BatchOperationRequest{
Source: []*sdkUserFile.File{
{
Path: oldDir,
},
},
Dest: &sdkUserFile.File{
Path: newDir,
},
})
return nil, err
}
func (d *HalalCloudOpen) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) {
_, err := d.sdkUserFileService.Rename(ctx, &sdkUserFile.File{
Path: obj.GetPath(),
Name: name,
})
return nil, err
}
func (d *HalalCloudOpen) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) {
id := obj.GetID()
sourcePath := obj.GetPath()
if len(id) > 0 {
sourcePath = ""
}
destID := dir.GetID()
destPath := dir.GetPath()
if len(destID) > 0 {
destPath = ""
}
dest := &sdkUserFile.File{
Path: destPath,
Identity: destID,
}
_, err := d.sdkUserFileService.Copy(ctx, &sdkUserFile.BatchOperationRequest{
Source: []*sdkUserFile.File{
{
Path: sourcePath,
Identity: id,
},
},
Dest: dest,
})
return nil, err
}
func (d *HalalCloudOpen) remove(ctx context.Context, obj model.Obj) error {
id := obj.GetID()
_, err := d.sdkUserFileService.Delete(ctx, &sdkUserFile.BatchOperationRequest{
Source: []*sdkUserFile.File{
{
Identity: id,
Path: obj.GetPath(),
},
},
})
return err
}
func (d *HalalCloudOpen) details(ctx context.Context) (*model.StorageDetails, error) {
ret, err := d.sdkUserService.GetStatisticsAndQuota(ctx)
if err != nil {
return nil, err
}
total := uint64(ret.DiskStatisticsQuota.BytesQuota)
free := uint64(ret.DiskStatisticsQuota.BytesFree)
return &model.StorageDetails{
DiskUsage: model.DiskUsage{
TotalSpace: total,
FreeSpace: free,
},
}, nil
}

View File

@@ -0,0 +1,108 @@
package halalcloudopen
import (
"context"
"crypto/sha1"
"io"
"strconv"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
"github.com/rclone/rclone/lib/readers"
)
func (d *HalalCloudOpen) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
if args.Redirect {
// return nil, model.ErrUnsupported
fid := file.GetID()
fpath := file.GetPath()
if fid != "" {
fpath = ""
}
fi, err := d.sdkUserFileService.GetDirectDownloadAddress(ctx, &sdkUserFile.DirectDownloadRequest{
Identity: fid,
Path: fpath,
})
if err != nil {
return nil, err
}
expireAt := fi.ExpireAt
duration := time.Until(time.UnixMilli(expireAt))
return &model.Link{
URL: fi.DownloadAddress,
Expiration: &duration,
}, nil
}
result, err := d.sdkUserFileService.ParseFileSlice(ctx, &sdkUserFile.File{
Identity: file.GetID(),
Path: file.GetPath(),
})
if err != nil {
return nil, err
}
fileAddrs := []*sdkUserFile.SliceDownloadInfo{}
var addressDuration int64
nodesNumber := len(result.RawNodes)
nodesIndex := nodesNumber - 1
startIndex, endIndex := 0, nodesIndex
for nodesIndex >= 0 {
if nodesIndex >= 200 {
endIndex = 200
} else {
endIndex = nodesNumber
}
for ; endIndex <= nodesNumber; endIndex += 200 {
if endIndex == 0 {
endIndex = 1
}
sliceAddress, err := d.sdkUserFileService.GetSliceDownloadAddress(ctx, &sdkUserFile.SliceDownloadAddressRequest{
Identity: result.RawNodes[startIndex:endIndex],
Version: 1,
})
if err != nil {
return nil, err
}
addressDuration, _ = strconv.ParseInt(sliceAddress.ExpireAt, 10, 64)
fileAddrs = append(fileAddrs, sliceAddress.Addresses...)
startIndex = endIndex
nodesIndex -= 200
}
}
size, _ := strconv.ParseInt(result.FileSize, 10, 64)
chunks := getChunkSizes(result.Sizes)
resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
length := httpRange.Length
if httpRange.Length < 0 || httpRange.Start+httpRange.Length >= size {
length = size - httpRange.Start
}
oo := &openObject{
ctx: ctx,
d: fileAddrs,
chunk: []byte{},
chunks: chunks,
skip: httpRange.Start,
sha: result.Sha1,
shaTemp: sha1.New(),
}
return readers.NewLimitedReadCloser(oo, length), nil
}
var duration time.Duration
if addressDuration != 0 {
duration = time.Until(time.UnixMilli(addressDuration))
} else {
duration = time.Until(time.Now().Add(time.Hour))
}
return &model.Link{
RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader),
Expiration: &duration,
}, nil
}

View File

@@ -0,0 +1,50 @@
package halalcloudopen
import (
"context"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient"
sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user"
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
)
func (d *HalalCloudOpen) Init(ctx context.Context) error {
if d.uploadThread < 1 || d.uploadThread > 32 {
d.uploadThread, d.UploadThread = 3, 3
}
if d.halalCommon == nil {
d.halalCommon = &halalCommon{
UserInfo: &sdkUser.User{},
refreshTokenFunc: func(token string) error {
d.Addition.RefreshToken = token
op.MustSaveDriverStorage(d)
return nil
},
}
}
if d.Addition.RefreshToken != "" {
d.halalCommon.SetRefreshToken(d.Addition.RefreshToken)
}
timeout := d.Addition.TimeOut
if timeout <= 0 {
timeout = 60
}
host := d.Addition.Host
if host == "" {
host = "openapi.2dland.cn"
}
client := apiclient.NewClient(nil, host, d.Addition.ClientID, d.Addition.ClientSecret, d.halalCommon, apiclient.WithTimeout(time.Second*time.Duration(timeout)))
d.sdkClient = client
d.sdkUserFileService = sdkUserFile.NewUserFileService(client)
d.sdkUserService = sdkUser.NewUserService(client)
userInfo, err := d.sdkUserService.Get(ctx, &sdkUser.User{})
if err != nil {
return err
}
d.halalCommon.UserInfo = userInfo
// 能够获取到用户信息,已经检查了 RefreshToken 的有效性,无需再次检查
return nil
}

View File

@@ -0,0 +1,48 @@
package halalcloudopen
import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
func (d *HalalCloudOpen) Drop(ctx context.Context) error {
return nil
}
func (d *HalalCloudOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
return d.getFiles(ctx, dir)
}
func (d *HalalCloudOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
return d.getLink(ctx, file, args)
}
func (d *HalalCloudOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
return d.makeDir(ctx, parentDir, dirName)
}
func (d *HalalCloudOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
return d.move(ctx, srcObj, dstDir)
}
func (d *HalalCloudOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
return d.rename(ctx, srcObj, newName)
}
func (d *HalalCloudOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
return d.copy(ctx, srcObj, dstDir)
}
func (d *HalalCloudOpen) Remove(ctx context.Context, obj model.Obj) error {
return d.remove(ctx, obj)
}
func (d *HalalCloudOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
return d.put(ctx, dstDir, stream, up)
}
func (d *HalalCloudOpen) GetDetails(ctx context.Context) (*model.StorageDetails, error) {
return d.details(ctx)
}

View File

@@ -0,0 +1,258 @@
package halalcloudopen
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
"github.com/ipfs/go-cid"
)
func (d *HalalCloudOpen) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
newPath := path.Join(dstDir.GetPath(), fileStream.GetName())
uploadTask, err := d.sdkUserFileService.CreateUploadTask(ctx, &sdkUserFile.File{
Path: newPath,
Size: fileStream.GetSize(),
})
if err != nil {
return nil, err
}
if uploadTask.Created {
return nil, nil
}
slicesList := make([]string, 0)
codec := uint64(0x55)
if uploadTask.BlockCodec > 0 {
codec = uint64(uploadTask.BlockCodec)
}
blockHashType := uploadTask.BlockHashType
mhType := uint64(0x12)
if blockHashType > 0 {
mhType = uint64(blockHashType)
}
prefix := cid.Prefix{
Codec: codec,
MhLength: -1,
MhType: mhType,
Version: 1,
}
blockSize := uploadTask.BlockSize
useSingleUpload := true
//
if fileStream.GetSize() <= int64(blockSize) || d.uploadThread <= 1 {
useSingleUpload = true
}
// Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety.
// read file
if useSingleUpload {
bufferSize := int(blockSize)
buffer := make([]byte, bufferSize)
reader := driver.NewLimitedUploadStream(ctx, fileStream)
teeReader := io.TeeReader(reader, driver.NewProgress(fileStream.GetSize(), up))
// fileStream.Seek(0, os.SEEK_SET)
for {
n, err := teeReader.Read(buffer)
if n > 0 {
data := buffer[:n]
uploadCid, err := postFileSlice(ctx, data, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes)
if err != nil {
return nil, err
}
slicesList = append(slicesList, uploadCid.String())
}
if err == io.EOF || n == 0 {
break
}
}
} else {
// TODO: implement multipart upload, currently using single-threaded upload to ensure safety.
bufferSize := int(blockSize)
buffer := make([]byte, bufferSize)
reader := driver.NewLimitedUploadStream(ctx, fileStream)
teeReader := io.TeeReader(reader, driver.NewProgress(fileStream.GetSize(), up))
for {
n, err := teeReader.Read(buffer)
if n > 0 {
data := buffer[:n]
uploadCid, err := postFileSlice(ctx, data, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes)
if err != nil {
return nil, err
}
slicesList = append(slicesList, uploadCid.String())
}
if err == io.EOF || n == 0 {
break
}
}
}
newFile, err := makeFile(ctx, slicesList, uploadTask.Task, uploadTask.UploadAddress, retryTimes)
if err != nil {
return nil, err
}
return NewObjFile(newFile), nil
}
func makeFile(ctx context.Context, fileSlice []string, taskID string, uploadAddress string, retry int) (*sdkUserFile.File, error) {
var lastError error = nil
for range retry {
newFile, err := doMakeFile(fileSlice, taskID, uploadAddress)
if err == nil {
return newFile, nil
}
if ctx.Err() != nil {
return nil, err
}
if strings.Contains(err.Error(), "not found") {
return nil, err
}
lastError = err
time.Sleep(slicePostErrorRetryInterval)
}
return nil, fmt.Errorf("mk file slice failed after %d times, error: %s", retry, lastError.Error())
}
func doMakeFile(fileSlice []string, taskID string, uploadAddress string) (*sdkUserFile.File, error) {
accessUrl := uploadAddress + "/" + taskID
getTimeOut := time.Minute * 2
u, err := url.Parse(accessUrl)
if err != nil {
return nil, err
}
n, _ := json.Marshal(fileSlice)
httpRequest := http.Request{
Method: http.MethodPost,
URL: u,
Header: map[string][]string{
"Accept": {"application/json"},
"Content-Type": {"application/json"},
//"Content-Length": {strconv.Itoa(len(n))},
},
Body: io.NopCloser(bytes.NewReader(n)),
}
httpClient := http.Client{
Timeout: getTimeOut,
}
httpResponse, err := httpClient.Do(&httpRequest)
if err != nil {
return nil, err
}
defer httpResponse.Body.Close()
if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated {
b, _ := io.ReadAll(httpResponse.Body)
message := string(b)
return nil, fmt.Errorf("mk file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message)
}
b, _ := io.ReadAll(httpResponse.Body)
var result *sdkUserFile.File
err = json.Unmarshal(b, &result)
if err != nil {
return nil, err
}
return result, nil
}
func postFileSlice(ctx context.Context, fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix, retry int) (cid.Cid, error) {
var lastError error = nil
for range retry {
newCid, err := doPostFileSlice(fileSlice, taskID, uploadAddress, preix)
if err == nil {
return newCid, nil
}
if ctx.Err() != nil {
return cid.Undef, err
}
time.Sleep(slicePostErrorRetryInterval)
lastError = err
}
return cid.Undef, fmt.Errorf("upload file slice failed after %d times, error: %s", retry, lastError.Error())
}
func doPostFileSlice(fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix) (cid.Cid, error) {
// 1. sum file slice
newCid, err := preix.Sum(fileSlice)
if err != nil {
return cid.Undef, err
}
// 2. post file slice
sliceCidString := newCid.String()
// /{taskID}/{sliceID}
accessUrl := uploadAddress + "/" + taskID + "/" + sliceCidString
getTimeOut := time.Second * 30
// get {accessUrl} in {getTimeOut}
u, err := url.Parse(accessUrl)
if err != nil {
return cid.Undef, err
}
// header: accept: application/json
// header: content-type: application/octet-stream
// header: content-length: {fileSlice.length}
// header: x-content-cid: {sliceCidString}
// header: x-task-id: {taskID}
httpRequest := http.Request{
Method: http.MethodGet,
URL: u,
Header: map[string][]string{
"Accept": {"application/json"},
},
}
httpClient := http.Client{
Timeout: getTimeOut,
}
httpResponse, err := httpClient.Do(&httpRequest)
if err != nil {
return cid.Undef, err
}
if httpResponse.StatusCode != http.StatusOK {
return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d", httpResponse.StatusCode)
}
var result bool
b, err := io.ReadAll(httpResponse.Body)
if err != nil {
return cid.Undef, err
}
err = json.Unmarshal(b, &result)
if err != nil {
return cid.Undef, err
}
if result {
return newCid, nil
}
httpRequest = http.Request{
Method: http.MethodPost,
URL: u,
Header: map[string][]string{
"Accept": {"application/json"},
"Content-Type": {"application/octet-stream"},
// "Content-Length": {strconv.Itoa(len(fileSlice))},
},
Body: io.NopCloser(bytes.NewReader(fileSlice)),
}
httpResponse, err = httpClient.Do(&httpRequest)
if err != nil {
return cid.Undef, err
}
defer httpResponse.Body.Close()
if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated {
b, _ := io.ReadAll(httpResponse.Body)
message := string(b)
return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message)
}
//
return newCid, nil
}

View File

@@ -0,0 +1,32 @@
package halalcloudopen
import (
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/op"
)
type Addition struct {
// Usually one of two
driver.RootPath
// define other
RefreshToken string `json:"refresh_token" required:"false" help:"If using a personal API approach, the RefreshToken is not required."`
UploadThread int `json:"upload_thread" type:"number" default:"3" help:"1 <= thread <= 32"`
ClientID string `json:"client_id" required:"true" default:""`
ClientSecret string `json:"client_secret" required:"true" default:""`
Host string `json:"host" required:"false" default:"openapi.2dland.cn"`
TimeOut int `json:"timeout" type:"number" default:"60" help:"timeout in seconds"`
}
var config = driver.Config{
Name: "HalalCloudOpen",
OnlyProxy: false,
DefaultRoot: "/",
NoLinkURL: false,
}
func init() {
op.RegisterDriver(func() driver.Driver {
return &HalalCloudOpen{}
})
}

View File

@@ -0,0 +1,60 @@
package halalcloudopen
import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
)
type ObjFile struct {
sdkFile *sdkUserFile.File
fileSize int64
modTime time.Time
createTime time.Time
}
func NewObjFile(f *sdkUserFile.File) model.Obj {
ofile := &ObjFile{sdkFile: f}
ofile.fileSize = f.Size
modTimeTs := f.UpdateTs
ofile.modTime = time.UnixMilli(modTimeTs)
createTimeTs := f.CreateTs
ofile.createTime = time.UnixMilli(createTimeTs)
return ofile
}
func (f *ObjFile) GetSize() int64 {
return f.fileSize
}
func (f *ObjFile) GetName() string {
return f.sdkFile.Name
}
func (f *ObjFile) ModTime() time.Time {
return f.modTime
}
func (f *ObjFile) IsDir() bool {
return f.sdkFile.Dir
}
func (f *ObjFile) GetHash() utils.HashInfo {
return utils.HashInfo{
// TODO: support more hash types
}
}
func (f *ObjFile) GetID() string {
return f.sdkFile.Identity
}
func (f *ObjFile) GetPath() string {
return f.sdkFile.Path
}
func (f *ObjFile) CreateTime() time.Time {
return f.createTime
}

View File

@@ -0,0 +1,185 @@
package halalcloudopen
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"net/http"
"sync"
"time"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
"github.com/ipfs/go-cid"
)
// get the next chunk
func (oo *openObject) getChunk(_ context.Context) (err error) {
if oo.id >= len(oo.chunks) {
return io.EOF
}
var chunk []byte
err = utils.Retry(3, time.Second, func() (err error) {
chunk, err = getRawFiles(oo.d[oo.id])
return err
})
if err != nil {
return err
}
oo.id++
oo.chunk = chunk
return nil
}
// Read reads up to len(p) bytes into p.
func (oo *openObject) Read(p []byte) (n int, err error) {
oo.mu.Lock()
defer oo.mu.Unlock()
if oo.closed {
return 0, fmt.Errorf("read on closed file")
}
// Skip data at the start if requested
for oo.skip > 0 {
//size := 1024 * 1024
_, size, err := oo.ChunkLocation(oo.id)
if err != nil {
return 0, err
}
if oo.skip < int64(size) {
break
}
oo.id++
oo.skip -= int64(size)
}
if len(oo.chunk) == 0 {
err = oo.getChunk(oo.ctx)
if err != nil {
return 0, err
}
if oo.skip > 0 {
oo.chunk = (oo.chunk)[oo.skip:]
oo.skip = 0
}
}
n = copy(p, oo.chunk)
oo.shaTemp.Write(p[:n])
oo.chunk = (oo.chunk)[n:]
return n, nil
}
// Close closed the file - MAC errors are reported here
func (oo *openObject) Close() (err error) {
oo.mu.Lock()
defer oo.mu.Unlock()
if oo.closed {
return nil
}
// 校验Sha1
if string(oo.shaTemp.Sum(nil)) != oo.sha {
return fmt.Errorf("failed to finish download: SHA mismatch")
}
oo.closed = true
return nil
}
func GetMD5Hash(text string) string {
tHash := md5.Sum([]byte(text))
return hex.EncodeToString(tHash[:])
}
type chunkSize struct {
position int64
size int
}
type openObject struct {
ctx context.Context
mu sync.Mutex
d []*sdkUserFile.SliceDownloadInfo
id int
skip int64
chunk []byte
chunks []chunkSize
closed bool
sha string
shaTemp hash.Hash
}
func getChunkSizes(sliceSize []*sdkUserFile.SliceSize) (chunks []chunkSize) {
chunks = make([]chunkSize, 0)
for _, s := range sliceSize {
// 对最后一个做特殊处理
endIndex := s.EndIndex
startIndex := s.StartIndex
if endIndex == 0 {
endIndex = startIndex
}
for j := startIndex; j <= endIndex; j++ {
size := s.Size
chunks = append(chunks, chunkSize{position: j, size: int(size)})
}
}
return chunks
}
func (oo *openObject) ChunkLocation(id int) (position int64, size int, err error) {
if id < 0 || id >= len(oo.chunks) {
return 0, 0, errors.New("invalid arguments")
}
return (oo.chunks)[id].position, (oo.chunks)[id].size, nil
}
func getRawFiles(addr *sdkUserFile.SliceDownloadInfo) ([]byte, error) {
if addr == nil {
return nil, errors.New("addr is nil")
}
client := http.Client{
Timeout: time.Duration(60 * time.Second), // Set timeout to 60 seconds
}
resp, err := client.Get(addr.DownloadAddress)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("bad status: %s, body: %s", resp.Status, body)
}
if addr.Encrypt > 0 {
cd := uint8(addr.Encrypt)
for idx := 0; idx < len(body); idx++ {
body[idx] = body[idx] ^ cd
}
}
storeType := addr.StoreType
if storeType != 10 {
sourceCid, err := cid.Decode(addr.Identity)
if err != nil {
return nil, err
}
checkCid, err := sourceCid.Prefix().Sum(body)
if err != nil {
return nil, err
}
if !checkCid.Equals(sourceCid) {
return nil, fmt.Errorf("bad cid: %s, body: %s", checkCid.String(), body)
}
}
return body, nil
}

View File

@@ -235,6 +235,7 @@ func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) {
func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
fullPath := file.GetPath()
link := &model.Link{}
var MFile model.File
if args.Type == "thumb" && utils.Ext(file.GetName()) != "svg" {
var buf *bytes.Buffer
var thumbPath *string
@@ -261,9 +262,9 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
return nil, err
}
link.ContentLength = int64(stat.Size())
link.MFile = open
MFile = open
} else {
link.MFile = bytes.NewReader(buf.Bytes())
MFile = bytes.NewReader(buf.Bytes())
link.ContentLength = int64(buf.Len())
}
} else {
@@ -272,13 +273,11 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
return nil, err
}
link.ContentLength = file.GetSize()
link.MFile = open
}
link.AddIfCloser(link.MFile)
if !d.Config().OnlyLinkMFile {
link.RangeReader = stream.GetRangeReaderFromMFile(link.ContentLength, link.MFile)
link.MFile = nil
MFile = open
}
link.SyncClosers.AddIfCloser(MFile)
link.RangeReader = stream.GetRangeReaderFromMFile(link.ContentLength, MFile)
link.RequireReference = link.SyncClosers.Length() > 0
return link, nil
}
@@ -375,18 +374,26 @@ func (d *Local) Remove(ctx context.Context, obj model.Obj) error {
err = os.Remove(obj.GetPath())
}
} else {
if !utils.Exists(d.RecycleBinPath) {
err = os.MkdirAll(d.RecycleBinPath, 0o755)
objPath := obj.GetPath()
objName := obj.GetName()
var relPath string
relPath, err = filepath.Rel(d.GetRootPath(), filepath.Dir(objPath))
if err != nil {
return err
}
recycleBinPath := filepath.Join(d.RecycleBinPath, relPath)
if !utils.Exists(recycleBinPath) {
err = os.MkdirAll(recycleBinPath, 0o755)
if err != nil {
return err
}
}
dstPath := filepath.Join(d.RecycleBinPath, obj.GetName())
dstPath := filepath.Join(recycleBinPath, objName)
if utils.Exists(dstPath) {
dstPath = filepath.Join(d.RecycleBinPath, obj.GetName()+"_"+time.Now().Format("20060102150405"))
dstPath = filepath.Join(recycleBinPath, objName+"_"+time.Now().Format("20060102150405"))
}
err = os.Rename(obj.GetPath(), dstPath)
err = os.Rename(objPath, dstPath)
}
if err != nil {
return err

View File

@@ -18,12 +18,12 @@ type Addition struct {
}
var config = driver.Config{
Name: "Local",
OnlyLinkMFile: false,
LocalSort: true,
NoCache: true,
DefaultRoot: "/",
NoLinkURL: true,
Name: "Local",
LocalSort: true,
OnlyProxy: true,
NoCache: true,
DefaultRoot: "/",
NoLinkURL: true,
}
func init() {

View File

@@ -36,7 +36,6 @@ type Addition struct {
var config = driver.Config{
Name: "MediaFire",
LocalSort: false,
OnlyLinkMFile: false,
OnlyProxy: false,
NoCache: false,
NoUpload: false,

View File

@@ -22,6 +22,7 @@ type Onedrive struct {
AccessToken string
root *Object
mutex sync.Mutex
ref *Onedrive
}
func (d *Onedrive) Config() driver.Config {
@@ -36,10 +37,22 @@ func (d *Onedrive) Init(ctx context.Context) error {
if d.ChunkSize < 1 {
d.ChunkSize = 5
}
if d.ref != nil {
return nil
}
return d.refreshToken()
}
func (d *Onedrive) InitReference(refStorage driver.Driver) error {
if ref, ok := refStorage.(*Onedrive); ok {
d.ref = ref
return nil
}
return errs.NotSupport
}
func (d *Onedrive) Drop(ctx context.Context) error {
d.ref = nil
return nil
}

View File

@@ -134,6 +134,9 @@ func (d *Onedrive) _refreshToken() error {
}
func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
if d.ref != nil {
return d.ref.Request(url, method, callback, resp)
}
req := base.RestyClient.R()
req.SetHeader("Authorization", "Bearer "+d.AccessToken)
if callback != nil {

View File

@@ -110,19 +110,29 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs)
func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
var resp common.Resp[FsGetResp]
headers := map[string]string{
"User-Agent": base.UserAgent,
}
// if PassUAToUpsteam is true, then pass the user-agent to the upstream
userAgent := base.UserAgent
if d.PassUAToUpsteam {
userAgent = args.Header.Get("user-agent")
if userAgent == "" {
userAgent = base.UserAgent
userAgent := args.Header.Get("user-agent")
if userAgent != "" {
headers["User-Agent"] = base.UserAgent
}
}
// if PassIPToUpsteam is true, then pass the ip address to the upstream
if d.PassIPToUpsteam {
ip := args.IP
if ip != "" {
headers["X-Forwarded-For"] = ip
headers["X-Real-Ip"] = ip
}
}
_, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) {
req.SetResult(&resp).SetBody(FsGetReq{
Path: file.GetPath(),
Password: d.MetaPassword,
}).SetHeader("user-agent", userAgent)
}).SetHeaders(headers)
})
if err != nil {
return nil, err
@@ -355,8 +365,15 @@ func (d *OpenList) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.O
return err
}
//func (d *OpenList) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
// return nil, errs.NotSupport
//}
func (d *OpenList) ResolveLinkCacheMode(_ string) driver.LinkCacheMode {
var mode driver.LinkCacheMode
if d.PassIPToUpsteam {
mode |= driver.LinkCacheIP
}
if d.PassUAToUpsteam {
mode |= driver.LinkCacheUA
}
return mode
}
var _ driver.Driver = (*OpenList)(nil)

View File

@@ -12,6 +12,7 @@ type Addition struct {
Username string `json:"username"`
Password string `json:"password"`
Token string `json:"token"`
PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"`
PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"`
ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"`
}
@@ -22,6 +23,7 @@ var config = driver.Config{
DefaultRoot: "/",
CheckStatus: true,
ProxyRangeOption: true,
LinkCacheMode: driver.LinkCacheAuto,
}
func init() {

View File

@@ -0,0 +1,290 @@
package protondrive
/*
Package protondrive
Author: Da3zKi7<da3zki7@duck.com>
Date: 2025-09-18
Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge
The power of open-source, the force of teamwork and the magic of reverse engineering!
D@' 3z K!7 - The King Of Cracking
Да здравствует Родина))
*/
import (
"context"
"fmt"
"io"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/ProtonMail/gopenpgp/v2/crypto"
proton_api_bridge "github.com/henrybear327/Proton-API-Bridge"
"github.com/henrybear327/Proton-API-Bridge/common"
"github.com/henrybear327/go-proton-api"
)
type ProtonDrive struct {
model.Storage
Addition
protonDrive *proton_api_bridge.ProtonDrive
apiBase string
appVersion string
protonJson string
userAgent string
sdkVersion string
webDriveAV string
c *proton.Client
// userKR *crypto.KeyRing
addrKRs map[string]*crypto.KeyRing
addrData map[string]proton.Address
MainShare *proton.Share
DefaultAddrKR *crypto.KeyRing
MainShareKR *crypto.KeyRing
}
func (d *ProtonDrive) Config() driver.Config {
return config
}
func (d *ProtonDrive) GetAddition() driver.Additional {
return &d.Addition
}
func (d *ProtonDrive) Init(ctx context.Context) (err error) {
defer func() {
if r := recover(); err == nil && r != nil {
err = fmt.Errorf("ProtonDrive initialization panic: %v", r)
}
}()
if d.Email == "" {
return fmt.Errorf("email is required")
}
if d.Password == "" {
return fmt.Errorf("password is required")
}
config := &common.Config{
AppVersion: d.appVersion,
UserAgent: d.userAgent,
FirstLoginCredential: &common.FirstLoginCredentialData{
Username: d.Email,
Password: d.Password,
TwoFA: d.TwoFACode,
},
EnableCaching: true,
ConcurrentBlockUploadCount: setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers),
//ConcurrentFileCryptoCount: 2,
UseReusableLogin: d.UseReusableLogin && d.ReusableCredential != (common.ReusableCredentialData{}),
ReplaceExistingDraft: true,
ReusableCredential: &d.ReusableCredential,
}
protonDrive, _, err := proton_api_bridge.NewProtonDrive(
ctx,
config,
d.authHandler,
func() {},
)
if err != nil && config.UseReusableLogin {
config.UseReusableLogin = false
protonDrive, _, err = proton_api_bridge.NewProtonDrive(ctx,
config,
d.authHandler,
func() {},
)
if err == nil {
op.MustSaveDriverStorage(d)
}
}
if err != nil {
return fmt.Errorf("failed to initialize ProtonDrive: %w", err)
}
if err := d.initClient(ctx); err != nil {
return err
}
d.protonDrive = protonDrive
d.MainShare = protonDrive.MainShare
if d.RootFolderID == "root" || d.RootFolderID == "" {
d.RootFolderID = protonDrive.RootLink.LinkID
}
d.MainShareKR = protonDrive.MainShareKR
d.DefaultAddrKR = protonDrive.DefaultAddrKR
return nil
}
func (d *ProtonDrive) Drop(ctx context.Context) error {
return nil
}
func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
entries, err := d.protonDrive.ListDirectory(ctx, dir.GetID())
if err != nil {
return nil, fmt.Errorf("failed to list directory: %w", err)
}
objects := make([]model.Obj, 0, len(entries))
for _, entry := range entries {
obj := &model.Object{
ID: entry.Link.LinkID,
Name: entry.Name,
Size: entry.Link.Size,
Modified: time.Unix(entry.Link.ModifyTime, 0),
IsFolder: entry.IsFolder,
}
objects = append(objects, obj)
}
return objects, nil
}
func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
link, err := d.getLink(ctx, file.GetID())
if err != nil {
return nil, fmt.Errorf("failed get file link: %+v", err)
}
fileSystemAttrs, err := d.protonDrive.GetActiveRevisionAttrs(ctx, link)
if err != nil {
return nil, fmt.Errorf("failed get file revision: %+v", err)
}
// 解密后的文件大小
size := fileSystemAttrs.Size
rangeReaderFunc := func(rangeCtx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
length := httpRange.Length
if length < 0 || httpRange.Start+length > size {
length = size - httpRange.Start
}
reader, _, _, err := d.protonDrive.DownloadFile(rangeCtx, link, httpRange.Start)
if err != nil {
return nil, fmt.Errorf("failed start download: %+v", err)
}
return utils.ReadCloser{
Reader: io.LimitReader(reader, length),
Closer: reader,
}, nil
}
expiration := time.Minute
return &model.Link{
RangeReader: &model.FileRangeReader{
RangeReaderIF: stream.RateLimitRangeReaderFunc(rangeReaderFunc),
},
ContentLength: size,
Expiration: &expiration,
}, nil
}
func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
id, err := d.protonDrive.CreateNewFolderByID(ctx, parentDir.GetID(), dirName)
if err != nil {
return nil, fmt.Errorf("failed to create directory: %w", err)
}
newDir := &model.Object{
ID: id,
Name: dirName,
IsFolder: true,
Modified: time.Now(),
}
return newDir, nil
}
func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
return d.DirectMove(ctx, srcObj, dstDir)
}
func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
if d.protonDrive == nil {
return nil, fmt.Errorf("protonDrive bridge is nil")
}
return d.DirectRename(ctx, srcObj, newName)
}
func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
if srcObj.IsDir() {
return nil, fmt.Errorf("directory copy not supported")
}
srcLink, err := d.getLink(ctx, srcObj.GetID())
if err != nil {
return nil, err
}
reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0)
if err != nil {
return nil, fmt.Errorf("failed to download source file: %w", err)
}
defer reader.Close()
actualSize := linkSize
if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 {
actualSize = fileSystemAttrs.Size
}
file := &stream.FileStream{
Ctx: ctx,
Obj: &model.Object{
Name: srcObj.GetName(),
// Use the accurate and real size
Size: actualSize,
Modified: srcObj.ModTime(),
},
Reader: reader,
}
defer file.Close()
return d.Put(ctx, dstDir, file, func(percentage float64) {})
}
func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error {
if obj.IsDir() {
return d.protonDrive.MoveFolderToTrashByID(ctx, obj.GetID(), false)
} else {
return d.protonDrive.MoveFileToTrashByID(ctx, obj.GetID())
}
}
func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
return d.uploadFile(ctx, dstDir.GetID(), file, up)
}
func (d *ProtonDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) {
about, err := d.protonDrive.About(ctx)
if err != nil {
return nil, err
}
total := uint64(about.MaxSpace)
free := total - uint64(about.UsedSpace)
return &model.StorageDetails{
DiskUsage: model.DiskUsage{
TotalSpace: total,
FreeSpace: free,
},
}, nil
}
var _ driver.Driver = (*ProtonDrive)(nil)

View File

@@ -0,0 +1,56 @@
package protondrive
/*
Package protondrive
Author: Da3zKi7<da3zki7@duck.com>
Date: 2025-09-18
Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge
The power of open-source, the force of teamwork and the magic of reverse engineering!
D@' 3z K!7 - The King Of Cracking
Да здравствует Родина))
*/
import (
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/henrybear327/Proton-API-Bridge/common"
)
type Addition struct {
driver.RootID
Email string `json:"email" required:"true" type:"string"`
Password string `json:"password" required:"true" type:"string"`
TwoFACode string `json:"two_fa_code" type:"string"`
ChunkSize int64 `json:"chunk_size" type:"number" default:"100"`
UseReusableLogin bool `json:"use_reusable_login" type:"bool" default:"true" help:"Use reusable login credentials instead of username/password"`
ReusableCredential common.ReusableCredentialData
}
var config = driver.Config{
Name: "ProtonDrive",
LocalSort: true,
OnlyProxy: true,
DefaultRoot: "root",
NoLinkURL: true,
}
func init() {
op.RegisterDriver(func() driver.Driver {
return &ProtonDrive{
Addition: Addition{
UseReusableLogin: true,
},
apiBase: "https://drive.proton.me/api",
appVersion: "windows-drive@1.11.3+rclone+proton",
protonJson: "application/vnd.protonmail.v1+json",
sdkVersion: "js@0.3.0",
userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)",
webDriveAV: "web-drive@5.2.0+0f69f7a8",
}
})
}

View File

@@ -0,0 +1,38 @@
package protondrive
/*
Package protondrive
Author: Da3zKi7<da3zki7@duck.com>
Date: 2025-09-18
Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge
The power of open-source, the force of teamwork and the magic of reverse engineering!
D@' 3z K!7 - The King Of Cracking
Да здравствует Родина))
*/
type MoveRequest struct {
ParentLinkID string `json:"ParentLinkID"`
NodePassphrase string `json:"NodePassphrase"`
NodePassphraseSignature *string `json:"NodePassphraseSignature"`
Name string `json:"Name"`
NameSignatureEmail string `json:"NameSignatureEmail"`
Hash string `json:"Hash"`
OriginalHash string `json:"OriginalHash"`
ContentHash *string `json:"ContentHash"` // Maybe null
}
type RenameRequest struct {
Name string `json:"Name"` // PGP encrypted name
NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email
Hash string `json:"Hash"` // New name hash
OriginalHash string `json:"OriginalHash"` // Current name hash
}
type RenameResponse struct {
Code int `json:"Code"`
}

View File

@@ -0,0 +1,670 @@
package protondrive
/*
Package protondrive
Author: Da3zKi7<da3zki7@duck.com>
Date: 2025-09-18
Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge
The power of open-source, the force of teamwork and the magic of reverse engineering!
D@' 3z K!7 - The King Of Cracking
Да здравствует Родина))
*/
import (
"bufio"
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/henrybear327/go-proton-api"
)
func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID string, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
_, err := d.getLink(ctx, parentLinkID)
if err != nil {
return nil, fmt.Errorf("failed to get parent link: %w", err)
}
var reader io.Reader
// Use buffered reader with larger buffer for better performance
var bufferSize int
// File > 100MB (default)
if file.GetSize() > d.ChunkSize*1024*1024 {
// 256KB for large files
bufferSize = 256 * 1024
// File > 10MB
} else if file.GetSize() > 10*1024*1024 {
// 128KB for medium files
bufferSize = 128 * 1024
} else {
// 64KB for small files
bufferSize = 64 * 1024
}
// reader = bufio.NewReader(file)
reader = bufio.NewReaderSize(file, bufferSize)
reader = &driver.ReaderUpdatingProgress{
Reader: &stream.SimpleReaderWithSize{
Reader: reader,
Size: file.GetSize(),
},
UpdateProgress: up,
}
reader = driver.NewLimitedUploadStream(ctx, reader)
id, _, err := d.protonDrive.UploadFileByReader(ctx, parentLinkID, file.GetName(), file.ModTime(), reader, 0)
if err != nil {
return nil, fmt.Errorf("failed to upload file: %w", err)
}
return &model.Object{
ID: id,
Name: file.GetName(),
Size: file.GetSize(),
Modified: file.ModTime(),
IsFolder: false,
}, nil
}
func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) {
parentLink, err := d.getLink(ctx, parentLinkID)
if err != nil {
return "", fmt.Errorf("failed to get parent link: %w", err)
}
// Get parent node keyring
parentNodeKR, err := d.getLinkKR(ctx, parentLink)
if err != nil {
return "", fmt.Errorf("failed to get parent keyring: %w", err)
}
// Temporary file (request)
tempReq := proton.CreateFileReq{
SignatureAddress: d.MainShare.Creator,
}
// Encrypt the filename
err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR)
if err != nil {
return "", fmt.Errorf("failed to encrypt filename: %w", err)
}
return tempReq.Name, nil
}
func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) {
parentLink, err := d.getLink(ctx, parentLinkID)
if err != nil {
return "", fmt.Errorf("failed to get parent link: %w", err)
}
// Get parent node keyring
parentNodeKR, err := d.getLinkKR(ctx, parentLink)
if err != nil {
return "", fmt.Errorf("failed to get parent keyring: %w", err)
}
signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR)
if err != nil {
return "", fmt.Errorf("failed to get signature verification keyring: %w", err)
}
parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR)
if err != nil {
return "", fmt.Errorf("failed to get parent hash key: %w", err)
}
nameHash, err := proton.GetNameHash(name, parentHashKey)
if err != nil {
return "", fmt.Errorf("failed to generate name hash: %w", err)
}
return nameHash, nil
}
func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) {
if link == nil {
return "", fmt.Errorf("link cannot be nil")
}
if link.Hash == "" {
return "", fmt.Errorf("link hash is empty")
}
return link.Hash, nil
}
func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) {
if linkID == "" {
return nil, fmt.Errorf("linkID cannot be empty")
}
link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID)
if err != nil {
return nil, err
}
return &link, nil
}
func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) {
if link == nil {
return nil, fmt.Errorf("link cannot be nil")
}
// Root Link or Root Dir
if link.ParentLinkID == "" {
signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail})
if err != nil {
return nil, err
}
return link.GetKeyRing(d.MainShareKR, signatureVerificationKR)
}
// Get parent keyring recursively
parentLink, err := d.getLink(ctx, link.ParentLinkID)
if err != nil {
return nil, err
}
parentNodeKR, err := d.getLinkKR(ctx, parentLink)
if err != nil {
return nil, err
}
signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail})
if err != nil {
return nil, err
}
return link.GetKeyRing(parentNodeKR, signatureVerificationKR)
}
var (
ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil")
ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys")
)
func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) {
user, err := c.GetUser(ctx)
if err != nil {
return nil, nil, nil, nil, err
}
// fmt.Printf("user %#v", user)
addrsArr, err := c.GetAddresses(ctx)
if err != nil {
return nil, nil, nil, nil, err
}
// fmt.Printf("addr %#v", addr)
if saltedKeyPass == nil {
if keyPass == nil {
return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil
}
// Due to limitations, salts are stored using cacheCredentialToFile
salts, err := c.GetSalts(ctx)
if err != nil {
return nil, nil, nil, nil, err
}
// fmt.Printf("salts %#v", salts)
saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID)
if err != nil {
return nil, nil, nil, nil, err
}
// fmt.Printf("saltedKeyPass ok")
}
userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil)
if err != nil {
return nil, nil, nil, nil, err
} else if userKR.CountDecryptionEntities() == 0 {
return nil, nil, nil, nil, ErrFailedToUnlockUserKeys
}
addrs := make(map[string]proton.Address)
for _, addr := range addrsArr {
addrs[addr.Email] = addr
}
return userKR, addrKRs, addrs, saltedKeyPass, nil
}
func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) {
ret, err := crypto.NewKeyRing(nil)
if err != nil {
return nil, err
}
for _, emailAddress := range emailAddresses {
if addr, ok := d.addrData[emailAddress]; ok {
if addrKR, exists := d.addrKRs[addr.ID]; exists {
err = d.addKeysFromKR(ret, addrKR)
if err != nil {
return nil, err
}
}
}
}
for _, kr := range verificationAddrKRs {
err = d.addKeysFromKR(ret, kr)
if err != nil {
return nil, err
}
}
if ret.CountEntities() == 0 {
return nil, fmt.Errorf("no keyring for signature verification")
}
return ret, nil
}
func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error {
for i := range newKRs {
for _, key := range newKRs[i].GetKeys() {
err := kr.AddKey(key)
if err != nil {
return err
}
}
}
return nil
}
func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
// fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName)
if d.MainShare == nil || d.DefaultAddrKR == nil {
return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v",
d.MainShare != nil, d.DefaultAddrKR != nil)
}
if d.protonDrive == nil {
return nil, fmt.Errorf("protonDrive bridge is nil")
}
srcLink, err := d.getLink(ctx, srcObj.GetID())
if err != nil {
return nil, fmt.Errorf("failed to find source: %w", err)
}
parentLinkID := srcLink.ParentLinkID
if parentLinkID == "" {
return nil, fmt.Errorf("cannot rename root folder")
}
encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID)
if err != nil {
return nil, fmt.Errorf("failed to encrypt filename: %w", err)
}
newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID)
if err != nil {
return nil, fmt.Errorf("failed to generate new hash: %w", err)
}
originalHash, err := d.getOriginalNameHash(srcLink)
if err != nil {
return nil, fmt.Errorf("failed to get original hash: %w", err)
}
renameReq := RenameRequest{
Name: encryptedName,
NameSignatureEmail: d.MainShare.Creator,
Hash: newHash,
OriginalHash: originalHash,
}
err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq)
if err != nil {
return nil, fmt.Errorf("rename API call failed: %w", err)
}
return &model.Object{
ID: srcLink.LinkID,
Name: newName,
Size: srcObj.GetSize(),
Modified: srcObj.ModTime(),
IsFolder: srcObj.IsDir(),
}, nil
}
func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error {
renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename",
d.MainShare.VolumeID, linkID)
reqBody, err := json.Marshal(req)
if err != nil {
return fmt.Errorf("failed to marshal rename request: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody))
if err != nil {
return fmt.Errorf("failed to create HTTP request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Accept", d.protonJson)
httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV)
httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion)
httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID)
httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken)
client := &http.Client{}
resp, err := client.Do(httpReq)
if err != nil {
return fmt.Errorf("failed to execute rename request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("rename failed with status %d", resp.StatusCode)
}
var renameResp RenameResponse
if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil {
return fmt.Errorf("failed to decode rename response: %w", err)
}
if renameResp.Code != 1000 {
return fmt.Errorf("rename failed with code %d", renameResp.Code)
}
return nil
}
func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error {
// fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name)
// fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash)
// fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash)
// fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID)
// fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name))
// fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail)
// fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash)
// fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase))
// fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature))
// fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID)
// fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID)
// fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID)
srcLink, _ := d.getLink(ctx, linkID)
if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID {
return fmt.Errorf("cannot move to same parent directory")
}
moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move",
d.MainShare.VolumeID, linkID)
reqBody, err := json.Marshal(req)
if err != nil {
return fmt.Errorf("failed to marshal move request: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody))
if err != nil {
return fmt.Errorf("failed to create HTTP request: %w", err)
}
httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken)
httpReq.Header.Set("Accept", d.protonJson)
httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV)
httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion)
httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID)
httpReq.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(httpReq)
if err != nil {
return fmt.Errorf("failed to execute move request: %w", err)
}
defer resp.Body.Close()
var moveResp RenameResponse
if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil {
return fmt.Errorf("failed to decode move response: %w", err)
}
if moveResp.Code != 1000 {
return fmt.Errorf("move operation failed with code: %d", moveResp.Code)
}
return nil
}
func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) {
// fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath())
srcLink, err := d.getLink(ctx, srcObj.GetID())
if err != nil {
return nil, fmt.Errorf("failed to find source: %w", err)
}
dstParentLinkID := dstDir.GetID()
if srcObj.IsDir() {
// Check if destination is a descendant of source
if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil {
return nil, err
}
}
// Encrypt the filename for the new location
encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID)
if err != nil {
return nil, fmt.Errorf("failed to encrypt filename: %w", err)
}
newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID)
if err != nil {
return nil, fmt.Errorf("failed to generate new hash: %w", err)
}
originalHash, err := d.getOriginalNameHash(srcLink)
if err != nil {
return nil, fmt.Errorf("failed to get original hash: %w", err)
}
// Re-encrypt node passphrase for new parent context
reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID)
if err != nil {
return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err)
}
moveReq := MoveRequest{
ParentLinkID: dstParentLinkID,
NodePassphrase: reencryptedPassphrase,
Name: encryptedName,
NameSignatureEmail: d.MainShare.Creator,
Hash: newHash,
OriginalHash: originalHash,
ContentHash: nil,
// *** Causes rejection ***
/* NodePassphraseSignature: srcLink.NodePassphraseSignature, */
}
//fmt.Printf("DEBUG MoveRequest validation:\n")
//fmt.Printf(" Name length: %d\n", len(moveReq.Name))
//fmt.Printf(" Hash: %s\n", moveReq.Hash)
//fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash)
//fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase))
/* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */
//fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail)
err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq)
if err != nil {
return nil, fmt.Errorf("move API call failed: %w", err)
}
return &model.Object{
ID: srcLink.LinkID,
Name: srcObj.GetName(),
Size: srcObj.GetSize(),
Modified: srcObj.ModTime(),
IsFolder: srcObj.IsDir(),
}, nil
}
func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) {
// Get source parent link with metadata
srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID)
if err != nil {
return "", fmt.Errorf("failed to get source parent link: %w", err)
}
// Get source parent keyring using link object
srcParentKR, err := d.getLinkKR(ctx, srcParentLink)
if err != nil {
return "", fmt.Errorf("failed to get source parent keyring: %w", err)
}
// Get destination parent link with metadata
dstParentLink, err := d.getLink(ctx, dstParentLinkID)
if err != nil {
return "", fmt.Errorf("failed to get destination parent link: %w", err)
}
// Get destination parent keyring using link object
dstParentKR, err := d.getLinkKR(ctx, dstParentLink)
if err != nil {
return "", fmt.Errorf("failed to get destination parent keyring: %w", err)
}
// Re-encrypt the node passphrase from source parent context to destination parent context
reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase)
if err != nil {
return "", fmt.Errorf("failed to re-encrypt key packet: %w", err)
}
return reencryptedPassphrase, nil
}
func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) {
parentLink, err := d.getLink(ctx, parentLinkID)
if err != nil {
return "", fmt.Errorf("failed to get parent link: %w", err)
}
// Get parent node keyring
parentNodeKR, err := d.getLinkKR(ctx, parentLink)
if err != nil {
return "", fmt.Errorf("failed to get parent keyring: %w", err)
}
// Get signature verification keyring
signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR)
if err != nil {
return "", fmt.Errorf("failed to get signature verification keyring: %w", err)
}
parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR)
if err != nil {
return "", fmt.Errorf("failed to get parent hash key: %w", err)
}
nameHash, err := proton.GetNameHash(name, parentHashKey)
if err != nil {
return "", fmt.Errorf("failed to generate name hash: %w", err)
}
return nameHash, nil
}
func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3)
oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase)
if err != nil {
return "", err
}
sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket)
if err != nil {
return "", err
}
newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey)
if err != nil {
return "", err
}
newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket)
return newSplitMessage.GetArmored()
}
func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error {
currentLinkID := dstParentLinkID
for currentLinkID != "" && currentLinkID != d.RootFolderID {
if currentLinkID == srcLinkID {
return fmt.Errorf("cannot move folder into itself or its subfolder")
}
currentLink, err := d.getLink(ctx, currentLinkID)
if err != nil {
return err
}
currentLinkID = currentLink.ParentLinkID
}
return nil
}
func (d *ProtonDrive) authHandler(auth proton.Auth) {
if auth.AccessToken != d.ReusableCredential.AccessToken || auth.RefreshToken != d.ReusableCredential.RefreshToken {
d.ReusableCredential.UID = auth.UID
d.ReusableCredential.AccessToken = auth.AccessToken
d.ReusableCredential.RefreshToken = auth.RefreshToken
if err := d.initClient(context.Background()); err != nil {
fmt.Printf("ProtonDrive: failed to reinitialize client after auth refresh: %v\n", err)
}
op.MustSaveDriverStorage(d)
}
}
func (d *ProtonDrive) initClient(ctx context.Context) error {
clientOptions := []proton.Option{
proton.WithAppVersion(d.appVersion),
proton.WithUserAgent(d.userAgent),
}
manager := proton.New(clientOptions...)
d.c = manager.NewClient(d.ReusableCredential.UID, d.ReusableCredential.AccessToken, d.ReusableCredential.RefreshToken)
saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.ReusableCredential.SaltedKeyPass)
if err != nil {
return fmt.Errorf("failed to decode salted key pass: %w", err)
}
_, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes)
if err != nil {
return fmt.Errorf("failed to get account keyrings: %w", err)
}
d.addrKRs = addrKRs
d.addrData = addrs
return nil
}

View File

@@ -69,15 +69,10 @@ func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*
Limiter: stream.ServerDownloadLimit,
Ctx: ctx,
}
if !d.Config().OnlyLinkMFile {
return &model.Link{
RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile),
SyncClosers: utils.NewSyncClosers(remoteFile),
}, nil
}
return &model.Link{
MFile: mFile,
SyncClosers: utils.NewSyncClosers(remoteFile),
RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile),
SyncClosers: utils.NewSyncClosers(remoteFile),
RequireReference: true,
}, nil
}

View File

@@ -16,12 +16,12 @@ type Addition struct {
}
var config = driver.Config{
Name: "SFTP",
LocalSort: true,
OnlyLinkMFile: false,
DefaultRoot: "/",
CheckStatus: true,
NoLinkURL: true,
Name: "SFTP",
LocalSort: true,
OnlyProxy: true,
DefaultRoot: "/",
CheckStatus: true,
NoLinkURL: true,
}
func init() {

View File

@@ -86,15 +86,10 @@ func (d *SMB) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*m
Limiter: stream.ServerDownloadLimit,
Ctx: ctx,
}
if !d.Config().OnlyLinkMFile {
return &model.Link{
RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile),
SyncClosers: utils.NewSyncClosers(remoteFile),
}, nil
}
return &model.Link{
MFile: mFile,
SyncClosers: utils.NewSyncClosers(remoteFile),
RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile),
SyncClosers: utils.NewSyncClosers(remoteFile),
RequireReference: true,
}, nil
}

View File

@@ -14,12 +14,12 @@ type Addition struct {
}
var config = driver.Config{
Name: "SMB",
LocalSort: true,
OnlyLinkMFile: false,
DefaultRoot: ".",
NoCache: true,
NoLinkURL: true,
Name: "SMB",
LocalSort: true,
OnlyProxy: true,
DefaultRoot: ".",
NoCache: true,
NoLinkURL: true,
}
func init() {

View File

@@ -12,8 +12,10 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/sign"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server/common"
log "github.com/sirupsen/logrus"
)
type Strm struct {
@@ -39,6 +41,9 @@ func (d *Strm) Init(ctx context.Context) error {
if d.Paths == "" {
return errors.New("paths is required")
}
if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 {
return errors.New("SaveStrmLocalPath is required")
}
d.pathMap = make(map[string][]string)
for _, path := range strings.Split(d.Paths, "\n") {
path = strings.TrimSpace(path)
@@ -47,6 +52,11 @@ func (d *Strm) Init(ctx context.Context) error {
}
k, v := getPair(path)
d.pathMap[k] = append(d.pathMap[k], v)
err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
if err != nil {
log.Errorf("insert strmTrie error: %v", err)
continue
}
}
if len(d.pathMap) == 1 {
for k := range d.pathMap {
@@ -58,26 +68,51 @@ func (d *Strm) Init(ctx context.Context) error {
d.autoFlatten = false
}
d.supportSuffix = supportSuffix()
if d.FilterFileTypes != "" {
types := strings.Split(d.FilterFileTypes, ",")
for _, ext := range types {
ext = strings.ToLower(strings.TrimSpace(ext))
if ext != "" {
d.supportSuffix[ext] = struct{}{}
}
var supportTypes []string
if d.FilterFileTypes == "" {
d.FilterFileTypes = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac"
}
supportTypes = strings.Split(d.FilterFileTypes, ",")
d.supportSuffix = map[string]struct{}{}
for _, ext := range supportTypes {
ext = strings.ToLower(strings.TrimSpace(ext))
if ext != "" {
d.supportSuffix[ext] = struct{}{}
}
}
d.downloadSuffix = downloadSuffix()
if d.DownloadFileTypes != "" {
downloadTypes := strings.Split(d.DownloadFileTypes, ",")
for _, ext := range downloadTypes {
ext = strings.ToLower(strings.TrimSpace(ext))
if ext != "" {
d.downloadSuffix[ext] = struct{}{}
var downloadTypes []string
if d.DownloadFileTypes == "" {
d.DownloadFileTypes = "ass,srt,vtt,sub,strm"
}
downloadTypes = strings.Split(d.DownloadFileTypes, ",")
d.downloadSuffix = map[string]struct{}{}
for _, ext := range downloadTypes {
ext = strings.ToLower(strings.TrimSpace(ext))
if ext != "" {
d.downloadSuffix[ext] = struct{}{}
}
}
if d.Version != 3 {
types := strings.Split("mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac", ",")
for _, ext := range types {
if _, ok := d.supportSuffix[ext]; !ok {
d.supportSuffix[ext] = struct{}{}
supportTypes = append(supportTypes, ext)
}
}
d.FilterFileTypes = strings.Join(supportTypes, ",")
types = strings.Split("ass,srt,vtt,sub,strm", ",")
for _, ext := range types {
if _, ok := d.downloadSuffix[ext]; !ok {
d.supportSuffix[ext] = struct{}{}
downloadTypes = append(downloadTypes, ext)
}
}
d.DownloadFileTypes = strings.Join(downloadTypes, ",")
d.Version = 3
}
return nil
}
@@ -86,6 +121,9 @@ func (d *Strm) Drop(ctx context.Context) error {
d.pathMap = nil
d.downloadSuffix = nil
d.supportSuffix = nil
for _, path := range strings.Split(d.Paths, "\n") {
RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d)
}
return nil
}
@@ -156,7 +194,7 @@ func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*
if file.GetID() == "strm" {
link := d.getLink(ctx, file.GetPath())
return &model.Link{
MFile: strings.NewReader(link),
RangeReader: stream.GetRangeReaderFromMFile(int64(len(link)), strings.NewReader(link)),
}, nil
}
// ftp,s3

164
drivers/strm/hook.go Normal file
View File

@@ -0,0 +1,164 @@
package strm
import (
"context"
"errors"
"io"
"os"
stdpath "path"
"strings"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
log "github.com/sirupsen/logrus"
"github.com/tchap/go-patricia/v2/patricia"
)
var strmTrie = patricia.NewTrie()
func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) {
path = utils.FixAndCleanPath(path)
updateLocal := func(driver *Strm, basePath string, objs []model.Obj) {
relParent := strings.TrimPrefix(basePath, driver.MountPath)
localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent)
for _, obj := range objs {
localPath := stdpath.Join(localParentPath, obj.GetName())
generateStrm(ctx, driver, obj, localPath)
}
deleteExtraFiles(localParentPath, objs)
}
_ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error {
strmDrivers := item.([]*Strm)
needPath := string(needPathPrefix)
restPath := strings.TrimPrefix(path, needPath)
if len(restPath) > 0 && restPath[0] != '/' {
return nil
}
for _, strmDriver := range strmDrivers {
strmObjs, _ := utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) {
ret := strmDriver.convert2strmObj(ctx, path, obj)
return &ret, nil
})
updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs)
}
return nil
})
}
func InsertStrm(dstPath string, d *Strm) error {
prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
existing := strmTrie.Get(prefix)
if existing == nil {
if !strmTrie.Insert(prefix, []*Strm{d}) {
return errors.New("failed to insert strm")
}
return nil
}
if lst, ok := existing.([]*Strm); ok {
strmTrie.Set(prefix, append(lst, d))
} else {
return errors.New("invalid trie item type")
}
return nil
}
func RemoveStrm(dstPath string, d *Strm) {
prefix := patricia.Prefix(strings.TrimRight(dstPath, "/"))
existing := strmTrie.Get(prefix)
if existing == nil {
return
}
lst, ok := existing.([]*Strm)
if !ok {
return
}
if len(lst) == 1 && lst[0] == d {
strmTrie.Delete(prefix)
return
}
for i, di := range lst {
if di == d {
newList := append(lst[:i], lst[i+1:]...)
strmTrie.Set(prefix, newList)
return
}
}
}
func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) {
link, err := driver.Link(ctx, obj, model.LinkArgs{})
if err != nil {
log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err)
return
}
seekableStream, err := stream.NewSeekableStream(&stream.FileStream{
Obj: obj,
Ctx: ctx,
}, link)
if err != nil {
_ = link.Close()
log.Warnf("failed to generate strm of obj %s: failed to get seekable stream: %v", localPath, err)
return
}
defer seekableStream.Close()
file, err := utils.CreateNestedFile(localPath)
if err != nil {
log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err)
return
}
defer file.Close()
if _, err := io.Copy(file, seekableStream); err != nil {
log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err)
}
}
func deleteExtraFiles(localPath string, objs []model.Obj) {
localFiles, err := getLocalFiles(localPath)
if err != nil {
log.Errorf("Failed to read local files from %s: %v", localPath, err)
return
}
objsSet := make(map[string]struct{})
for _, obj := range objs {
if obj.IsDir() {
continue
}
objsSet[stdpath.Join(localPath, obj.GetName())] = struct{}{}
}
for _, localFile := range localFiles {
if _, exists := objsSet[localFile]; !exists {
err := os.Remove(localFile)
if err != nil {
log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err)
} else {
log.Infof("Deleted file %s", localFile)
}
}
}
}
func getLocalFiles(localPath string) ([]string, error) {
var files []string
entries, err := os.ReadDir(localPath)
if err != nil {
return nil, err
}
for _, entry := range entries {
if !entry.IsDir() {
files = append(files, stdpath.Join(localPath, entry.Name()))
}
}
return files, nil
}
func init() {
op.RegisterObjsUpdateHook(UpdateLocalStrm)
}

View File

@@ -8,21 +8,23 @@ import (
type Addition struct {
Paths string `json:"paths" required:"true" type:"text"`
SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of the strm file"`
FilterFileTypes string `json:"filterFileTypes" type:"text" default:"strm" required:"false" help:"Supports suffix name of strm file"`
DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass" required:"false" help:"Files need to download with strm (usally subtitles)"`
DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Files need to download with strm (usally subtitles)"`
FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Supports suffix name of strm file"`
EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"`
LocalModel bool `json:"localModel" default:"false" help:"enable local mode"`
WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"`
SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"`
SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"`
Version int
}
var config = driver.Config{
Name: "Strm",
LocalSort: true,
NoCache: true,
NoUpload: true,
DefaultRoot: "/",
OnlyLinkMFile: true,
OnlyProxy: true,
NoLinkURL: true,
Name: "Strm",
LocalSort: true,
OnlyProxy: true,
NoCache: true,
NoUpload: true,
DefaultRoot: "/",
NoLinkURL: true,
}
func init() {

View File

@@ -1,36 +0,0 @@
package strm
func supportSuffix() map[string]struct{} {
return map[string]struct{}{
// video
"mp4": {},
"mkv": {},
"flv": {},
"avi": {},
"wmv": {},
"ts": {},
"rmvb": {},
"webm": {},
// audio
"mp3": {},
"flac": {},
"aac": {},
"wav": {},
"ogg": {},
"m4a": {},
"wma": {},
"alac": {},
}
}
func downloadSuffix() map[string]struct{} {
return map[string]struct{}{
// strm
"strm": {},
// subtitles
"ass": {},
"srt": {},
"vtt": {},
"sub": {},
}
}

View File

@@ -61,36 +61,12 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
var validObjs []model.Obj
for _, obj := range objs {
id, name, path := "", obj.GetName(), ""
size := int64(0)
if !obj.IsDir() {
path = stdpath.Join(reqPath, obj.GetName())
ext := strings.ToLower(utils.Ext(name))
if _, ok := d.supportSuffix[ext]; ok {
id = "strm"
name = strings.TrimSuffix(name, ext) + "strm"
size = int64(len(d.getLink(ctx, path)))
} else if _, ok := d.downloadSuffix[ext]; ok {
size = obj.GetSize()
} else {
continue
}
}
objRes := model.Object{
ID: id,
Path: path,
Name: name,
Size: size,
Modified: obj.ModTime(),
IsFolder: obj.IsDir(),
}
objRes := d.convert2strmObj(ctx, reqPath, obj)
thumb, ok := model.GetThumb(obj)
if !ok {
validObjs = append(validObjs, &objRes)
continue
}
validObjs = append(validObjs, &model.ObjThumb{
Object: objRes,
Thumbnail: model.Thumbnail{
@@ -101,6 +77,32 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]
return validObjs, nil
}
func (d *Strm) convert2strmObj(ctx context.Context, reqPath string, obj model.Obj) model.Object {
id, name, path := "", obj.GetName(), ""
size := int64(0)
if !obj.IsDir() {
path = stdpath.Join(reqPath, obj.GetName())
ext := strings.ToLower(utils.Ext(name))
if _, ok := d.supportSuffix[ext]; ok {
id = "strm"
name = strings.TrimSuffix(name, ext) + "strm"
size = int64(len(d.getLink(ctx, path)))
} else if _, ok := d.downloadSuffix[ext]; ok {
size = obj.GetSize()
} else {
}
}
return model.Object{
ID: id,
Path: path,
Name: name,
Size: size,
Modified: obj.ModTime(),
IsFolder: obj.IsDir(),
}
}
func (d *Strm) getLink(ctx context.Context, path string) string {
finalPath := path
if d.EncodePath {
@@ -110,7 +112,7 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
signPath := sign.Sign(path)
finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath)
}
if d.LocalModel {
if d.WithoutUrl {
return finalPath
}
apiUrl := d.SiteUrl
@@ -119,7 +121,9 @@ func (d *Strm) getLink(ctx context.Context, path string) string {
} else {
apiUrl = common.GetApiUrl(ctx)
}
if !strings.HasPrefix(finalPath, "/") {
finalPath = "/" + finalPath
}
return fmt.Sprintf("%s/d%s",
apiUrl,
finalPath)

View File

@@ -164,7 +164,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo
if err := d.singleUploadRequest(fileId, func(req *resty.Request) {
uploadParams := map[string]string{
"partName": func() string {
digits := len(fmt.Sprintf("%d", totalParts))
digits := len(strconv.Itoa(totalParts))
return file.GetName() + fmt.Sprintf(".%0*d", digits, 1)
}(),
"partNo": strconv.Itoa(1),
@@ -333,7 +333,7 @@ func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task ch
err := d.singleUploadRequest(fileId, func(req *resty.Request) {
uploadParams := map[string]string{
"partName": func() string {
digits := len(fmt.Sprintf("%d", totalParts))
digits := len(strconv.Itoa(totalParts))
return task.fileName + fmt.Sprintf(".%0*d", digits, task.chunkIdx)
}(),
"partNo": strconv.Itoa(task.chunkIdx),

View File

@@ -16,7 +16,6 @@ type Addition struct {
var config = driver.Config{
Name: "Template",
LocalSort: false,
OnlyLinkMFile: false,
OnlyProxy: false,
NoCache: false,
NoUpload: false,

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
@@ -68,6 +69,7 @@ func (x *Thunder) Init(ctx context.Context) (err error) {
PackageName: "com.xunlei.downloadprovider",
UserAgent: "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)",
DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)",
Space: x.Space,
refreshCTokenCk: func(token string) {
x.CaptchaToken = token
op.MustSaveDriverStorage(x)
@@ -167,6 +169,7 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) {
UserAgent: x.UserAgent,
DownloadUserAgent: x.DownloadUserAgent,
UseVideoUrl: x.UseVideoUrl,
Space: x.Space,
refreshCTokenCk: func(token string) {
x.CaptchaToken = token
@@ -281,7 +284,7 @@ func (xc *XunLeiCommon) Link(ctx context.Context, file model.Obj, args model.Lin
_, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) {
r.SetContext(ctx)
r.SetPathParam("fileID", file.GetID())
//r.SetQueryParam("space", "")
r.SetQueryParam("space", xc.Space)
}, &lFile)
if err != nil {
return nil, err
@@ -322,6 +325,7 @@ func (xc *XunLeiCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirNam
"kind": FOLDER,
"name": dirName,
"parent_id": parentDir.GetID(),
"space": xc.Space,
})
}, nil)
return err
@@ -331,8 +335,9 @@ func (xc *XunLeiCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) erro
_, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) {
r.SetContext(ctx)
r.SetBody(&base.Json{
"to": base.Json{"parent_id": dstDir.GetID()},
"ids": []string{srcObj.GetID()},
"to": base.Json{"parent_id": dstDir.GetID()},
"ids": []string{srcObj.GetID()},
"space": xc.Space,
})
}, nil)
return err
@@ -342,7 +347,10 @@ func (xc *XunLeiCommon) Rename(ctx context.Context, srcObj model.Obj, newName st
_, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) {
r.SetContext(ctx)
r.SetPathParam("fileID", srcObj.GetID())
r.SetBody(&base.Json{"name": newName})
r.SetBody(&base.Json{
"name": newName,
"space": xc.Space,
})
}, nil)
return err
}
@@ -351,8 +359,9 @@ func (xc *XunLeiCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) erro
_, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) {
r.SetContext(ctx)
r.SetBody(&base.Json{
"to": base.Json{"parent_id": dstDir.GetID()},
"ids": []string{srcObj.GetID()},
"to": base.Json{"parent_id": dstDir.GetID()},
"ids": []string{srcObj.GetID()},
"space": xc.Space,
})
}, nil)
return err
@@ -362,6 +371,7 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error {
_, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) {
r.SetContext(ctx)
r.SetPathParam("fileID", obj.GetID())
r.SetQueryParam("space", xc.Space)
r.SetBody("{}")
}, nil)
return err
@@ -387,6 +397,7 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi
"size": file.GetSize(),
"hash": gcid,
"upload_type": UPLOAD_TYPE_RESUMABLE,
"space": xc.Space,
})
}, &resp)
if err != nil {
@@ -430,7 +441,7 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.
_, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) {
r.SetContext(ctx)
r.SetQueryParams(map[string]string{
"space": "",
"space": xc.Space,
"__type": "drive",
"refresh": "true",
"__sync": "true",
@@ -440,6 +451,17 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.
"limit": "100",
"filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`,
})
// 获取硬盘挂载目录等
if xc.Space != "" {
r.SetQueryParamsFromValues(url.Values{
"with": []string{
"withCategoryDiskMountPath",
"withCategoryDriveCachePath",
"withCategoryHistoryDownloadPath",
"withReadOnlyFS",
},
})
}
}, &fileList)
if err != nil {
return nil, err
@@ -576,6 +598,7 @@ func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, par
"name": fileName,
"parent_id": parentDir.GetID(),
"upload_type": UPLOAD_TYPE_URL,
"space": xc.Space,
"url": base.Json{
"url": fileUrl,
},
@@ -602,6 +625,7 @@ func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) (
"type": "offline",
"limit": "10000",
"page_token": nextPageToken,
"space": xc.Space,
})
}, &resp)
@@ -618,6 +642,7 @@ func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string
SetQueryParams(map[string]string{
"task_ids": strings.Join(taskIDs, ","),
"delete_files": strconv.FormatBool(deleteFiles),
"space": xc.Space,
})
}, nil)
if err != nil {

View File

@@ -46,6 +46,8 @@ type ExpertAddition struct {
//优先使用视频链接代替下载链接
UseVideoUrl bool `json:"use_video_url"`
Space string `json:"space" default:"" help:"device id for remote device"`
}
// 登录特征,用于判断是否重新登录
@@ -80,6 +82,8 @@ type Addition struct {
CreditKey string `json:"credit_key" help:"credit key,used for login"`
// 登录设备ID
DeviceID string `json:"device_id" default:""`
Space string `json:"space" default:"" help:"device id for remote device"`
}
// 登录特征,用于判断是否重新登录
@@ -90,7 +94,6 @@ func (i *Addition) GetIdentity() string {
var config = driver.Config{
Name: "Thunder",
LocalSort: true,
OnlyProxy: true,
}
var configExpert = driver.Config{

View File

@@ -68,6 +68,7 @@ type Common struct {
UserAgent string
DownloadUserAgent string
UseVideoUrl bool
Space string
// 验证码token刷新成功回调
refreshCTokenCk func(token string)

View File

@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
@@ -840,7 +841,7 @@ func (xc *XunLeiBrowserCommon) OfflineList(ctx context.Context, nextPageToken st
func (xc *XunLeiBrowserCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string) error {
queryParams := map[string]string{
"task_ids": strings.Join(taskIDs, ","),
"_t": fmt.Sprintf("%d", time.Now().UnixMilli()),
"_t": strconv.FormatInt(time.Now().UnixMilli(), 10),
}
if xc.UseFluentPlay {
queryParams["space"] = ThunderBrowserDriveFluentPlayFolderType

View File

@@ -2,11 +2,11 @@ package virtual
import (
"context"
"io"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils/random"
)
@@ -42,16 +42,14 @@ func (d *Virtual) List(ctx context.Context, dir model.Obj, args model.ListArgs)
return res, nil
}
type DummyMFile struct {
io.Reader
}
type DummyMFile struct{}
func (f DummyMFile) Read(p []byte) (n int, err error) {
return f.Reader.Read(p)
return random.Rand.Read(p)
}
func (f DummyMFile) ReadAt(p []byte, off int64) (n int, err error) {
return f.Reader.Read(p)
return random.Rand.Read(p)
}
func (DummyMFile) Seek(offset int64, whence int) (int64, error) {
@@ -60,7 +58,7 @@ func (DummyMFile) Seek(offset int64, whence int) (int64, error) {
func (d *Virtual) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
return &model.Link{
MFile: DummyMFile{Reader: random.Rand},
RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), DummyMFile{}),
}, nil
}

View File

@@ -14,11 +14,11 @@ type Addition struct {
}
var config = driver.Config{
Name: "Virtual",
OnlyLinkMFile: true,
LocalSort: true,
NeedMs: true,
NoLinkURL: true,
Name: "Virtual",
LocalSort: true,
OnlyProxy: true,
NeedMs: true,
NoLinkURL: true,
}
func init() {

27
go.mod
View File

@@ -38,6 +38,7 @@ require (
github.com/golang-jwt/jwt/v4 v4.5.2
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499
github.com/hekmon/transmissionrpc/v3 v3.0.0
github.com/ipfs/go-ipfs-api v0.7.0
github.com/itsHenry35/gofakes3 v0.0.8
@@ -52,6 +53,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/pkg/sftp v1.13.9
github.com/pquerna/otp v1.5.0
github.com/quic-go/quic-go v0.54.1
github.com/rclone/rclone v1.70.3
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d
github.com/shirou/gopsutil/v4 v4.25.5
@@ -82,12 +84,24 @@ require (
require (
cloud.google.com/go/compute/metadata v0.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
github.com/ProtonMail/go-srp v0.0.7 // indirect
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
github.com/PuerkitoBio/goquery v1.10.3 // indirect
github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
github.com/ebitengine/purego v0.8.4 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
github.com/henrybear327/go-proton-api v1.0.0 // indirect
github.com/geoffgarside/ber v1.2.0 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -102,7 +116,13 @@ require (
github.com/minio/xxml v0.0.3 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/otiai10/mint v1.6.3 // indirect
github.com/relvacode/iso8601 v1.6.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.uber.org/mock v0.5.0 // indirect
golang.org/x/mod v0.27.0 // indirect
gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
)
@@ -194,6 +214,7 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/henrybear327/Proton-API-Bridge v1.0.0
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/ipfs/go-cid v0.5.0
github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -267,7 +288,7 @@ require (
golang.org/x/sys v0.34.0
golang.org/x/term v0.33.0 // indirect
golang.org/x/text v0.27.0
golang.org/x/tools v0.34.0 // indirect
golang.org/x/tools v0.35.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.6 // indirect
@@ -277,4 +298,8 @@ require (
lukechampine.com/blake3 v1.1.7 // indirect
)
replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0
replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed
// replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go

56
go.sum
View File

@@ -37,6 +37,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk=
github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE=
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc=
github.com/OpenListTeam/115-sdk-go v0.2.2 h1:JCrGHqQjBX3laOA6Hw4CuBovSg7g+FC5s0LEAYsRciU=
@@ -53,8 +55,22 @@ github.com/OpenListTeam/times v0.1.0 h1:qknxw+qj5CYKgXAwydA102UEpPcpU8TYNGRmwRyP
github.com/OpenListTeam/times v0.1.0/go.mod h1:Jx7qen5NCYzKk2w14YuvU48YYMcPa1P9a+EJePC15Pc=
github.com/OpenListTeam/wopan-sdk-go v0.1.5 h1:iKKcVzIqBgtGDbn0QbdWrCazSGxXFmYFyrnFBG+U8dI=
github.com/OpenListTeam/wopan-sdk-go v0.1.5/go.mod h1:otynv0CgSNUClPpUgZ44qCZGcMRe0dc83Pkk65xAunI=
github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo=
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk=
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo=
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI=
github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk=
github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE=
github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s=
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg=
github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0=
github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg=
@@ -72,6 +88,8 @@ github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAP
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ=
github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
@@ -164,6 +182,9 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0=
github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
@@ -198,6 +219,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA=
github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg=
@@ -239,6 +261,10 @@ github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJL
github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8=
github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg=
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
@@ -364,6 +390,10 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006100629-ba7a40dce261 h1:47L8SHM80cXszQydLrpp9MhVkFLLWCvrU9XmJ6XtRu0=
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006100629-ba7a40dce261/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM=
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499 h1:4ovnBdiGDFi8putQGxhipuuhXItAgh4/YnzufPYkZkQ=
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -385,6 +415,12 @@ github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI
github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M=
github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ=
github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg=
github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0=
github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts=
github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw=
github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc=
github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -585,8 +621,14 @@ github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQP
github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE=
github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo=
github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU=
github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -702,6 +744,8 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs=
@@ -714,6 +758,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
@@ -756,6 +801,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -775,10 +822,12 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -821,6 +870,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -839,6 +889,7 @@ golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXct
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
@@ -854,6 +905,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
@@ -895,8 +947,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=
golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@@ -39,7 +39,21 @@ func InitConfig() {
if !filepath.IsAbs(dataDir) {
flags.DataDir = filepath.Join(pwd, flags.DataDir)
}
configPath := filepath.Join(flags.DataDir, "config.json")
// Determine config file path: use flags.ConfigPath if provided, otherwise default to <dataDir>/config.json
configPath := flags.ConfigPath
if configPath == "" {
configPath = filepath.Join(flags.DataDir, "config.json")
} else {
// if relative, resolve relative to working directory
if !filepath.IsAbs(configPath) {
if absPath, err := filepath.Abs(configPath); err == nil {
configPath = absPath
} else {
configPath = filepath.Join(pwd, configPath)
}
}
}
configPath = filepath.Clean(configPath)
log.Infof("reading config file: %s", configPath)
if !utils.Exists(configPath) {
log.Infof("config file not exists, creating default config file")
@@ -126,6 +140,10 @@ func InitConfig() {
log.Fatalf("create temp dir error: %+v", err)
}
log.Debugf("config: %+v", conf.Conf)
// Validate and display proxy configuration status
validateProxyConfig()
base.InitClient()
initURL()
}
@@ -165,3 +183,14 @@ func CleanTempDir() {
}
}
}
// validateProxyConfig validates proxy configuration and displays status at startup
func validateProxyConfig() {
if conf.Conf.ProxyAddress != "" {
if _, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
log.Infof("Proxy enabled: %s", conf.Conf.ProxyAddress)
} else {
log.Errorf("Invalid proxy address format: %s, error: %v", conf.Conf.ProxyAddress, err)
}
}
}

View File

@@ -148,7 +148,10 @@ func InitialSettings() []model.SettingItem {
{Key: "audio_cover", Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW},
{Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
{Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
{Key: conf.PreviewDownloadByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW},
{Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
{Key: conf.SharePreviewDownloadByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
{Key: conf.SharePreviewArchivesByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW},
{Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
{Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
// global settings

View File

@@ -25,6 +25,6 @@ func LoadStorages() {
storages[i].MountPath, storages[i].Driver, storages[i].Order)
}
}
conf.StoragesLoaded = true
conf.SendStoragesLoadedSignal()
}(storages)
}

101
internal/cache/keyed_cache.go vendored Normal file
View File

@@ -0,0 +1,101 @@
package cache
import (
"sync"
"time"
)
type KeyedCache[T any] struct {
entries map[string]*CacheEntry[T]
mu sync.RWMutex
ttl time.Duration
}
func NewKeyedCache[T any](ttl time.Duration) *KeyedCache[T] {
c := &KeyedCache[T]{
entries: make(map[string]*CacheEntry[T]),
ttl: ttl,
}
gcFuncs = append(gcFuncs, c.GC)
return c
}
func (c *KeyedCache[T]) Set(key string, value T) {
c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(c.ttl)))
}
func (c *KeyedCache[T]) SetWithTTL(key string, value T, ttl time.Duration) {
c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(ttl)))
}
func (c *KeyedCache[T]) SetWithExpirable(key string, value T, exp Expirable) {
c.mu.Lock()
defer c.mu.Unlock()
c.entries[key] = &CacheEntry[T]{
data: value,
Expirable: exp,
}
}
func (c *KeyedCache[T]) Get(key string) (T, bool) {
c.mu.RLock()
entry, exists := c.entries[key]
if !exists {
c.mu.RUnlock()
return *new(T), false
}
expired := entry.Expired()
c.mu.RUnlock()
if !expired {
return entry.data, true
}
c.mu.Lock()
if c.entries[key] == entry {
delete(c.entries, key)
c.mu.Unlock()
return *new(T), false
}
c.mu.Unlock()
return *new(T), false
}
func (c *KeyedCache[T]) Delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
delete(c.entries, key)
}
func (c *KeyedCache[T]) Take(key string) (T, bool) {
c.mu.Lock()
defer c.mu.Unlock()
if entry, exists := c.entries[key]; exists {
delete(c.entries, key)
return entry.data, true
}
return *new(T), false
}
func (c *KeyedCache[T]) Clear() {
c.mu.Lock()
defer c.mu.Unlock()
c.entries = make(map[string]*CacheEntry[T])
}
func (c *KeyedCache[T]) GC() {
c.mu.Lock()
defer c.mu.Unlock()
expiredKeys := make([]string, 0, len(c.entries))
for key, entry := range c.entries {
if entry.Expired() {
expiredKeys = append(expiredKeys, key)
}
}
for _, key := range expiredKeys {
delete(c.entries, key)
}
}

18
internal/cache/type.go vendored Normal file
View File

@@ -0,0 +1,18 @@
package cache
import "time"
type Expirable interface {
Expired() bool
}
type ExpirationTime time.Time
func (e ExpirationTime) Expired() bool {
return time.Now().After(time.Time(e))
}
type CacheEntry[T any] struct {
Expirable
data T
}

113
internal/cache/typed_cache.go vendored Normal file
View File

@@ -0,0 +1,113 @@
package cache
import (
"sync"
"time"
)
type TypedCache[T any] struct {
entries map[string]map[string]*CacheEntry[T]
mu sync.RWMutex
ttl time.Duration
}
func NewTypedCache[T any](ttl time.Duration) *TypedCache[T] {
c := &TypedCache[T]{
entries: make(map[string]map[string]*CacheEntry[T]),
ttl: ttl,
}
gcFuncs = append(gcFuncs, c.GC)
return c
}
func (c *TypedCache[T]) SetType(key, typeKey string, value T) {
c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(c.ttl)))
}
func (c *TypedCache[T]) SetTypeWithTTL(key, typeKey string, value T, ttl time.Duration) {
c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(ttl)))
}
func (c *TypedCache[T]) SetTypeWithExpirable(key, typeKey string, value T, exp Expirable) {
c.mu.Lock()
defer c.mu.Unlock()
cache, exists := c.entries[key]
if !exists {
cache = make(map[string]*CacheEntry[T])
c.entries[key] = cache
}
cache[typeKey] = &CacheEntry[T]{
data: value,
Expirable: exp,
}
}
func (c *TypedCache[T]) GetType(key, typeKey string) (T, bool) {
c.mu.RLock()
cache, exists := c.entries[key]
if !exists {
c.mu.RUnlock()
return *new(T), false
}
entry, exists := cache[typeKey]
if !exists {
c.mu.RUnlock()
return *new(T), false
}
expired := entry.Expired()
c.mu.RUnlock()
if !expired {
return entry.data, true
}
c.mu.Lock()
if cache[typeKey] == entry {
delete(cache, typeKey)
if len(cache) == 0 {
delete(c.entries, key)
}
c.mu.Unlock()
return *new(T), false
}
c.mu.Unlock()
return *new(T), false
}
func (c *TypedCache[T]) DeleteKey(key string) {
c.mu.Lock()
defer c.mu.Unlock()
delete(c.entries, key)
}
func (c *TypedCache[T]) Clear() {
c.mu.Lock()
defer c.mu.Unlock()
c.entries = make(map[string]map[string]*CacheEntry[T])
}
func (c *TypedCache[T]) GC() {
c.mu.Lock()
defer c.mu.Unlock()
expiredKeys := make(map[string][]string)
for tk, entries := range c.entries {
for key, entry := range entries {
if !entry.Expired() {
continue
}
if _, ok := expiredKeys[tk]; !ok {
expiredKeys[tk] = make([]string, 0, len(entries))
}
expiredKeys[tk] = append(expiredKeys[tk], key)
}
}
for tk, keys := range expiredKeys {
for _, key := range keys {
delete(c.entries[tk], key)
}
if len(c.entries[tk]) == 0 {
delete(c.entries, tk)
}
}
}

24
internal/cache/utils.go vendored Normal file
View File

@@ -0,0 +1,24 @@
package cache
import (
"time"
"github.com/OpenListTeam/OpenList/v4/pkg/cron"
log "github.com/sirupsen/logrus"
)
var (
cacheGcCron *cron.Cron
gcFuncs []func()
)
func init() {
// TODO Move to bootstrap
cacheGcCron = cron.NewCron(time.Hour)
cacheGcCron.Do(func() {
log.Infof("Start cache GC")
for _, f := range gcFuncs {
f()
}
})
}

View File

@@ -35,6 +35,7 @@ type Scheme struct {
UnixFile string `json:"unix_file" env:"UNIX_FILE"`
UnixFilePerm string `json:"unix_file_perm" env:"UNIX_FILE_PERM"`
EnableH2c bool `json:"enable_h2c" env:"ENABLE_H2C"`
EnableH3 bool `json:"enable_h3" env:"ENABLE_H3"`
}
type LogConfig struct {
@@ -130,6 +131,7 @@ type Config struct {
FTP FTP `json:"ftp" envPrefix:"FTP_"`
SFTP SFTP `json:"sftp" envPrefix:"SFTP_"`
LastLaunchedVersion string `json:"last_launched_version"`
ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"`
}
func DefaultConfig(dataDir string) *Config {
@@ -243,5 +245,6 @@ func DefaultConfig(dataDir string) *Config {
Listen: ":5222",
},
LastLaunchedVersion: "",
ProxyAddress: "",
}
}

View File

@@ -24,17 +24,20 @@ const (
HideStorageDetailsInManagePage = "hide_storage_details_in_manage_page"
// preview
TextTypes = "text_types"
AudioTypes = "audio_types"
VideoTypes = "video_types"
ImageTypes = "image_types"
ProxyTypes = "proxy_types"
ProxyIgnoreHeaders = "proxy_ignore_headers"
AudioAutoplay = "audio_autoplay"
VideoAutoplay = "video_autoplay"
PreviewArchivesByDefault = "preview_archives_by_default"
ReadMeAutoRender = "readme_autorender"
FilterReadMeScripts = "filter_readme_scripts"
TextTypes = "text_types"
AudioTypes = "audio_types"
VideoTypes = "video_types"
ImageTypes = "image_types"
ProxyTypes = "proxy_types"
ProxyIgnoreHeaders = "proxy_ignore_headers"
AudioAutoplay = "audio_autoplay"
VideoAutoplay = "video_autoplay"
PreviewDownloadByDefault = "preview_download_by_default"
PreviewArchivesByDefault = "preview_archives_by_default"
SharePreviewDownloadByDefault = "share_preview_download_by_default"
SharePreviewArchivesByDefault = "share_preview_archives_by_default"
ReadMeAutoRender = "readme_autorender"
FilterReadMeScripts = "filter_readme_scripts"
// global
HideFiles = "hide_files"

View File

@@ -3,6 +3,7 @@ package conf
import (
"net/url"
"regexp"
"sync"
)
var (
@@ -23,8 +24,6 @@ var FilenameCharMap = make(map[string]string)
var PrivacyReg []*regexp.Regexp
var (
// StoragesLoaded loaded success if empty
StoragesLoaded = false
// 单个Buffer最大限制
MaxBufferLimit = 16 * 1024 * 1024
// 超过该阈值的Buffer将使用 mmap 分配,可主动释放内存
@@ -35,3 +34,39 @@ var (
ManageHtml string
IndexHtml string
)
var (
// StoragesLoaded loaded success if empty
StoragesLoaded = false
storagesLoadMu sync.RWMutex
storagesLoadSignal chan struct{} = make(chan struct{})
)
func StoragesLoadSignal() <-chan struct{} {
storagesLoadMu.RLock()
ch := storagesLoadSignal
storagesLoadMu.RUnlock()
return ch
}
func SendStoragesLoadedSignal() {
storagesLoadMu.Lock()
select {
case <-storagesLoadSignal:
// already closed
default:
StoragesLoaded = true
close(storagesLoadSignal)
}
storagesLoadMu.Unlock()
}
func ResetStoragesLoadSignal() {
storagesLoadMu.Lock()
select {
case <-storagesLoadSignal:
StoragesLoaded = false
storagesLoadSignal = make(chan struct{})
default:
// not closed -> nothing to do
}
storagesLoadMu.Unlock()
}

View File

@@ -60,3 +60,7 @@ func DeleteSharingById(id string) error {
s := model.SharingDB{ID: id}
return errors.WithStack(db.Where(s).Delete(&s).Error)
}
func DeleteSharingsByCreatorId(creatorId uint) error {
return errors.WithStack(db.Where("creator_id = ?", creatorId).Delete(&model.SharingDB{}).Error)
}

View File

@@ -1,6 +1,7 @@
package db
import (
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/pkg/errors"
)
@@ -30,6 +31,7 @@ func GetTaskDataFunc(type_s string, enabled bool) func() ([]byte, error) {
return nil
}
return func() ([]byte, error) {
<-conf.StoragesLoadSignal()
return []byte(task.PersistData), nil
}
}

View File

@@ -3,11 +3,9 @@ package driver
type Config struct {
Name string `json:"name"`
LocalSort bool `json:"local_sort"`
// if the driver returns Link with MFile, this should be set to true
OnlyLinkMFile bool `json:"only_local"`
OnlyProxy bool `json:"only_proxy"`
NoCache bool `json:"no_cache"`
NoUpload bool `json:"no_upload"`
OnlyProxy bool `json:"only_proxy"`
NoCache bool `json:"no_cache"`
NoUpload bool `json:"no_upload"`
// if need get message from user, such as validate code
NeedMs bool `json:"need_ms"`
DefaultRoot string `json:"default_root"`
@@ -19,8 +17,24 @@ type Config struct {
ProxyRangeOption bool `json:"-"`
// if the driver returns Link without URL, this should be set to true
NoLinkURL bool `json:"-"`
// Link cache behaviour:
// - LinkCacheAuto: let driver decide per-path (implement driver.LinkCacheModeResolver)
// - LinkCacheNone: no extra info added to cache key (default)
// - flags (OR-able) can add more attributes to cache key (IP, UA, ...)
LinkCacheMode `json:"-"`
}
type LinkCacheMode int8
const (
LinkCacheAuto LinkCacheMode = -1 // Let the driver decide per-path (use driver.LinkCacheModeResolver)
LinkCacheNone LinkCacheMode = 0 // No extra info added to cache key (default)
)
const (
LinkCacheIP LinkCacheMode = 1 << iota // include client IP in cache key
LinkCacheUA // include User-Agent in cache key
)
func (c Config) MustProxy() bool {
return c.OnlyProxy || c.OnlyLinkMFile || c.NoLinkURL
return c.OnlyProxy || c.NoLinkURL
}

View File

@@ -47,11 +47,6 @@ type Getter interface {
Get(ctx context.Context, path string) (model.Obj, error)
}
type GetObjInfo interface {
// GetObjInfo get file info by path
GetObjInfo(ctx context.Context, path string) (model.Obj, error)
}
//type Writer interface {
// Mkdir
// Move
@@ -218,3 +213,8 @@ type WithDetails interface {
type Reference interface {
InitReference(storage Driver) error
}
type LinkCacheModeResolver interface {
// ResolveLinkCacheMode returns the LinkCacheMode for the given path.
ResolveLinkCacheMode(path string) LinkCacheMode
}

View File

@@ -1,12 +1,11 @@
package errs
import "errors"
func UnwrapOrSelf(err error) error {
// errors.Unwrap has no fallback mechanism
unwrapped := errors.Unwrap(err)
if unwrapped == nil {
u, ok := err.(interface {
Unwrap() error
})
if !ok {
return err
}
return unwrapped
return u.Unwrap()
}

View File

@@ -41,6 +41,18 @@ func (t *ArchiveDownloadTask) Run() error {
if err := t.ReinitCtx(); err != nil {
return err
}
if t.SrcStorage == nil {
if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil {
t.SrcStorage = srcStorage
} else {
return err
}
if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil {
t.DstStorage = dstStorage
} else {
return err
}
}
t.ClearEndTime()
t.SetStartTime(time.Now())
defer func() { t.SetEndTime(time.Now()) }()

View File

@@ -48,6 +48,19 @@ func (t *FileTransferTask) Run() error {
if err := t.ReinitCtx(); err != nil {
return err
}
if t.SrcStorage == nil {
if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil {
t.SrcStorage = srcStorage
} else {
return err
}
if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil {
t.DstStorage = dstStorage
} else {
return err
}
}
t.ClearEndTime()
t.SetStartTime(time.Now())
defer func() { t.SetEndTime(time.Now()) }()
@@ -139,7 +152,7 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str
if taskType == move {
task_group.RefreshAndRemove(dstDirPath, task_group.SrcPathToRemove(srcObjPath))
} else {
op.DeleteCache(t.DstStorage, dstDirActualPath)
op.Cache.DeleteDirectory(t.DstStorage, dstDirActualPath)
}
}
return nil, err
@@ -173,7 +186,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer
dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName())
if t.TaskType == copy {
if t.Ctx().Value(conf.NoTaskKey) != nil {
defer op.DeleteCache(t.DstStorage, dstActualPath)
defer op.Cache.DeleteDirectory(t.DstStorage, dstActualPath)
} else {
task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(dstActualPath))
}

View File

@@ -15,7 +15,7 @@ func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) {
path = utils.FixAndCleanPath(path)
// maybe a virtual file
if path != "/" {
virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, stdpath.Dir(path), !args.WithStorageDetails)
virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, stdpath.Dir(path), !args.WithStorageDetails, false)
for _, f := range virtualFiles {
if f.GetName() == stdpath.Base(path) {
return f, nil

View File

@@ -15,7 +15,7 @@ import (
func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) {
meta, _ := ctx.Value(conf.MetaKey).(*model.Meta)
user, _ := ctx.Value(conf.UserKey).(*model.User)
virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails)
virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails, args.Refresh)
storage, actualPath, err := op.GetStorageAndActualPath(path)
if err != nil && len(virtualFiles) == 0 {
return nil, errors.WithMessage(err, "failed get storage")

View File

@@ -28,7 +28,6 @@ type Link struct {
URL string `json:"url"` // most common way
Header http.Header `json:"header"` // needed header (for url)
RangeReader RangeReaderIF `json:"-"` // recommended way if can't use URL
MFile File `json:"-"` // best for local,smb... file system, which exposes MFile
Expiration *time.Duration // local cache expire Duration
@@ -38,6 +37,8 @@ type Link struct {
ContentLength int64 `json:"-"` // 转码视频、缩略图
utils.SyncClosers `json:"-"`
// 如果SyncClosers中的资源被关闭后Link将不可用则此值应为 true
RequireReference bool `json:"-"`
}
type OtherArgs struct {

View File

@@ -33,7 +33,7 @@ func (s *Sharing) Valid() bool {
if len(s.Files) == 0 {
return false
}
if !s.Creator.CanShare() {
if s.Creator == nil || !s.Creator.CanShare() {
return false
}
if s.Expires != nil && !s.Expires.IsZero() && s.Expires.Before(time.Now()) {

View File

@@ -12,6 +12,7 @@ import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/rclone/rclone/lib/mmap"
@@ -403,7 +404,7 @@ var errInfiniteRetry = errors.New("infinite retry")
func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) {
resp, err := d.cfg.HttpClient(d.ctx, params)
if err != nil {
statusCode, ok := errors.Unwrap(err).(HttpStatusCodeError)
statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError)
if !ok {
return 0, err
}

View File

@@ -283,11 +283,15 @@ func HttpClient() *http.Client {
}
func NewHttpClient() *http.Client {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify},
}
SetProxyIfConfigured(transport)
return &http.Client{
Timeout: time.Hour * 48,
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify},
},
Timeout: time.Hour * 48,
Transport: transport,
}
}

View File

@@ -7,12 +7,15 @@ import (
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/go-resty/resty/v2"
log "github.com/sirupsen/logrus"
)
@@ -350,3 +353,23 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea
// return an io.ReadCloser that is limited to `length` bytes.
return &LimitedReadCloser{readCloser, length_int}, nil
}
// SetProxyIfConfigured sets proxy for HTTP Transport if configured
func SetProxyIfConfigured(transport *http.Transport) {
// If proxy address is configured, override environment variable settings
if conf.Conf.ProxyAddress != "" {
if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
transport.Proxy = http.ProxyURL(proxyURL)
}
}
}
// SetRestyProxyIfConfigured sets proxy for Resty client if configured
func SetRestyProxyIfConfigured(client *resty.Client) {
// If proxy address is configured, override environment variable settings
if conf.Conf.ProxyAddress != "" {
if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
client.SetProxy(proxyURL.String())
}
}
}

View File

@@ -34,6 +34,20 @@ func (t *TransferTask) Run() error {
if err := t.ReinitCtx(); err != nil {
return err
}
if t.SrcStorage == nil && t.SrcStorageMp != "" {
if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil {
t.SrcStorage = srcStorage
} else {
return err
}
if t.DstStorage == nil {
if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil {
t.DstStorage = dstStorage
} else {
return err
}
}
}
t.ClearEndTime()
t.SetStartTime(time.Now())
defer func() { t.SetEndTime(time.Now()) }()
@@ -64,9 +78,8 @@ func (t *TransferTask) Run() error {
return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, s, t.SetProgress)
}
return transferStdPath(t)
} else {
return transferObjPath(t)
}
return transferObjPath(t)
}
func (t *TransferTask) GetName() string {

View File

@@ -10,6 +10,7 @@ import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/archive/tool"
"github.com/OpenListTeam/OpenList/v4/internal/cache"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
@@ -17,12 +18,12 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/singleflight"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/go-cache"
gocache "github.com/OpenListTeam/go-cache"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
var archiveMetaCache = cache.NewMemCache(cache.WithShards[*model.ArchiveMetaProvider](64))
var archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) {
@@ -37,14 +38,14 @@ func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg
return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err)
}
if m.Expiration != nil {
archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](*m.Expiration))
archiveMetaCache.Set(key, m, gocache.WithEx[*model.ArchiveMetaProvider](*m.Expiration))
}
return m, nil
}
if storage.Config().OnlyLinkMFile {
meta, err := fn()
return meta, err
}
// if storage.Config().NoLinkSingleflight {
// meta, err := fn()
// return meta, err
// }
if !args.Refresh {
if meta, ok := archiveMetaCache.Get(key); ok {
log.Debugf("use cache when get %s archive meta", path)
@@ -158,7 +159,7 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg
return obj, archiveMetaProvider, err
}
var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
var archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
var archiveListG singleflight.Group[[]model.Obj]
func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) {
@@ -199,7 +200,7 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)
archiveListCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
archiveListCache.Set(key, files, gocache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
} else {
log.Debugf("del cache: %s", key)
archiveListCache.Del(key)
@@ -354,75 +355,50 @@ func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args mo
return nil, nil, errors.WithStack(errs.ObjectNotFound)
}
type extractLink struct {
*model.Link
Obj model.Obj
type objWithLink struct {
link *model.Link
obj model.Obj
}
var extractCache = cache.NewMemCache(cache.WithShards[*extractLink](16))
var extractG = singleflight.Group[*extractLink]{Remember: true}
var extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
var extractG = singleflight.Group[*objWithLink]{}
func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status)
}
key := stdpath.Join(Key(storage, path), args.InnerPath)
if link, ok := extractCache.Get(key); ok {
return link.Link, link.Obj, nil
if ol, ok := extractCache.Get(key); ok {
if ol.link.Expiration != nil || ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference {
return ol.link, ol.obj, nil
}
}
var forget any
var linkM *extractLink
fn := func() (*extractLink, error) {
link, err := driverExtract(ctx, storage, path, args)
fn := func() (*objWithLink, error) {
ol, err := driverExtract(ctx, storage, path, args)
if err != nil {
return nil, errors.Wrapf(err, "failed extract archive")
}
if link.MFile != nil && forget != nil {
linkM = link
return nil, errLinkMFileCache
if ol.link.Expiration != nil {
extractCache.SetWithTTL(key, ol, *ol.link.Expiration)
} else {
extractCache.SetWithExpirable(key, ol, &ol.link.SyncClosers)
}
if link.Link.Expiration != nil {
extractCache.Set(key, link, cache.WithEx[*extractLink](*link.Link.Expiration))
}
link.AddIfCloser(forget)
return link, nil
return ol, nil
}
if storage.Config().OnlyLinkMFile {
link, err := fn()
for {
ol, err, _ := extractG.Do(key, fn)
if err != nil {
return nil, nil, err
}
return link.Link, link.Obj, nil
}
forget = utils.CloseFunc(func() error {
if forget != nil {
forget = nil
linkG.Forget(key)
if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference {
return ol.link, ol.obj, nil
}
return nil
})
link, err, _ := extractG.Do(key, fn)
for err == nil && !link.AcquireReference() {
link, err, _ = extractG.Do(key, fn)
}
if err == errLinkMFileCache {
if linkM != nil {
return linkM.Link, linkM.Obj, nil
}
forget = nil
link, err = fn()
}
if err != nil {
return nil, nil, err
}
return link.Link, link.Obj, nil
}
func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*extractLink, error) {
func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*objWithLink, error) {
storageAr, ok := storage.(driver.ArchiveReader)
if !ok {
return nil, errs.DriverExtractNotSupported
@@ -438,7 +414,7 @@ func driverExtract(ctx context.Context, storage driver.Driver, path string, args
return nil, errors.WithStack(errs.NotFile)
}
link, err := storageAr.Extract(ctx, archiveFile, args)
return &extractLink{Link: link, Obj: extracted}, err
return &objWithLink{link: link, obj: extracted}, err
}
type streamWithParent struct {
@@ -500,16 +476,16 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
if err == nil {
if len(newObjs) > 0 {
for _, newObj := range newObjs {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
Cache.addDirectoryObject(storage, dstDirPath, model.WrapObjName(newObj))
}
} else if !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
Cache.DeleteDirectory(storage, dstDirPath)
}
}
case driver.ArchiveDecompress:
err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
if err == nil && !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
Cache.DeleteDirectory(storage, dstDirPath)
}
default:
return errs.NotImplement

257
internal/op/cache.go Normal file
View File

@@ -0,0 +1,257 @@
package op
import (
stdpath "path"
"sync"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/cache"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
type CacheManager struct {
dirCache *cache.KeyedCache[*directoryCache] // Cache for directory listings
linkCache *cache.TypedCache[*objWithLink] // Cache for file links
userCache *cache.KeyedCache[*model.User] // Cache for user data
settingCache *cache.KeyedCache[any] // Cache for settings
detailCache *cache.KeyedCache[*model.StorageDetails] // Cache for storage details
}
func NewCacheManager() *CacheManager {
return &CacheManager{
dirCache: cache.NewKeyedCache[*directoryCache](time.Minute * 5),
linkCache: cache.NewTypedCache[*objWithLink](time.Minute * 30),
userCache: cache.NewKeyedCache[*model.User](time.Hour),
settingCache: cache.NewKeyedCache[any](time.Hour),
detailCache: cache.NewKeyedCache[*model.StorageDetails](time.Minute * 30),
}
}
// global instance
var Cache = NewCacheManager()
func Key(storage driver.Driver, path string) string {
return stdpath.Join(storage.GetStorage().MountPath, path)
}
// update object in dirCache.
// if it's a directory, remove all its children from dirCache too.
// if it's a file, remove its link from linkCache.
func (cm *CacheManager) updateDirectoryObject(storage driver.Driver, dirPath string, oldObj model.Obj, newObj model.Obj) {
key := Key(storage, dirPath)
if !oldObj.IsDir() {
cm.linkCache.DeleteKey(stdpath.Join(key, oldObj.GetName()))
cm.linkCache.DeleteKey(stdpath.Join(key, newObj.GetName()))
}
if storage.Config().NoCache {
return
}
if cache, exist := cm.dirCache.Get(key); exist {
if oldObj.IsDir() {
cm.deleteDirectoryTree(stdpath.Join(key, oldObj.GetName()))
}
cache.UpdateObject(oldObj.GetName(), newObj)
}
}
// add new object to dirCache
func (cm *CacheManager) addDirectoryObject(storage driver.Driver, dirPath string, newObj model.Obj) {
if storage.Config().NoCache {
return
}
cache, exist := cm.dirCache.Get(Key(storage, dirPath))
if exist {
cache.UpdateObject(newObj.GetName(), newObj)
}
}
// recursively delete directory and its children from dirCache
func (cm *CacheManager) DeleteDirectoryTree(storage driver.Driver, dirPath string) {
if storage.Config().NoCache {
return
}
cm.deleteDirectoryTree(Key(storage, dirPath))
}
func (cm *CacheManager) deleteDirectoryTree(key string) {
if dirCache, exists := cm.dirCache.Take(key); exists {
for _, obj := range dirCache.objs {
if obj.IsDir() {
cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName()))
}
}
}
}
// remove directory from dirCache
func (cm *CacheManager) DeleteDirectory(storage driver.Driver, dirPath string) {
if storage.Config().NoCache {
return
}
cm.dirCache.Delete(Key(storage, dirPath))
}
// remove object from dirCache.
// if it's a directory, remove all its children from dirCache too.
// if it's a file, remove its link from linkCache.
func (cm *CacheManager) removeDirectoryObject(storage driver.Driver, dirPath string, obj model.Obj) {
key := Key(storage, dirPath)
if !obj.IsDir() {
cm.linkCache.DeleteKey(stdpath.Join(key, obj.GetName()))
}
if storage.Config().NoCache {
return
}
if cache, exist := cm.dirCache.Get(key); exist {
if obj.IsDir() {
cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName()))
}
cache.RemoveObject(obj.GetName())
}
}
// cache user data
func (cm *CacheManager) SetUser(username string, user *model.User) {
cm.userCache.Set(username, user)
}
// cached user data
func (cm *CacheManager) GetUser(username string) (*model.User, bool) {
return cm.userCache.Get(username)
}
// remove user data from cache
func (cm *CacheManager) DeleteUser(username string) {
cm.userCache.Delete(username)
}
// caches setting
func (cm *CacheManager) SetSetting(key string, setting *model.SettingItem) {
cm.settingCache.Set(key, setting)
}
// cached setting
func (cm *CacheManager) GetSetting(key string) (*model.SettingItem, bool) {
if data, exists := cm.settingCache.Get(key); exists {
if setting, ok := data.(*model.SettingItem); ok {
return setting, true
}
}
return nil, false
}
// cache setting groups
func (cm *CacheManager) SetSettingGroup(key string, settings []model.SettingItem) {
cm.settingCache.Set(key, settings)
}
// cached setting group
func (cm *CacheManager) GetSettingGroup(key string) ([]model.SettingItem, bool) {
if data, exists := cm.settingCache.Get(key); exists {
if settings, ok := data.([]model.SettingItem); ok {
return settings, true
}
}
return nil, false
}
func (cm *CacheManager) SetStorageDetails(storage driver.Driver, details *model.StorageDetails) {
if storage.Config().NoCache {
return
}
expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration)
cm.detailCache.SetWithTTL(storage.GetStorage().MountPath, details, expiration)
}
func (cm *CacheManager) GetStorageDetails(storage driver.Driver) (*model.StorageDetails, bool) {
return cm.detailCache.Get(storage.GetStorage().MountPath)
}
func (cm *CacheManager) InvalidateStorageDetails(storage driver.Driver) {
cm.detailCache.Delete(storage.GetStorage().MountPath)
}
// clears all caches
func (cm *CacheManager) ClearAll() {
cm.dirCache.Clear()
cm.linkCache.Clear()
cm.userCache.Clear()
cm.settingCache.Clear()
cm.detailCache.Clear()
}
type directoryCache struct {
objs []model.Obj
sorted []model.Obj
mu sync.RWMutex
dirtyFlags uint8
}
const (
dirtyRemove uint8 = 1 << iota // 对象删除:刷新 sorted 副本,但不需要 full sort/extract
dirtyUpdate // 对象更新:需要执行 full sort + extract
)
func newDirectoryCache(objs []model.Obj) *directoryCache {
sorted := make([]model.Obj, len(objs))
copy(sorted, objs)
return &directoryCache{
objs: objs,
sorted: sorted,
}
}
func (dc *directoryCache) RemoveObject(name string) {
dc.mu.Lock()
defer dc.mu.Unlock()
for i, obj := range dc.objs {
if obj.GetName() == name {
dc.objs = append(dc.objs[:i], dc.objs[i+1:]...)
dc.dirtyFlags |= dirtyRemove
break
}
}
}
func (dc *directoryCache) UpdateObject(oldName string, newObj model.Obj) {
dc.mu.Lock()
defer dc.mu.Unlock()
if oldName != "" {
for i, obj := range dc.objs {
if obj.GetName() == oldName {
dc.objs[i] = newObj
dc.dirtyFlags |= dirtyUpdate
return
}
}
}
dc.objs = append(dc.objs, newObj)
dc.dirtyFlags |= dirtyUpdate
}
func (dc *directoryCache) GetSortedObjects(meta driver.Meta) []model.Obj {
dc.mu.RLock()
if dc.dirtyFlags == 0 {
dc.mu.RUnlock()
return dc.sorted
}
dc.mu.RUnlock()
dc.mu.Lock()
defer dc.mu.Unlock()
sorted := make([]model.Obj, len(dc.objs))
copy(sorted, dc.objs)
dc.sorted = sorted
if dc.dirtyFlags&dirtyUpdate != 0 {
storage := meta.GetStorage()
if meta.Config().LocalSort {
model.SortFiles(sorted, storage.OrderBy, storage.OrderDirection)
}
model.ExtractFolder(sorted, storage.ExtractFolder)
}
dc.dirtyFlags = 0
return sorted
}

View File

@@ -4,115 +4,20 @@ import (
"context"
stderrors "errors"
stdpath "path"
"slices"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/generic_sync"
"github.com/OpenListTeam/OpenList/v4/pkg/singleflight"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/go-cache"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// In order to facilitate adding some other things before and after file op
var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
var listG singleflight.Group[[]model.Obj]
func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == newObj.GetName() {
objs = slices.Delete(objs, i, i+1)
break
}
}
for i, obj := range objs {
if obj.GetName() == oldObj.GetName() {
objs[i] = newObj
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func delCacheObj(storage driver.Driver, path string, obj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, oldObj := range objs {
if oldObj.GetName() == obj.GetName() {
objs = append(objs[:i], objs[i+1:]...)
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
var addSortDebounceMap generic_sync.MapOf[string, func(func())]
func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == newObj.GetName() {
objs[i] = newObj
return
}
}
// Simple separation of files and folders
if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() {
objs = append(objs, newObj)
} else {
objs = append([]model.Obj{newObj}, objs...)
}
if storage.Config().LocalSort {
debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute))
log.Debug("addCacheObj: wait start sort")
debounce(func() {
log.Debug("addCacheObj: start sort")
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
addSortDebounceMap.Delete(key)
})
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func ClearCache(storage driver.Driver, path string) {
objs, ok := listCache.Get(Key(storage, path))
if ok {
for _, obj := range objs {
if obj.IsDir() {
ClearCache(storage, stdpath.Join(path, obj.GetName()))
}
}
}
listCache.Del(Key(storage, path))
}
func DeleteCache(storage driver.Driver, path string) {
listCache.Del(Key(storage, path))
}
func Key(storage driver.Driver, path string) string {
return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path))
}
// List files in storage, not contains virtual file
func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs) ([]model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -122,11 +27,12 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
log.Debugf("op.List %s", path)
key := Key(storage, path)
if !args.Refresh {
if files, ok := listCache.Get(key); ok {
if dirCache, exists := Cache.dirCache.Get(key); exists {
log.Debugf("use cache when list %s", path)
return files, nil
return dirCache.GetSortedObjects(storage), nil
}
}
dir, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, errors.WithMessage(err, "failed get dir")
@@ -135,6 +41,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
if !dir.IsDir() {
return nil, errors.WithStack(errs.NotFolder)
}
objs, err, _ := listG.Do(key, func() ([]model.Obj, error) {
files, err := storage.List(ctx, dir, args)
if err != nil {
@@ -150,7 +57,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
model.WrapObjsName(files)
// call hooks
go func(reqPath string, files []model.Obj) {
HandleObjsUpdateHook(reqPath, files)
HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files)
}(utils.GetFullPath(storage.GetStorage().MountPath, path), files)
// sort objs
@@ -162,10 +69,11 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)
listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
ttl := time.Minute * time.Duration(storage.GetStorage().CacheExpiration)
Cache.dirCache.SetWithTTL(key, newDirectoryCache(files), ttl)
} else {
log.Debugf("del cache: %s", key)
listCache.Del(key)
Cache.deleteDirectoryTree(key)
}
}
return files, nil
@@ -252,100 +160,68 @@ func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.O
return model.UnwrapObj(obj), err
}
var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))
var linkG = singleflight.Group[*model.Link]{Remember: true}
var errLinkMFileCache = stderrors.New("ErrLinkMFileCache")
var linkG = singleflight.Group[*objWithLink]{}
// Link get link, if is an url. should have an expiry time
func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status)
}
var (
file model.Obj
err error
)
// use cache directly
dir, name := stdpath.Split(stdpath.Join(storage.GetStorage().MountPath, path))
if cacheFiles, ok := listCache.Get(strings.TrimSuffix(dir, "/")); ok {
for _, f := range cacheFiles {
if f.GetName() == name {
file = model.UnwrapObj(f)
break
}
}
} else {
if g, ok := storage.(driver.GetObjInfo); ok {
file, err = g.GetObjInfo(ctx, path)
} else {
file, err = GetUnwrap(ctx, storage, path)
mode := storage.Config().LinkCacheMode
if mode == -1 {
mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path)
}
typeKey := args.Type
if mode&driver.LinkCacheIP == 1 {
typeKey += "/" + args.IP
}
if mode&driver.LinkCacheUA == 1 {
typeKey += "/" + args.Header.Get("User-Agent")
}
key := Key(storage, path)
if ol, exists := Cache.linkCache.GetType(key, typeKey); exists {
if ol.link.Expiration != nil ||
ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference {
return ol.link, ol.obj, nil
}
}
if file == nil {
fn := func() (*objWithLink, error) {
file, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, nil, errors.WithMessage(err, "failed to get file")
return nil, errors.WithMessage(err, "failed to get file")
}
if file.IsDir() {
return nil, errors.WithStack(errs.NotFile)
}
return nil, nil, errors.WithStack(errs.ObjectNotFound)
}
if file.IsDir() {
return nil, nil, errors.WithStack(errs.NotFile)
}
key := stdpath.Join(Key(storage, path), args.Type)
if link, ok := linkCache.Get(key); ok {
return link, file, nil
}
var forget any
var linkM *model.Link
fn := func() (*model.Link, error) {
link, err := storage.Link(ctx, file, args)
if err != nil {
return nil, errors.Wrapf(err, "failed get link")
}
if link.MFile != nil && forget != nil {
linkM = link
return nil, errLinkMFileCache
}
ol := &objWithLink{link: link, obj: file}
if link.Expiration != nil {
linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration))
Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration)
} else {
Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers)
}
link.AddIfCloser(forget)
return link, nil
return ol, nil
}
if storage.Config().OnlyLinkMFile {
link, err := fn()
retry := 0
for {
ol, err, _ := linkG.Do(key+"/"+typeKey, fn)
if err != nil {
return nil, nil, err
}
return link, file, err
}
forget = utils.CloseFunc(func() error {
if forget != nil {
forget = nil
linkG.Forget(key)
if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference {
if retry > 1 {
log.Warnf("Link retry successed after %d times: %s %s", retry, key, typeKey)
}
return ol.link, ol.obj, nil
}
return nil
})
link, err, _ := linkG.Do(key, fn)
for err == nil && !link.AcquireReference() {
link, err, _ = linkG.Do(key, fn)
retry++
}
if err == errLinkMFileCache {
if linkM != nil {
return linkM, file, nil
}
forget = nil
link, err = fn()
}
if err != nil {
return nil, nil, err
}
return link, file, nil
}
// Other api
@@ -365,7 +241,7 @@ func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (
}
}
var mkdirG singleflight.Group[interface{}]
var mkdirG singleflight.Group[any]
func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -373,7 +249,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache
}
path = utils.FixAndCleanPath(path)
key := Key(storage, path)
_, err, _ := mkdirG.Do(key, func() (interface{}, error) {
_, err, _ := mkdirG.Do(key, func() (any, error) {
// check if dir exists
f, err := GetUnwrap(ctx, storage, path)
if err != nil {
@@ -395,15 +271,19 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache
newObj, err = s.MakeDir(ctx, parentDir, dirName)
if err == nil {
if newObj != nil {
addCacheObj(storage, parentPath, model.WrapObjName(newObj))
if !storage.Config().NoCache {
if dirCache, exist := Cache.dirCache.Get(Key(storage, parentPath)); exist {
dirCache.UpdateObject("", newObj)
}
}
} else if !utils.IsBool(lazyCache...) {
DeleteCache(storage, parentPath)
Cache.DeleteDirectory(storage, parentPath)
}
}
case driver.Mkdir:
err = s.MakeDir(ctx, parentDir, dirName)
if err == nil && !utils.IsBool(lazyCache...) {
DeleteCache(storage, parentPath)
Cache.DeleteDirectory(storage, parentPath)
}
default:
return nil, errs.NotImplement
@@ -427,7 +307,11 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
srcDirPath := stdpath.Dir(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
if dstDirPath == srcDirPath {
return stderrors.New("move in place")
}
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
@@ -437,26 +321,25 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.MoveResult:
var newObj model.Obj
newObj, err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
Cache.removeDirectoryObject(storage, srcDirPath, srcRawObj)
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
Cache.addDirectoryObject(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
Cache.DeleteDirectory(storage, dstDirPath)
}
}
case driver.Move:
err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
Cache.removeDirectoryObject(storage, srcDirPath, srcRawObj)
if !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
Cache.DeleteDirectory(storage, dstDirPath)
}
}
default:
@@ -475,28 +358,29 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string,
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObj(srcRawObj)
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.RenameResult:
var newObj model.Obj
newObj, err = s.Rename(ctx, srcObj, dstName)
if err == nil {
srcDirPath := stdpath.Dir(srcPath)
if newObj != nil {
updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
DeleteCache(storage, srcDirPath)
if srcRawObj.IsDir() {
ClearCache(storage, srcPath)
Cache.updateDirectoryObject(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj))
} else {
Cache.removeDirectoryObject(storage, srcDirPath, srcRawObj)
if !utils.IsBool(lazyCache...) {
Cache.DeleteDirectory(storage, srcDirPath)
}
}
}
case driver.Rename:
err = s.Rename(ctx, srcObj, dstName)
if err == nil && !utils.IsBool(lazyCache...) {
DeleteCache(storage, srcDirPath)
if srcRawObj.IsDir() {
ClearCache(storage, srcPath)
if err == nil {
srcDirPath := stdpath.Dir(srcPath)
Cache.removeDirectoryObject(storage, srcDirPath, srcRawObj)
if !utils.IsBool(lazyCache...) {
Cache.DeleteDirectory(storage, srcDirPath)
}
}
default:
@@ -512,10 +396,14 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
}
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
srcObj, err := GetUnwrap(ctx, storage, srcPath)
if dstDirPath == stdpath.Dir(srcPath) {
return stderrors.New("copy in place")
}
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObj(srcRawObj)
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
@@ -527,15 +415,17 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
newObj, err = s.Copy(ctx, srcObj, dstDir)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
Cache.addDirectoryObject(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
Cache.DeleteDirectory(storage, dstDirPath)
}
}
case driver.Copy:
err = s.Copy(ctx, srcObj, dstDir)
if err == nil && !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
if err == nil {
if !utils.IsBool(lazyCache...) {
Cache.DeleteDirectory(storage, dstDirPath)
}
}
default:
return errs.NotImplement
@@ -566,11 +456,7 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error {
case driver.Remove:
err = s.Remove(ctx, model.UnwrapObj(rawObj))
if err == nil {
delCacheObj(storage, dirPath, rawObj)
// clear folder cache recursively
if rawObj.IsDir() {
ClearCache(storage, path)
}
Cache.removeDirectoryObject(storage, dirPath, rawObj)
}
default:
return errs.NotImplement
@@ -640,16 +526,20 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
var newObj model.Obj
newObj, err = s.Put(ctx, parentDir, file, up)
if err == nil {
Cache.linkCache.DeleteKey(Key(storage, dstPath))
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
Cache.addDirectoryObject(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
Cache.DeleteDirectory(storage, dstDirPath)
}
}
case driver.Put:
err = s.Put(ctx, parentDir, file, up)
if err == nil && !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
if err == nil {
Cache.linkCache.DeleteKey(Key(storage, dstPath))
if !utils.IsBool(lazyCache...) {
Cache.DeleteDirectory(storage, dstDirPath)
}
}
default:
return errs.NotImplement
@@ -664,13 +554,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
}
} else {
// upload success, remove old obj
err := Remove(ctx, storage, tempPath)
if err != nil {
return err
} else {
key := Key(storage, stdpath.Join(dstDirPath, file.GetName()))
linkCache.Del(key)
}
err = Remove(ctx, storage, tempPath)
}
}
return errors.WithStack(err)
@@ -681,7 +565,8 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url
return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status)
}
dstDirPath = utils.FixAndCleanPath(dstDirPath)
_, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName))
dstPath := stdpath.Join(dstDirPath, dstName)
_, err := GetUnwrap(ctx, storage, dstPath)
if err == nil {
return errors.New("obj already exists")
}
@@ -698,16 +583,20 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url
var newObj model.Obj
newObj, err = s.PutURL(ctx, dstDir, dstName, url)
if err == nil {
Cache.linkCache.DeleteKey(Key(storage, dstPath))
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
Cache.addDirectoryObject(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
Cache.DeleteDirectory(storage, dstDirPath)
}
}
case driver.PutURL:
err = s.PutURL(ctx, dstDir, dstName, url)
if err == nil && !utils.IsBool(lazyCache...) {
DeleteCache(storage, dstDirPath)
if err == nil {
Cache.linkCache.DeleteKey(Key(storage, dstPath))
if !utils.IsBool(lazyCache...) {
Cache.DeleteDirectory(storage, dstDirPath)
}
}
default:
return errs.NotImplement

View File

@@ -1,6 +1,7 @@
package op
import (
"context"
"regexp"
"strings"
@@ -13,7 +14,7 @@ import (
)
// Obj
type ObjsUpdateHook = func(parent string, objs []model.Obj)
type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj)
var (
objsUpdateHooks = make([]ObjsUpdateHook, 0)
@@ -23,9 +24,9 @@ func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
objsUpdateHooks = append(objsUpdateHooks, hook)
}
func HandleObjsUpdateHook(parent string, objs []model.Obj) {
func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) {
for _, hook := range objsUpdateHooks {
hook(parent, objs)
hook(ctx, parent, objs)
}
}

View File

@@ -5,26 +5,21 @@ import (
"sort"
"strconv"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/db"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/singleflight"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/go-cache"
"github.com/pkg/errors"
)
var settingCache = cache.NewMemCache(cache.WithShards[*model.SettingItem](4))
var settingG singleflight.Group[*model.SettingItem]
var settingCacheF = func(item *model.SettingItem) {
settingCache.Set(item.Key, item, cache.WithEx[*model.SettingItem](time.Hour))
Cache.SetSetting(item.Key, item)
}
var settingGroupCache = cache.NewMemCache(cache.WithShards[[]model.SettingItem](4))
var settingGroupG singleflight.Group[[]model.SettingItem]
var settingGroupCacheF = func(key string, item []model.SettingItem) {
settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour))
var settingGroupCacheF = func(key string, items []model.SettingItem) {
Cache.SetSettingGroup(key, items)
}
var settingChangingCallbacks = make([]func(), 0)
@@ -34,8 +29,7 @@ func RegisterSettingChangingCallback(f func()) {
}
func SettingCacheUpdate() {
settingCache.Clear()
settingGroupCache.Clear()
Cache.ClearAll()
for _, cb := range settingChangingCallbacks {
cb()
}
@@ -60,7 +54,7 @@ func GetSettingsMap() map[string]string {
}
func GetSettingItems() ([]model.SettingItem, error) {
if items, ok := settingGroupCache.Get("ALL_SETTING_ITEMS"); ok {
if items, exists := Cache.GetSettingGroup("ALL_SETTING_ITEMS"); exists {
return items, nil
}
items, err, _ := settingGroupG.Do("ALL_SETTING_ITEMS", func() ([]model.SettingItem, error) {
@@ -75,7 +69,7 @@ func GetSettingItems() ([]model.SettingItem, error) {
}
func GetPublicSettingItems() ([]model.SettingItem, error) {
if items, ok := settingGroupCache.Get("ALL_PUBLIC_SETTING_ITEMS"); ok {
if items, exists := Cache.GetSettingGroup("ALL_PUBLIC_SETTING_ITEMS"); exists {
return items, nil
}
items, err, _ := settingGroupG.Do("ALL_PUBLIC_SETTING_ITEMS", func() ([]model.SettingItem, error) {
@@ -90,7 +84,7 @@ func GetPublicSettingItems() ([]model.SettingItem, error) {
}
func GetSettingItemByKey(key string) (*model.SettingItem, error) {
if item, ok := settingCache.Get(key); ok {
if item, exists := Cache.GetSetting(key); exists {
return item, nil
}
@@ -118,8 +112,8 @@ func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) {
}
func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) {
key := strconv.Itoa(group)
if items, ok := settingGroupCache.Get(key); ok {
key := fmt.Sprintf("GROUP_%d", group)
if items, exists := Cache.GetSettingGroup(key); exists {
return items, nil
}
items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) {
@@ -135,11 +129,14 @@ func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) {
func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) {
sort.Ints(groups)
key := strings.Join(utils.MustSliceConvert(groups, func(i int) string {
return strconv.Itoa(i)
}), ",")
if items, ok := settingGroupCache.Get(key); ok {
keyParts := make([]string, 0, len(groups))
for _, g := range groups {
keyParts = append(keyParts, strconv.Itoa(g))
}
key := "GROUPS_" + strings.Join(keyParts, "_")
if items, exists := Cache.GetSettingGroup(key); exists {
return items, nil
}
items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) {
@@ -165,10 +162,10 @@ func SaveSettingItems(items []model.SettingItem) error {
}
}
err := db.SaveSettingItems(items)
if err != nil {
if err != nil {
return fmt.Errorf("failed save setting: %+v", err)
}
SettingCacheUpdate()
SettingCacheUpdate()
return nil
}

View File

@@ -137,3 +137,7 @@ func DeleteSharing(sid string) error {
sharingCache.Del(sid)
return db.DeleteSharingById(sid)
}
func DeleteSharingsByCreatorId(creatorId uint) error {
return db.DeleteSharingsByCreatorId(creatorId)
}

Some files were not shown because too many files have changed in this diff Show More