github-mirror/rclone
github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-04-16 16:18:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

union: move count fallback to original GetNumObjects

Browse Source 
Signed-off-by: GitHub <noreply@github.com>
This commit is contained in:
Fernando Fernández 2025-04-11 20:08:28 +00:00 committed by GitHub
parent 7a09de9f27
commit f03e759f29
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 37 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
backend/union/policy/eplno.go
View File

@ -18,28 +18,11 @@ type EpLno struct {
EpAll
}
func (p *EpLno) getUpstreamObjectCount(ctx context.Context, upstream *upstream.Fs) int64 {
numObj, err := upstream.GetNumObjects()
if err != nil {
uName := upstream.Name()
fs.LogPrintf(fs.LogLevelNotice, nil,
"Number of Objects is not supported for upstream %s, falling back to listing (this may be slower)...", uName)
listing, err := upstream.List(ctx, "")
if err != nil {
fs.LogPrintf(fs.LogLevelError, nil, "Listing failed, treating as 0: %v", err)
return 0
}
numObj = int64(len(listing))
fs.LogPrintf(fs.LogLevelDebug, nil, "Counted %d objects for upstream %s by listing", numObj, uName)
}
return numObj
}
func (p *EpLno) lno(ctx context.Context, upstreams []*upstream.Fs) (*upstream.Fs, error) {
func (p *EpLno) lno(upstreams []*upstream.Fs) (*upstream.Fs, error) {
var minNumObj int64 = math.MaxInt64
var lnoUpstream *upstream.Fs
for _, u := range upstreams {
numObj := p.getUpstreamObjectCount(ctx, u)
numObj := u.GetNumObjects()
if minNumObj > numObj {
minNumObj = numObj
lnoUpstream = u
@ -51,11 +34,11 @@ func (p *EpLno) lno(ctx context.Context, upstreams []*upstream.Fs) (*upstream.Fs
return lnoUpstream, nil
}
func (p *EpLno) lnoEntries(ctx context.Context, entries []upstream.Entry) (upstream.Entry, error) {
func (p *EpLno) lnoEntries(entries []upstream.Entry) (upstream.Entry, error) {
var minNumObj int64 = math.MaxInt64
var lnoEntry upstream.Entry
for _, e := range entries {
numObj := p.getUpstreamObjectCount(ctx, e.UpstreamFs())
numObj := e.UpstreamFs().GetNumObjects()
if minNumObj > numObj {
minNumObj = numObj
lnoEntry = e
@ -70,7 +53,7 @@ func (p *EpLno) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
if err != nil {
return nil, err
}
u, err := p.lno(ctx, upstreams)
u, err := p.lno(upstreams)
return []*upstream.Fs{u}, err
}
@ -80,7 +63,7 @@ func (p *EpLno) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, erro
if err != nil {
return nil, err
}
e, err := p.lnoEntries(context.Background(), entries)
e, err := p.lnoEntries(entries)
return []upstream.Entry{e}, err
}
@ -90,7 +73,7 @@ func (p *EpLno) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
if err != nil {
return nil, err
}
u, err := p.lno(ctx, upstreams)
u, err := p.lno(upstreams)
return []*upstream.Fs{u}, err
}
@ -100,7 +83,7 @@ func (p *EpLno) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, erro
if err != nil {
return nil, err
}
e, err := p.lnoEntries(context.Background(), entries)
e, err := p.lnoEntries(entries)
return []upstream.Entry{e}, err
}
@ -113,7 +96,7 @@ func (p *EpLno) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
if err != nil {
return nil, err
}
return p.lno(ctx, upstreams)
return p.lno(upstreams)
}
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
@ -121,5 +104,5 @@ func (p *EpLno) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error)
if len(entries) == 0 {
return nil, fs.ErrorObjectNotFound
}
return p.lnoEntries(context.Background(), entries)
return p.lnoEntries(entries)
}

2
backend/union/policy/lno.go
View File

@ -28,6 +28,6 @@ func (p *Lno) Create(ctx context.Context, upstreams []*upstream.Fs, path string)
if len(upstreams) == 0 {
return nil, fs.ErrorPermissionDenied
}
u, err := p.lno(ctx, upstreams)
u, err := p.lno(upstreams)
return []*upstream.Fs{u}, err
}

34
backend/union/upstream/upstream.go
View File

@ -438,19 +438,37 @@ func (f *Fs) GetUsedSpace() (int64, error) {
}
// GetNumObjects get the number of objects of the fs
func (f *Fs) GetNumObjects() (int64, error) {
func (f *Fs) GetNumObjects() int64 {
var err error
if f.cacheExpiry.Load() <= time.Now().Unix() {
err := f.updateUsage()
if err != nil {
return 0, ErrUsageFieldNotSupported
err = f.updateUsage()
}
if f.usage.Objects == nil {
uName := f.Name()
fs.LogPrintf(
fs.LogLevelWarning,
nil,
"Number of objects not supported for upstream %s, falling back to listing (this will be slower)...",
uName,
)
count, _, _, err := operations.Count(context.Background(), f)
if err == nil {
fs.Debugf(nil, "Counted %d objects for upstream %s by listing", count, uName)
f.cacheMutex.Lock()
f.usage.Objects = &count
f.cacheMutex.Unlock()
}
}
if err != nil {
fs.Errorf(nil, "Error getting number of objects, treating as 0: %v", err)
return 0
}
f.cacheMutex.RLock()
defer f.cacheMutex.RUnlock()
if f.usage.Objects == nil {
return 0, ErrUsageFieldNotSupported
}
return *f.usage.Objects, nil
return *f.usage.Objects
}
func (f *Fs) updateUsage() (err error) {