Update README.md

removed warp as project sponsor

.github/workflows/build.yml

@@ -26,12 +26,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.23']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -42,14 +42,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -58,14 +58,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -75,20 +75,14 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.21
+          - job_name: go1.23
             os: ubuntu-latest
-            go: '1.21'
-            quicktest: true
-            racequicktest: true
-
-          - job_name: go1.22
-            os: ubuntu-latest
-            go: '1.22'
+            go: '1.23'
             quicktest: true
             racequicktest: true
 
@@ -232,6 +226,8 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
 
       - name: Install Go
         id: setup-go
@@ -295,6 +291,10 @@ jobs:
       - name: Scan for vulnerabilities
         run: govulncheck ./...
 
+      - name: Scan edits of autogenerated files
+        run: bin/check_autogenerated_edits.py
+        if: github.event_name == 'pull_request'
+
   android:
     if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
     timeout-minutes: 30
@@ -311,7 +311,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '>=1.23.0-rc.1'
+          go-version: '>=1.24.0-rc.1'
 
       - name: Set global environment variables
         shell: bash

.github/workflows/build_publish_docker_plugin.yml

@@ -7,39 +7,43 @@ name: Release Build for Docker Plugin
 on:
   release:
     types: [published]
+  workflow_dispatch:
+    inputs:
+      manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
+        default: true
 
 jobs:
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build docker plugin job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Build and publish docker plugin
-              shell: bash
-              run: |
-                VER=${GITHUB_REF#refs/tags/}
-                PLUGIN_USER=rclone
-                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-                    export PLUGIN_USER PLUGIN_ARCH
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-                done
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}
+  build_docker_volume_plugin:
+    if: inputs.manual || github.repository == 'rclone/rclone'
+    name: Build docker plugin job
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free some space
+        shell: bash
+        run: |
+          df -h .
+          # Remove android SDK
+          sudo rm -rf /usr/local/lib/android || true
+          # Remove .net runtime
+          sudo rm -rf /usr/share/dotnet || true
+          df -h .
+      - name: Checkout master
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build and publish docker plugin
+        shell: bash
+        run: |
+          VER=${GITHUB_REF#refs/tags/}
+          PLUGIN_USER=rclone
+          docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                       --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+          for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+              export PLUGIN_USER PLUGIN_ARCH
+              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+          done
+          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

MANUAL.md

@@ -1,78 +1,7 @@
 % rclone(1) User Manual
 % Nick Craig-Wood
-% Feb 14, 2025
+% Jan 12, 2025
 
-# NAME
-
-rclone - manage files on cloud storage
-
-# SYNOPSIS
-
-```
-Usage:
-  rclone [flags]
-  rclone [command]
-
-Available commands:
-  about       Get quota information from the remote.
-  authorize   Remote authorization.
-  backend     Run a backend-specific command.
-  bisync      Perform bidirectional synchronization between two paths.
-  cat         Concatenates any files and sends them to stdout.
-  check       Checks the files in the source and destination match.
-  checksum    Checks the files in the destination against a SUM file.
-  cleanup     Clean up the remote if possible.
-  completion  Output completion script for a given shell.
-  config      Enter an interactive configuration session.
-  copy        Copy files from source to dest, skipping identical files.
-  copyto      Copy files from source to dest, skipping identical files.
-  copyurl     Copy the contents of the URL supplied content to dest:path.
-  cryptcheck  Cryptcheck checks the integrity of an encrypted remote.
-  cryptdecode Cryptdecode returns unencrypted file names.
-  dedupe      Interactively find duplicate filenames and delete/rename them.
-  delete      Remove the files in path.
-  deletefile  Remove a single file from remote.
-  gendocs     Output markdown docs for rclone to the directory supplied.
-  gitannex    Speaks with git-annex over stdin/stdout.
-  hashsum     Produces a hashsum file for all the objects in the path.
-  help        Show help for rclone commands, flags and backends.
-  link        Generate public link to file/folder.
-  listremotes List all the remotes in the config file and defined in environment variables.
-  ls          List the objects in the path with size and path.
-  lsd         List all directories/containers/buckets in the path.
-  lsf         List directories and objects in remote:path formatted for parsing.
-  lsjson      List directories and objects in the path in JSON format.
-  lsl         List the objects in path with modification time, size and path.
-  md5sum      Produces an md5sum file for all the objects in the path.
-  mkdir       Make the path if it doesn't already exist.
-  mount       Mount the remote as file system on a mountpoint.
-  move        Move files from source to dest.
-  moveto      Move file or directory from source to dest.
-  ncdu        Explore a remote with a text based user interface.
-  nfsmount    Mount the remote as file system on a mountpoint.
-  obscure     Obscure password for use in the rclone config file.
-  purge       Remove the path and all of its contents.
-  rc          Run a command against a running rclone.
-  rcat        Copies standard input to file on remote.
-  rcd         Run rclone listening to remote control commands only.
-  rmdir       Remove the empty directory at path.
-  rmdirs      Remove empty directories under the path.
-  selfupdate  Update the rclone binary.
-  serve       Serve a remote over a protocol.
-  settier     Changes storage class/tier of objects in remote.
-  sha1sum     Produces an sha1sum file for all the objects in the path.
-  size        Prints the total size and number of objects in remote:path.
-  sync        Make source and dest identical, modifying destination only.
-  test        Run a test command
-  touch       Create new file or change file modification time.
-  tree        List the contents of the remote in a tree like fashion.
-  version     Show the version number.
-
-Use "rclone [command] --help" for more information about a command.
-Use "rclone help flags" for to see the global flags.
-Use "rclone help backends" for a list of supported services.
-
-```
 # Rclone syncs your files to cloud storage
 
 <img width="50%" src="https://rclone.org/img/logo_on_light__horizontal_color.svg" alt="rclone logo" style="float:right; padding: 5px;" >
@@ -1761,9 +1690,6 @@ include/exclude filters - everything will be removed.  Use the
 delete files. To delete empty directories only, use command
 [rmdir](https://rclone.org/commands/rclone_rmdir/) or [rmdirs](https://rclone.org/commands/rclone_rmdirs/).
 
-The concurrency of this operation is controlled by the `--checkers` global flag. However, some backends will
-implement this command directly, in which case `--checkers` will be ignored.
-
 **Important**: Since this can cause data loss, test first with the
 `--dry-run` or the `--interactive`/`-i` flag.
 
@@ -3819,12 +3745,12 @@ password to re-encrypt the config.
 
 When `--password-command` is called to change the password then the
 environment variable `RCLONE_PASSWORD_CHANGE=1` will be set. So if
-changing passwords programmatically you can use the environment
+changing passwords programatically you can use the environment
 variable to distinguish which password you must supply.
 
 Alternatively you can remove the password first (with `rclone config
 encryption remove`), then set it again with this command which may be
-easier if you don't mind the unencrypted config file being on the disk
+easier if you don't mind the unecrypted config file being on the disk
 briefly.
 
 
@@ -4364,7 +4290,7 @@ destination if there is one with the same name.
 Setting `--stdout` or making the output file name `-`
 will cause the output to be written to standard output.
 
-## Troubleshooting
+## Troublshooting
 
 If you can't get `rclone copyurl` to work then here are some things you can try:
 
@@ -10655,7 +10581,7 @@ that it uses an on disk cache, but the cache entries are held as
 symlinks. Rclone will use the handle of the underlying file as the NFS
 handle which improves performance. This sort of cache can't be backed
 up and restored as the underlying handles will change. This is Linux
-only. It requires running rclone as root or with `CAP_DAC_READ_SEARCH`.
+only. It requres running rclone as root or with `CAP_DAC_READ_SEARCH`.
 You can run rclone with this extra permission by doing this to the
 rclone binary `sudo setcap cap_dac_read_search+ep /path/to/rclone`.
 
@@ -11482,7 +11408,7 @@ secret_access_key = SECRET_ACCESS_KEY
 use_multipart_uploads = false
 ```
 
-Note that setting `use_multipart_uploads = false` is to work around
+Note that setting `disable_multipart_uploads = true` is to work around
 [a bug](#bugs) which will be fixed in due course.
 
 ## Bugs
@@ -14518,11 +14444,6 @@ it to `false`.  It is also possible to specify `--boolean=false` or
 parsed as `--boolean` and the `false` is parsed as an extra command
 line argument for rclone.
 
-Options documented to take a `stringArray` parameter accept multiple 
-values. To pass more than one value, repeat the option; for example: 
-`--include value1 --include value2`.
-
-
 ### Time or duration options {#time-option}
 
 TIME or DURATION options can be specified as a duration string or a
@@ -16834,7 +16755,7 @@ so they take exactly the same form.
 The options set by environment variables can be seen with the `-vv` flag, e.g. `rclone version -vv`.
 
 Options that can appear multiple times (type `stringArray`) are
-treated slightly differently as environment variables can only be
+treated slighly differently as environment variables can only be
 defined once. In order to allow a simple mechanism for adding one or
 many items, the input is treated as a [CSV encoded](https://godoc.org/encoding/csv)
 string. For example
@@ -20016,7 +19937,7 @@ the `--vfs-cache-mode` is off, it will return an empty result.
        ],
     }
 
-The `expiry` time is the time until the file is eligible for being
+The `expiry` time is the time until the file is elegible for being
 uploaded in floating point seconds. This may go negative. As rclone
 only transfers `--transfers` files at once, only the lowest
 `--transfers` expiry times will have `uploading` as `true`. So there
@@ -21097,7 +21018,7 @@ Flags for general networking and HTTP stuff.
       --tpslimit float                     Limit HTTP transactions per second to this
       --tpslimit-burst int                 Max burst of transactions for --tpslimit (default 1)
       --use-cookies                        Enable session cookiejar
-      --user-agent string                  Set the user-agent to a specified string (default "rclone/v1.69.1")
+      --user-agent string                  Set the user-agent to a specified string (default "rclone/v1.69.0")
 ```
 
 
@@ -23145,7 +23066,7 @@ See the [bisync filters](#filtering) section and generic
 [--filter-from](https://rclone.org/filtering/#filter-from-read-filtering-patterns-from-a-file)
 documentation.
 An [example filters file](#example-filters-file) contains filters for
-non-allowed files for syncing with Dropbox.
+non-allowed files for synching with Dropbox.
 
 If you make changes to your filters file then bisync requires a run
 with `--resync`. This is a safety feature, which prevents existing files
@@ -23322,7 +23243,7 @@ Using `--check-sync=false` will disable it and may significantly reduce the
 sync run times for very large numbers of files.
 
 The check may be run manually with `--check-sync=only`. It runs only the
-integrity check and terminates without actually syncing.
+integrity check and terminates without actually synching.
 
 Note that currently, `--check-sync` **only checks listing snapshots and NOT the
 actual files on the remotes.** Note also that the listing snapshots will not
@@ -23799,7 +23720,7 @@ The `--include*`, `--exclude*`, and `--filter` flags are also supported.
 
 ### How to filter directories
 
-Filtering portions of the directory tree is a critical feature for syncing.
+Filtering portions of the directory tree is a critical feature for synching.
 
 Examples of directory trees (always beneath the Path1/Path2 root level)
 you may want to exclude from your sync:
@@ -23908,7 +23829,7 @@ quashed by adding `--quiet` to the bisync command line.
 
 ## Example exclude-style filters files for use with Dropbox {#exclude-filters}
 
-- Dropbox disallows syncing the listed temporary and configuration/data files.
+- Dropbox disallows synching the listed temporary and configuration/data files.
   The `- <filename>` filters exclude these files where ever they may occur
   in the sync tree. Consider adding similar exclusions for file types
   you don't need to sync, such as core dump and software build files.
@@ -24242,7 +24163,7 @@ test command flags can be equally prefixed by a single `-` or double dash.
 
 - `go test . -case basic -remote local -remote2 local`
   runs the `test_basic` test case using only the local filesystem,
-  syncing one local directory with another local directory.
+  synching one local directory with another local directory.
   Test script output is to the console, while commands within scenario.txt
   have their output sent to the `.../workdir/test.log` file,
   which is finally compared to the golden copy.
@@ -24473,9 +24394,6 @@ about _Unison_ and synchronization in general.
 
 ## Changelog
 
-### `v1.69.1`
-* Fixed an issue causing listings to not capture concurrent modifications under certain conditions
-
 ### `v1.68`
 * Fixed an issue affecting backends that round modtimes to a lower precision.
 
@@ -25762,7 +25680,7 @@ Notes on above:
    that `USER_NAME` has been created.
 2. The Resource entry must include both resource ARNs, as one implies
    the bucket and the other implies the bucket's objects.
-3. When using [s3-no-check-bucket](#s3-no-check-bucket) and the bucket already exists, the `"arn:aws:s3:::BUCKET_NAME"` doesn't have to be included.
+3. When using [s3-no-check-bucket](#s3-no-check-bucket) and the bucket already exsits, the `"arn:aws:s3:::BUCKET_NAME"` doesn't have to be included.
 
 For reference, [here's an Ansible script](https://gist.github.com/ebridges/ebfc9042dd7c756cd101cfa807b7ae2b)
 that will generate one or more buckets that will work with `rclone sync`.
@@ -28740,7 +28658,7 @@ location_constraint = au-nsw
 ### Rclone Serve S3 {#rclone}
 
 Rclone can serve any remote over the S3 protocol. For details see the
-[rclone serve s3](https://rclone.org/commands/rclone_serve_s3/) documentation.
+[rclone serve s3](https://rclone.org/commands/rclone_serve_http/) documentation.
 
 For example, to serve `remote:path` over s3, run the server like this:
 
@@ -28760,8 +28678,8 @@ secret_access_key = SECRET_ACCESS_KEY
 use_multipart_uploads = false
 ```
 
-Note that setting `use_multipart_uploads = false` is to work around
-[a bug](https://rclone.org/commands/rclone_serve_s3/#bugs) which will be fixed in due course.
+Note that setting `disable_multipart_uploads = true` is to work around
+[a bug](https://rclone.org/commands/rclone_serve_http/#bugs) which will be fixed in due course.
 
 ### Scaleway
 
@@ -29857,49 +29775,27 @@ Option endpoint.
 Endpoint for Linode Object Storage API.
 Choose a number from below, or type in your own value.
 Press Enter to leave empty.
- 1 / Amsterdam (Netherlands), nl-ams-1
-   \ (nl-ams-1.linodeobjects.com)
- 2 / Atlanta, GA (USA), us-southeast-1
+ 1 / Atlanta, GA (USA), us-southeast-1
    \ (us-southeast-1.linodeobjects.com)
- 3 / Chennai (India), in-maa-1
-   \ (in-maa-1.linodeobjects.com)
- 4 / Chicago, IL (USA), us-ord-1
+ 2 / Chicago, IL (USA), us-ord-1
    \ (us-ord-1.linodeobjects.com)
- 5 / Frankfurt (Germany), eu-central-1
+ 3 / Frankfurt (Germany), eu-central-1
    \ (eu-central-1.linodeobjects.com)
- 6 / Jakarta (Indonesia), id-cgk-1
-   \ (id-cgk-1.linodeobjects.com)
- 7 / London 2 (Great Britain), gb-lon-1
-   \ (gb-lon-1.linodeobjects.com)
- 8 / Los Angeles, CA (USA), us-lax-1
-   \ (us-lax-1.linodeobjects.com)
- 9 / Madrid (Spain), es-mad-1
-   \ (es-mad-1.linodeobjects.com)
-10 / Melbourne (Australia), au-mel-1
-   \ (au-mel-1.linodeobjects.com)
-11 / Miami, FL (USA), us-mia-1
-   \ (us-mia-1.linodeobjects.com)
-12 / Milan (Italy), it-mil-1
+ 4 / Milan (Italy), it-mil-1
    \ (it-mil-1.linodeobjects.com)
-13 / Newark, NJ (USA), us-east-1
+ 5 / Newark, NJ (USA), us-east-1
    \ (us-east-1.linodeobjects.com)
-14 / Osaka (Japan), jp-osa-1
-   \ (jp-osa-1.linodeobjects.com)
-15 / Paris (France), fr-par-1
+ 6 / Paris (France), fr-par-1
    \ (fr-par-1.linodeobjects.com)
-16 / São Paulo (Brazil), br-gru-1
-   \ (br-gru-1.linodeobjects.com)
-17 / Seattle, WA (USA), us-sea-1
+ 7 / Seattle, WA (USA), us-sea-1
    \ (us-sea-1.linodeobjects.com)
-18 / Singapore, ap-south-1
+ 8 / Singapore ap-south-1
    \ (ap-south-1.linodeobjects.com)
-19 / Singapore 2, sg-sin-1
-   \ (sg-sin-1.linodeobjects.com)
-20 / Stockholm (Sweden), se-sto-1
+ 9 / Stockholm (Sweden), se-sto-1
    \ (se-sto-1.linodeobjects.com)
-21 / Washington, DC, (USA), us-iad-1
+10 / Washington, DC, (USA), us-iad-1
    \ (us-iad-1.linodeobjects.com)
-endpoint> 5
+endpoint> 3
 
 Option acl.
 Canned ACL used when creating buckets and storing or copying objects.
@@ -34519,7 +34415,7 @@ strong random number generator.  The nonce is incremented for each
 chunk read making sure each nonce is unique for each block written.
 The chance of a nonce being reused is minuscule.  If you wrote an
 exabyte of data (10¹⁸ bytes) you would have a probability of
-approximately 2×10⁻³² of reusing a nonce.
+approximately 2×10⁻³² of re-using a nonce.
 
 #### Chunk
 
@@ -41665,7 +41561,7 @@ Enter a value.
 config_2fa> 2FACODE
 Remote config
 --------------------
-[iclouddrive]
+[koofr]
 - type: iclouddrive
 - apple_id: APPLEID
 - password: *** ENCRYPTED ***
@@ -41682,20 +41578,6 @@ y/e/d> y
 
 ADP is currently unsupported and need to be disabled
 
-On iPhone, Settings `>` Apple Account `>` iCloud `>` 'Access iCloud Data on the Web' must be ON, and 'Advanced Data Protection' OFF.
-
-## Troubleshooting
-
-### Missing PCS cookies from the request
-
-This means you have Advanced Data Protection (ADP) turned on. This is not supported at the moment. If you want to use rclone you will have to turn it off. See above for how to turn it off.
-
-You will need to clear the `cookies` and the `trust_token` fields in the config. Or you can delete the remote config and start again.
-
-You should then run `rclone reconnect remote:`.
-
-Note that changing the ADP setting may not take effect immediately - you may need to wait a few hours or a day before you can get rclone to work - keep clearing the config entry and running `rclone reconnect remote:` until rclone functions properly.
-
 
 ### Standard options
 
@@ -46153,7 +46035,7 @@ Properties:
     - "us"
         - Microsoft Cloud for US Government
     - "de"
-        - Microsoft Cloud Germany (deprecated - try global region first).
+        - Microsoft Cloud Germany
     - "cn"
         - Azure and Office 365 operated by Vnet Group in China
 
@@ -46770,28 +46652,6 @@ See the [metadata](https://rclone.org/docs/#metadata) docs for more info.
 
 
 
-### Impersonate other users as Admin
-
-Unlike Google Drive and impersonating any domain user via service accounts, OneDrive requires you to authenticate as an admin account, and manually setup a remote per user you wish to impersonate.
-
-1. In [Microsoft 365 Admin Center](https://admin.microsoft.com), open each user you need to "impersonate" and go to the OneDrive section. There is a heading called "Get access to files", you need to click to create the link, this creates the link of the format `https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/` but also changes the permissions so you your admin user has access.
-2. Then in powershell run the following commands:
-```console
-Install-Module Microsoft.Graph -Scope CurrentUser -Repository PSGallery -Force
-Import-Module Microsoft.Graph.Files
-Connect-MgGraph -Scopes "Files.ReadWrite.All"
-# Follow the steps to allow access to your admin user
-# Then run this for each user you want to impersonate to get the Drive ID
-Get-MgUserDefaultDrive -UserId '{emailaddress}'
-# This will give you output of the format:
-# Name     Id                                                                 DriveType CreatedDateTime
-# ----     --                                                                 --------- ---------------
-# OneDrive b!XYZ123                                                           business  14/10/2023 1:00:58 pm
-
-```
-3. Then in rclone add a onedrive remote type, and use the `Type in driveID` with the DriveID you got in the previous step. One remote per user. It will then confirm the drive ID, and hopefully give you a message of `Found drive "root" of type "business"` and then include the URL of the format `https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/Documents`
-
-
 ## Limitations
 
 If you don't use rclone for 90 days the refresh token will
@@ -56649,32 +56509,6 @@ Options:
 
 # Changelog
 
-## v1.69.1 - 2025-02-14
-
-[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.69.1)
-
-* Bug Fixes
-    * lib/oauthutil: Fix redirect URL mismatch errors (Nick Craig-Wood)
-    * bisync: Fix listings missing concurrent modifications (nielash)
-    * serve s3: Fix list objects encoding-type (Nick Craig-Wood)
-    * fs: Fix confusing "didn't find section in config file" error (Nick Craig-Wood)
-    * doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies)
-    * build: Added parallel docker builds and caching for go build in the container (Anagh Kumar Baranwal)
-* VFS
-    * Fix the cache failing to upload symlinks when `--links` was specified (Nick Craig-Wood)
-    * Fix race detected by race detector (Nick Craig-Wood)
-    * Close the change notify channel on Shutdown (izouxv)
-* B2
-    * Fix "fatal error: concurrent map writes" (Nick Craig-Wood)
-* Iclouddrive
-    * Add notes on ADP and Missing PCS cookies (Nick Craig-Wood)
-* Onedrive
-    * Mark German (de) region as deprecated (Nick Craig-Wood)
-* S3
-    * Added new storage class to magalu provider (Bruno Fernandes)
-    * Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe)
-    * Add latest Linode Object Storage endpoints (jbagwell-akamai)
-
 ## v1.69.0 - 2025-01-12
 
 [See commits](https://github.com/rclone/rclone/compare/v1.68.0...v1.69.0)

MANUAL.txt

@@ -1,75 +1,6 @@
 rclone(1) User Manual
 Nick Craig-Wood
-Feb 14, 2025
-
-NAME
-
-rclone - manage files on cloud storage
-
-SYNOPSIS
-
-    Usage:
-      rclone [flags]
-      rclone [command]
-
-    Available commands:
-      about       Get quota information from the remote.
-      authorize   Remote authorization.
-      backend     Run a backend-specific command.
-      bisync      Perform bidirectional synchronization between two paths.
-      cat         Concatenates any files and sends them to stdout.
-      check       Checks the files in the source and destination match.
-      checksum    Checks the files in the destination against a SUM file.
-      cleanup     Clean up the remote if possible.
-      completion  Output completion script for a given shell.
-      config      Enter an interactive configuration session.
-      copy        Copy files from source to dest, skipping identical files.
-      copyto      Copy files from source to dest, skipping identical files.
-      copyurl     Copy the contents of the URL supplied content to dest:path.
-      cryptcheck  Cryptcheck checks the integrity of an encrypted remote.
-      cryptdecode Cryptdecode returns unencrypted file names.
-      dedupe      Interactively find duplicate filenames and delete/rename them.
-      delete      Remove the files in path.
-      deletefile  Remove a single file from remote.
-      gendocs     Output markdown docs for rclone to the directory supplied.
-      gitannex    Speaks with git-annex over stdin/stdout.
-      hashsum     Produces a hashsum file for all the objects in the path.
-      help        Show help for rclone commands, flags and backends.
-      link        Generate public link to file/folder.
-      listremotes List all the remotes in the config file and defined in environment variables.
-      ls          List the objects in the path with size and path.
-      lsd         List all directories/containers/buckets in the path.
-      lsf         List directories and objects in remote:path formatted for parsing.
-      lsjson      List directories and objects in the path in JSON format.
-      lsl         List the objects in path with modification time, size and path.
-      md5sum      Produces an md5sum file for all the objects in the path.
-      mkdir       Make the path if it doesn't already exist.
-      mount       Mount the remote as file system on a mountpoint.
-      move        Move files from source to dest.
-      moveto      Move file or directory from source to dest.
-      ncdu        Explore a remote with a text based user interface.
-      nfsmount    Mount the remote as file system on a mountpoint.
-      obscure     Obscure password for use in the rclone config file.
-      purge       Remove the path and all of its contents.
-      rc          Run a command against a running rclone.
-      rcat        Copies standard input to file on remote.
-      rcd         Run rclone listening to remote control commands only.
-      rmdir       Remove the empty directory at path.
-      rmdirs      Remove empty directories under the path.
-      selfupdate  Update the rclone binary.
-      serve       Serve a remote over a protocol.
-      settier     Changes storage class/tier of objects in remote.
-      sha1sum     Produces an sha1sum file for all the objects in the path.
-      size        Prints the total size and number of objects in remote:path.
-      sync        Make source and dest identical, modifying destination only.
-      test        Run a test command
-      touch       Create new file or change file modification time.
-      tree        List the contents of the remote in a tree like fashion.
-      version     Show the version number.
-
-    Use "rclone [command] --help" for more information about a command.
-    Use "rclone help flags" for to see the global flags.
-    Use "rclone help backends" for a list of supported services.
+Jan 12, 2025
 
 Rclone syncs your files to cloud storage
 
@@ -1669,10 +1600,6 @@ include/exclude filters - everything will be removed. Use the delete
 command if you want to selectively delete files. To delete empty
 directories only, use command rmdir or rmdirs.
 
-The concurrency of this operation is controlled by the --checkers global
-flag. However, some backends will implement this command directly, in
-which case --checkers will be ignored.
-
 Important: Since this can cause data loss, test first with the --dry-run
 or the --interactive/-i flag.
 
@@ -3540,12 +3467,12 @@ re-encrypt the config.
 
 When --password-command is called to change the password then the
 environment variable RCLONE_PASSWORD_CHANGE=1 will be set. So if
-changing passwords programmatically you can use the environment variable
+changing passwords programatically you can use the environment variable
 to distinguish which password you must supply.
 
 Alternatively you can remove the password first (with
 rclone config encryption remove), then set it again with this command
-which may be easier if you don't mind the unencrypted config file being
+which may be easier if you don't mind the unecrypted config file being
 on the disk briefly.
 
     rclone config encryption set [flags]
@@ -4022,7 +3949,7 @@ there is one with the same name.
 Setting --stdout or making the output file name - will cause the output
 to be written to standard output.
 
-Troubleshooting
+Troublshooting
 
 If you can't get rclone copyurl to work then here are some things you
 can try:
@@ -10175,7 +10102,7 @@ uses an on disk cache, but the cache entries are held as symlinks.
 Rclone will use the handle of the underlying file as the NFS handle
 which improves performance. This sort of cache can't be backed up and
 restored as the underlying handles will change. This is Linux only. It
-requires running rclone as root or with CAP_DAC_READ_SEARCH. You can run
+requres running rclone as root or with CAP_DAC_READ_SEARCH. You can run
 rclone with this extra permission by doing this to the rclone binary
 sudo setcap cap_dac_read_search+ep /path/to/rclone.
 
@@ -10976,8 +10903,8 @@ which is defined like this:
     secret_access_key = SECRET_ACCESS_KEY
     use_multipart_uploads = false
 
-Note that setting use_multipart_uploads = false is to work around a bug
-which will be fixed in due course.
+Note that setting disable_multipart_uploads = true is to work around a
+bug which will be fixed in due course.
 
 Bugs
 
@@ -13968,10 +13895,6 @@ also possible to specify --boolean=false or --boolean=true. Note that
 --boolean false is not valid - this is parsed as --boolean and the false
 is parsed as an extra command line argument for rclone.
 
-Options documented to take a stringArray parameter accept multiple
-values. To pass more than one value, repeat the option; for example:
---include value1 --include value2.
-
 Time or duration options
 
 TIME or DURATION options can be specified as a duration string or a time
@@ -16254,7 +16177,7 @@ The options set by environment variables can be seen with the -vv flag,
 e.g. rclone version -vv.
 
 Options that can appear multiple times (type stringArray) are treated
-slightly differently as environment variables can only be defined once.
+slighly differently as environment variables can only be defined once.
 In order to allow a simple mechanism for adding one or many items, the
 input is treated as a CSV encoded string. For example
 
@@ -19497,7 +19420,7 @@ This is only useful if --vfs-cache-mode > off. If you call it when the
        ],
     }
 
-The expiry time is the time until the file is eligible for being
+The expiry time is the time until the file is elegible for being
 uploaded in floating point seconds. This may go negative. As rclone only
 transfers --transfers files at once, only the lowest --transfers expiry
 times will have uploading as true. So there may be files with negative
@@ -20646,7 +20569,7 @@ Flags for general networking and HTTP stuff.
           --tpslimit float                     Limit HTTP transactions per second to this
           --tpslimit-burst int                 Max burst of transactions for --tpslimit (default 1)
           --use-cookies                        Enable session cookiejar
-          --user-agent string                  Set the user-agent to a specified string (default "rclone/v1.69.1")
+          --user-agent string                  Set the user-agent to a specified string (default "rclone/v1.69.0")
 
 Performance
 
@@ -22608,7 +22531,7 @@ Also see the all files changed check.
 By using rclone filter features you can exclude file types or directory
 sub-trees from the sync. See the bisync filters section and generic
 --filter-from documentation. An example filters file contains filters
-for non-allowed files for syncing with Dropbox.
+for non-allowed files for synching with Dropbox.
 
 If you make changes to your filters file then bisync requires a run with
 --resync. This is a safety feature, which prevents existing files on the
@@ -22781,7 +22704,7 @@ of a sync. Using --check-sync=false will disable it and may
 significantly reduce the sync run times for very large numbers of files.
 
 The check may be run manually with --check-sync=only. It runs only the
-integrity check and terminates without actually syncing.
+integrity check and terminates without actually synching.
 
 Note that currently, --check-sync only checks listing snapshots and NOT
 the actual files on the remotes. Note also that the listing snapshots
@@ -23314,7 +23237,7 @@ supported.
 How to filter directories
 
 Filtering portions of the directory tree is a critical feature for
-syncing.
+synching.
 
 Examples of directory trees (always beneath the Path1/Path2 root level)
 you may want to exclude from your sync: - Directory trees containing
@@ -23425,7 +23348,7 @@ This noise can be quashed by adding --quiet to the bisync command line.
 
 Example exclude-style filters files for use with Dropbox
 
--   Dropbox disallows syncing the listed temporary and
+-   Dropbox disallows synching the listed temporary and
     configuration/data files. The `- ` filters exclude these files where
     ever they may occur in the sync tree. Consider adding similar
     exclusions for file types you don't need to sync, such as core dump
@@ -23745,7 +23668,7 @@ dash.
 Running tests
 
 -   go test . -case basic -remote local -remote2 local runs the
-    test_basic test case using only the local filesystem, syncing one
+    test_basic test case using only the local filesystem, synching one
     local directory with another local directory. Test script output is
     to the console, while commands within scenario.txt have their output
     sent to the .../workdir/test.log file, which is finally compared to
@@ -23978,11 +23901,6 @@ Unison and synchronization in general.
 
 Changelog
 
-v1.69.1
-
--   Fixed an issue causing listings to not capture concurrent
-    modifications under certain conditions
-
 v1.68
 
 -   Fixed an issue affecting backends that round modtimes to a lower
@@ -25274,7 +25192,7 @@ Notes on above:
     that USER_NAME has been created.
 2.  The Resource entry must include both resource ARNs, as one implies
     the bucket and the other implies the bucket's objects.
-3.  When using s3-no-check-bucket and the bucket already exists, the
+3.  When using s3-no-check-bucket and the bucket already exsits, the
     "arn:aws:s3:::BUCKET_NAME" doesn't have to be included.
 
 For reference, here's an Ansible script that will generate one or more
@@ -28237,8 +28155,8 @@ this:
     secret_access_key = SECRET_ACCESS_KEY
     use_multipart_uploads = false
 
-Note that setting use_multipart_uploads = false is to work around a bug
-which will be fixed in due course.
+Note that setting disable_multipart_uploads = true is to work around a
+bug which will be fixed in due course.
 
 Scaleway
 
@@ -29285,49 +29203,27 @@ This will guide you through an interactive setup process.
     Endpoint for Linode Object Storage API.
     Choose a number from below, or type in your own value.
     Press Enter to leave empty.
-     1 / Amsterdam (Netherlands), nl-ams-1
-       \ (nl-ams-1.linodeobjects.com)
-     2 / Atlanta, GA (USA), us-southeast-1
+     1 / Atlanta, GA (USA), us-southeast-1
        \ (us-southeast-1.linodeobjects.com)
-     3 / Chennai (India), in-maa-1
-       \ (in-maa-1.linodeobjects.com)
-     4 / Chicago, IL (USA), us-ord-1
+     2 / Chicago, IL (USA), us-ord-1
        \ (us-ord-1.linodeobjects.com)
-     5 / Frankfurt (Germany), eu-central-1
+     3 / Frankfurt (Germany), eu-central-1
        \ (eu-central-1.linodeobjects.com)
-     6 / Jakarta (Indonesia), id-cgk-1
-       \ (id-cgk-1.linodeobjects.com)
-     7 / London 2 (Great Britain), gb-lon-1
-       \ (gb-lon-1.linodeobjects.com)
-     8 / Los Angeles, CA (USA), us-lax-1
-       \ (us-lax-1.linodeobjects.com)
-     9 / Madrid (Spain), es-mad-1
-       \ (es-mad-1.linodeobjects.com)
-    10 / Melbourne (Australia), au-mel-1
-       \ (au-mel-1.linodeobjects.com)
-    11 / Miami, FL (USA), us-mia-1
-       \ (us-mia-1.linodeobjects.com)
-    12 / Milan (Italy), it-mil-1
+     4 / Milan (Italy), it-mil-1
        \ (it-mil-1.linodeobjects.com)
-    13 / Newark, NJ (USA), us-east-1
+     5 / Newark, NJ (USA), us-east-1
        \ (us-east-1.linodeobjects.com)
-    14 / Osaka (Japan), jp-osa-1
-       \ (jp-osa-1.linodeobjects.com)
-    15 / Paris (France), fr-par-1
+     6 / Paris (France), fr-par-1
        \ (fr-par-1.linodeobjects.com)
-    16 / São Paulo (Brazil), br-gru-1
-       \ (br-gru-1.linodeobjects.com)
-    17 / Seattle, WA (USA), us-sea-1
+     7 / Seattle, WA (USA), us-sea-1
        \ (us-sea-1.linodeobjects.com)
-    18 / Singapore, ap-south-1
+     8 / Singapore ap-south-1
        \ (ap-south-1.linodeobjects.com)
-    19 / Singapore 2, sg-sin-1
-       \ (sg-sin-1.linodeobjects.com)
-    20 / Stockholm (Sweden), se-sto-1
+     9 / Stockholm (Sweden), se-sto-1
        \ (se-sto-1.linodeobjects.com)
-    21 / Washington, DC, (USA), us-iad-1
+    10 / Washington, DC, (USA), us-iad-1
        \ (us-iad-1.linodeobjects.com)
-    endpoint> 5
+    endpoint> 3
 
     Option acl.
     Canned ACL used when creating buckets and storing or copying objects.
@@ -33861,7 +33757,7 @@ The initial nonce is generated from the operating systems crypto strong
 random number generator. The nonce is incremented for each chunk read
 making sure each nonce is unique for each block written. The chance of a
 nonce being reused is minuscule. If you wrote an exabyte of data (10¹⁸
-bytes) you would have a probability of approximately 2×10⁻³² of reusing
+bytes) you would have a probability of approximately 2×10⁻³² of re-using
 a nonce.
 
 Chunk
@@ -41082,7 +40978,7 @@ This will guide you through an interactive setup process:
     config_2fa> 2FACODE
     Remote config
     --------------------
-    [iclouddrive]
+    [koofr]
     - type: iclouddrive
     - apple_id: APPLEID
     - password: *** ENCRYPTED ***
@@ -41098,27 +40994,6 @@ Advanced Data Protection
 
 ADP is currently unsupported and need to be disabled
 
-On iPhone, Settings > Apple Account > iCloud > 'Access iCloud Data on
-the Web' must be ON, and 'Advanced Data Protection' OFF.
-
-Troubleshooting
-
-Missing PCS cookies from the request
-
-This means you have Advanced Data Protection (ADP) turned on. This is
-not supported at the moment. If you want to use rclone you will have to
-turn it off. See above for how to turn it off.
-
-You will need to clear the cookies and the trust_token fields in the
-config. Or you can delete the remote config and start again.
-
-You should then run rclone reconnect remote:.
-
-Note that changing the ADP setting may not take effect immediately - you
-may need to wait a few hours or a day before you can get rclone to work
-- keep clearing the config entry and running rclone reconnect remote:
-until rclone functions properly.
-
 Standard options
 
 Here are the Standard options specific to iclouddrive (iCloud Drive).
@@ -45714,8 +45589,7 @@ Properties:
     -   "us"
         -   Microsoft Cloud for US Government
     -   "de"
-        -   Microsoft Cloud Germany (deprecated - try global region
-            first).
+        -   Microsoft Cloud Germany
     -   "cn"
         -   Azure and Office 365 operated by Vnet Group in China
 
@@ -46374,38 +46248,6 @@ Here are the possible system metadata items for the onedrive backend.
 
 See the metadata docs for more info.
 
-Impersonate other users as Admin
-
-Unlike Google Drive and impersonating any domain user via service
-accounts, OneDrive requires you to authenticate as an admin account, and
-manually setup a remote per user you wish to impersonate.
-
-1.  In Microsoft 365 Admin Center, open each user you need to
-    "impersonate" and go to the OneDrive section. There is a heading
-    called "Get access to files", you need to click to create the link,
-    this creates the link of the format
-    https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/
-    but also changes the permissions so you your admin user has access.
-2.  Then in powershell run the following commands:
-
-    Install-Module Microsoft.Graph -Scope CurrentUser -Repository PSGallery -Force
-    Import-Module Microsoft.Graph.Files
-    Connect-MgGraph -Scopes "Files.ReadWrite.All"
-    # Follow the steps to allow access to your admin user
-    # Then run this for each user you want to impersonate to get the Drive ID
-    Get-MgUserDefaultDrive -UserId '{emailaddress}'
-    # This will give you output of the format:
-    # Name     Id                                                                 DriveType CreatedDateTime
-    # ----     --                                                                 --------- ---------------
-    # OneDrive b!XYZ123                                                           business  14/10/2023 1:00:58 pm
-
-3.  Then in rclone add a onedrive remote type, and use the
-    Type in driveID with the DriveID you got in the previous step. One
-    remote per user. It will then confirm the drive ID, and hopefully
-    give you a message of Found drive "root" of type "business" and then
-    include the URL of the format
-    https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/Documents
-
 Limitations
 
 If you don't use rclone for 90 days the refresh token will expire. This
@@ -56315,37 +56157,6 @@ Options:
 
 Changelog
 
-v1.69.1 - 2025-02-14
-
-See commits
-
--   Bug Fixes
-    -   lib/oauthutil: Fix redirect URL mismatch errors (Nick
-        Craig-Wood)
-    -   bisync: Fix listings missing concurrent modifications (nielash)
-    -   serve s3: Fix list objects encoding-type (Nick Craig-Wood)
-    -   fs: Fix confusing "didn't find section in config file" error
-        (Nick Craig-Wood)
-    -   doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt
-        Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies)
-    -   build: Added parallel docker builds and caching for go build in
-        the container (Anagh Kumar Baranwal)
--   VFS
-    -   Fix the cache failing to upload symlinks when --links was
-        specified (Nick Craig-Wood)
-    -   Fix race detected by race detector (Nick Craig-Wood)
-    -   Close the change notify channel on Shutdown (izouxv)
--   B2
-    -   Fix "fatal error: concurrent map writes" (Nick Craig-Wood)
--   Iclouddrive
-    -   Add notes on ADP and Missing PCS cookies (Nick Craig-Wood)
--   Onedrive
-    -   Mark German (de) region as deprecated (Nick Craig-Wood)
--   S3
-    -   Added new storage class to magalu provider (Bruno Fernandes)
-    -   Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe)
-    -   Add latest Linode Object Storage endpoints (jbagwell-akamai)
-
 v1.69.0 - 2025-01-12
 
 See commits

README.md

@@ -1,20 +1,4 @@
-<div align="center">
-<sup>Special thanks to our sponsor:</sup>
-<br>
-<br>
-<a href="https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103">
-  <div>
-    <img src="https://rclone.org/img/logos/warp-github.svg" width="300" alt="Warp">
-  </div>
-  <b>Warp is a modern, Rust-based terminal with AI built in so you and your team can build great software, faster.</b>
-  <div>
-    <sup>Visit warp.dev to learn more.</sup>
-  </div>
-</a>
-<br>
-<hr>
-</div>
-<br>
+
 
 [<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
 [<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)

RELEASE.md

@@ -47,13 +47,20 @@ Early in the next release cycle update the dependencies.
   * `git commit -a -v -m "build: update all dependencies"`
 
 If the `make updatedirect` upgrades the version of go in the `go.mod`
-then go to manual mode. `go1.20` here is the lowest supported version
+
+    go 1.22.0
+    
+then go to manual mode. `go1.22` here is the lowest supported version
 in the `go.mod`.
 
+If `make updatedirect` added a `toolchain` directive then remove it.
+We don't want to force a toolchain on our users. Linux packagers are
+often using a version of Go that is a few versions out of date.
+
 ```
 go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
 go get -d $(cat /tmp/potential-upgrades)
-go mod tidy -go=1.20 -compat=1.20
+go mod tidy -go=1.22 -compat=1.22
 ```
 
 If the `go mod tidy` fails use the output from it to remove the
@@ -124,8 +131,8 @@ Now
 
   * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
-  * Do the steps as above
   * make startstable
+  * Do the steps as above
   * git co master
   * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
   * git checkout ${BASE_TAG}-stable docs/content/changelog.md

VERSION

@@ -1 +1 @@
-v1.69.1
+v1.70.0

backend/azureblob/azureblob_internal_test.go

@@ -3,16 +3,149 @@
 package azureblob
 
 import (
+	"context"
+	"encoding/base64"
+	"strings"
 	"testing"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func (f *Fs) InternalTest(t *testing.T) {
-	// Check first feature flags are set on this
-	// remote
+func TestBlockIDCreator(t *testing.T) {
+	// Check creation and random number
+	bic, err := newBlockIDCreator()
+	require.NoError(t, err)
+	bic2, err := newBlockIDCreator()
+	require.NoError(t, err)
+	assert.NotEqual(t, bic.random, bic2.random)
+	assert.NotEqual(t, bic.random, [8]byte{})
+
+	// Set random to known value for tests
+	bic.random = [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
+	chunkNumber := uint64(0xFEDCBA9876543210)
+
+	// Check creation of ID
+	want := base64.StdEncoding.EncodeToString([]byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, 1, 2, 3, 4, 5, 6, 7, 8})
+	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", want)
+	got := bic.newBlockID(chunkNumber)
+	assert.Equal(t, want, got)
+	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", got)
+
+	// Test checkID is working
+	assert.NoError(t, bic.checkID(chunkNumber, got))
+	assert.ErrorContains(t, bic.checkID(chunkNumber, "$"+got), "illegal base64")
+	assert.ErrorContains(t, bic.checkID(chunkNumber, "AAAA"+got), "bad block ID length")
+	assert.ErrorContains(t, bic.checkID(chunkNumber+1, got), "expecting decoded")
+	assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
+}
+
+func (f *Fs) testFeatures(t *testing.T) {
+	// Check first feature flags are set on this remote
 	enabled := f.Features().SetTier
 	assert.True(t, enabled)
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
+
+type ReadSeekCloser struct {
+	*strings.Reader
+}
+
+func (r *ReadSeekCloser) Close() error {
+	return nil
+}
+
+// Stage a block at remote but don't commit it
+func (f *Fs) stageBlockWithoutCommit(ctx context.Context, t *testing.T, remote string) {
+	var (
+		containerName, blobPath = f.split(remote)
+		containerClient         = f.cntSVC(containerName)
+		blobClient              = containerClient.NewBlockBlobClient(blobPath)
+		data                    = "uncommitted data"
+		blockID                 = "1"
+		blockIDBase64           = base64.StdEncoding.EncodeToString([]byte(blockID))
+	)
+	r := &ReadSeekCloser{strings.NewReader(data)}
+	_, err := blobClient.StageBlock(ctx, blockIDBase64, r, nil)
+	require.NoError(t, err)
+
+	// Verify the block is staged but not committed
+	blockList, err := blobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
+	require.NoError(t, err)
+	found := false
+	for _, block := range blockList.UncommittedBlocks {
+		if *block.Name == blockIDBase64 {
+			found = true
+			break
+		}
+	}
+	require.True(t, found, "Block ID not found in uncommitted blocks")
+}
+
+// This tests uploading a blob where it has uncommitted blocks with a different ID size.
+//
+// https://gauravmantri.com/2013/05/18/windows-azure-blob-storage-dealing-with-the-specified-blob-or-block-content-is-invalid-error/
+//
+// TestIntegration/FsMkdir/FsPutFiles/Internal/WriteUncommittedBlocks
+func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
+	var (
+		ctx    = context.Background()
+		remote = "testBlob"
+	)
+
+	// Multipart copy the blob please
+	oldUseCopyBlob, oldCopyCutoff := f.opt.UseCopyBlob, f.opt.CopyCutoff
+	f.opt.UseCopyBlob = false
+	f.opt.CopyCutoff = f.opt.ChunkSize
+	defer func() {
+		f.opt.UseCopyBlob, f.opt.CopyCutoff = oldUseCopyBlob, oldCopyCutoff
+	}()
+
+	// Create a blob with uncommitted blocks
+	f.stageBlockWithoutCommit(ctx, t, remote)
+
+	// Now attempt to overwrite the block with a different sized block ID to provoke this error
+
+	// Check the object does not exist
+	_, err := f.NewObject(ctx, remote)
+	require.Equal(t, fs.ErrorObjectNotFound, err)
+
+	// Upload a multipart file over the block with uncommitted chunks of a different ID size
+	size := 4*int(f.opt.ChunkSize) - 1
+	contents := random.String(size)
+	item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
+	o := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+
+	// Check size
+	assert.Equal(t, int64(size), o.Size())
+
+	// Create a new blob with uncommitted blocks
+	newRemote := "testBlob2"
+	f.stageBlockWithoutCommit(ctx, t, newRemote)
+
+	// Copy over that block
+	dst, err := f.Copy(ctx, o, newRemote)
+	require.NoError(t, err)
+
+	// Check basics
+	assert.Equal(t, int64(size), dst.Size())
+	assert.Equal(t, newRemote, dst.Remote())
+
+	// Check contents
+	gotContents := fstests.ReadObject(ctx, t, dst, -1)
+	assert.Equal(t, contents, gotContents)
+
+	// Remove the object
+	require.NoError(t, dst.Remove(ctx))
+}
+
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("Features", f.testFeatures)
+	t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
+}

backend/azureblob/azureblob_test.go

@@ -15,13 +15,17 @@ import (
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
+	name := "TestAzureBlob"
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "TestAzureBlob:",
+		RemoteName:  name + ":",
 		NilObject:   (*Object)(nil),
 		TiersToTest: []string{"Hot", "Cool", "Cold"},
 		ChunkedUpload: fstests.ChunkedUploadConfig{
 			MinChunkSize: defaultChunkSize,
 		},
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "use_copy_blob", Value: "false"},
+		},
 	})
 }
 
@@ -40,6 +44,7 @@ func TestIntegration2(t *testing.T) {
 		},
 		ExtraConfig: []fstests.ExtraConfigItem{
 			{Name: name, Key: "directory_markers", Value: "true"},
+			{Name: name, Key: "use_copy_blob", Value: "false"},
 		},
 	})
 }
@@ -48,8 +53,13 @@ func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadChunkSize(cs)
 }
 
+func (f *Fs) SetCopyCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setCopyCutoff(cs)
+}
+
 var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetCopyCutoffer     = (*Fs)(nil)
 )
 
 func TestValidateAccessTier(t *testing.T) {

backend/azurefiles/azurefiles.go

@@ -237,6 +237,30 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Help:      "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
 			Advanced:  true,
 			Sensitive: true,
+		}, {
+			Name: "disable_instance_discovery",
+			Help: `Skip requesting Microsoft Entra instance metadata
+This should be set true only by applications authenticating in
+disconnected clouds, or private clouds such as Azure Stack.
+It determines whether rclone requests Microsoft Entra instance
+metadata from ` + "`https://login.microsoft.com/`" + ` before
+authenticating.
+Setting this to true will skip this request, making you responsible
+for ensuring the configured authority is valid and trustworthy.
+`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name: "use_az",
+			Help: `Use Azure CLI tool az for authentication
+Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
+as the sole means of authentication.
+Setting this can be useful if you wish to use the az CLI on a host with
+a System Managed Identity that you do not want to use.
+Don't set env_auth at the same time.
+`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",
@@ -319,10 +343,12 @@ type Options struct {
 	Username                   string               `config:"username"`
 	Password                   string               `config:"password"`
 	ServicePrincipalFile       string               `config:"service_principal_file"`
+	DisableInstanceDiscovery   bool                 `config:"disable_instance_discovery"`
 	UseMSI                     bool                 `config:"use_msi"`
 	MSIObjectID                string               `config:"msi_object_id"`
 	MSIClientID                string               `config:"msi_client_id"`
 	MSIResourceID              string               `config:"msi_mi_res_id"`
+	UseAZ                      bool                 `config:"use_az"`
 	Endpoint                   string               `config:"endpoint"`
 	ChunkSize                  fs.SizeSuffix        `config:"chunk_size"`
 	MaxStreamSize              fs.SizeSuffix        `config:"max_stream_size"`
@@ -414,7 +440,8 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		}
 		// Read credentials from the environment
 		options := azidentity.DefaultAzureCredentialOptions{
-			ClientOptions: policyClientOptions,
+			ClientOptions:            policyClientOptions,
+			DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
 		}
 		cred, err = azidentity.NewDefaultAzureCredential(&options)
 		if err != nil {
@@ -425,6 +452,13 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		if err != nil {
 			return nil, fmt.Errorf("create new shared key credential failed: %w", err)
 		}
+	case opt.UseAZ:
+		var options = azidentity.AzureCLICredentialOptions{}
+		cred, err = azidentity.NewAzureCLICredential(&options)
+		fmt.Println(cred)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
+		}
 	case opt.SASURL != "":
 		client, err = service.NewClientWithNoCredential(opt.SASURL, &clientOpt)
 		if err != nil {

backend/azurefiles/azurefiles_internal_test.go

@@ -61,7 +61,7 @@ const chars = "abcdefghijklmnopqrstuvwzyxABCDEFGHIJKLMNOPQRSTUVWZYX"
 
 func randomString(charCount int) string {
 	strBldr := strings.Builder{}
-	for i := 0; i < charCount; i++ {
+	for range charCount {
 		randPos := rand.Int63n(52)
 		strBldr.WriteByte(chars[randPos])
 	}

backend/b2/api/types.go

@@ -130,10 +130,10 @@ type AuthorizeAccountResponse struct {
 	AbsoluteMinimumPartSize int      `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file.
 	AccountID               string   `json:"accountId"`               // The identifier for the account.
 	Allowed                 struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
-		BucketID     string      `json:"bucketId"`     // When present, access is restricted to one bucket.
-		BucketName   string      `json:"bucketName"`   // When present, name of bucket - may be empty
-		Capabilities []string    `json:"capabilities"` // A list of strings, each one naming a capability the key has.
-		NamePrefix   interface{} `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
+		BucketID     string   `json:"bucketId"`     // When present, access is restricted to one bucket.
+		BucketName   string   `json:"bucketName"`   // When present, name of bucket - may be empty
+		Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.
+		NamePrefix   any      `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
 	} `json:"allowed"`
 	APIURL              string `json:"apiUrl"`              // The base URL to use for all API calls except for uploading and downloading files.
 	AuthorizationToken  string `json:"authorizationToken"`  // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.

backend/b2/b2.go

@@ -16,6 +16,7 @@ import (
 	"io"
 	"net/http"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -30,7 +31,8 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/multipart"
@@ -588,12 +590,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
 
 // hasPermission returns if the current AuthorizationToken has the selected permission
 func (f *Fs) hasPermission(permission string) bool {
-	for _, capability := range f.info.Allowed.Capabilities {
-		if capability == permission {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(f.info.Allowed.Capabilities, permission)
 }
 
 // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
@@ -921,7 +918,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // of listing recursively that doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
 	bucket, directory := f.split(dir)
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	listR := func(bucket, directory, prefix string, addBucket bool) error {
 		last := ""
 		return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
@@ -1274,7 +1271,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 	toBeDeleted := make(chan *api.File, f.ci.Transfers)
 	var wg sync.WaitGroup
 	wg.Add(f.ci.Transfers)
-	for i := 0; i < f.ci.Transfers; i++ {
+	for range f.ci.Transfers {
 		go func() {
 			defer wg.Done()
 			for object := range toBeDeleted {
@@ -1317,16 +1314,22 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 				// Check current version of the file
 				if deleteHidden && object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove hide marker") {
+						toBeDeleted <- object
+					}
 				} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove pending upload") {
+						toBeDeleted <- object
+					}
 				} else {
 					fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))
 				}
 			} else {
 				fs.Debugf(remote, "Deleting (id %q)", object.ID)
-				toBeDeleted <- object
+				if !operations.SkipDestructive(ctx, object.Name, "delete") {
+					toBeDeleted <- object
+				}
 			}
 			last = remote
 			tr.Done(ctx, nil)
@@ -1932,7 +1935,7 @@ func init() {
 // urlEncode encodes in with % encoding
 func urlEncode(in string) string {
 	var out bytes.Buffer
-	for i := 0; i < len(in); i++ {
+	for i := range len(in) {
 		c := in[i]
 		if noNeedToEncode[c] {
 			_ = out.WriteByte(c)
@@ -2253,7 +2256,7 @@ See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
 	},
 }
 
-func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	var newRule api.LifecycleRule
 	if daysStr := opt["daysFromHidingToDeleting"]; daysStr != "" {
 		days, err := strconv.Atoi(daysStr)
@@ -2282,8 +2285,10 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 
 	}
 
+	skip := operations.SkipDestructive(ctx, name, "update lifecycle rules")
+
 	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil {
+	if !skip && (newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil) {
 		bucketID, err := f.getBucketID(ctx, bucketName)
 		if err != nil {
 			return nil, err
@@ -2340,7 +2345,7 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
 	},
 }
 
-func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	maxAge := defaultMaxAge
 	if opt["max-age"] != "" {
 		maxAge, err = fs.ParseDuration(opt["max-age"])
@@ -2363,7 +2368,7 @@ it would do.
 `,
 }
 
-func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	return nil, f.cleanUp(ctx, true, false, 0)
 }
 
@@ -2382,7 +2387,7 @@ var commandHelp = []fs.CommandHelp{
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "lifecycle":
 		return f.lifecycleCommand(ctx, name, arg, opt)

backend/b2/b2_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/sha1"
 	"fmt"
 	"path"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -13,6 +14,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/bucket"
@@ -457,24 +459,161 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	})
 
 	t.Run("Cleanup", func(t *testing.T) {
-		require.NoError(t, f.cleanUp(ctx, true, false, 0))
-		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
-		fstest.CheckListing(t, f, items)
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-		fstest.CheckListing(t, f, items)
+		t.Run("DryRun", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should be unchanged after dry run
+			before := listAllFiles(ctx, t, f, dirName)
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			after := listAllFiles(ctx, t, f, dirName)
+			assert.Equal(t, before, after)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should reflect current state after cleanup
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
+			fstest.CheckListing(t, f, items)
+		})
 	})
 
 	// Purge gets tested later
 }
 
+func (f *Fs) InternalTestCleanupUnfinished(t *testing.T) {
+	ctx := context.Background()
+
+	// B2CleanupHidden tests cleaning up hidden files
+	t.Run("CleanupUnfinished", func(t *testing.T) {
+		dirName := "unfinished"
+		fileCount := 5
+		expectedFiles := []string{}
+		for i := 1; i < fileCount; i++ {
+			fileName := fmt.Sprintf("%s/unfinished-%d", dirName, i)
+			expectedFiles = append(expectedFiles, fileName)
+			obj := &Object{
+				fs:     f,
+				remote: fileName,
+			}
+			objInfo := object.NewStaticObjectInfo(fileName, fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)
+			_, err := f.newLargeUpload(ctx, obj, nil, objInfo, f.opt.ChunkSize, false, nil)
+			require.NoError(t, err)
+		}
+		checkListing(ctx, t, f, dirName, expectedFiles)
+
+		t.Run("DryRun", func(t *testing.T) {
+			// Listing should not change after dry run
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, expectedFiles)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			// Listing should be empty after real cleanup
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, []string{})
+		})
+	})
+}
+
+func listAllFiles(ctx context.Context, t *testing.T, f *Fs, dirName string) []string {
+	bucket, directory := f.split(dirName)
+	foundFiles := []string{}
+	require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
+		if !isDirectory {
+			foundFiles = append(foundFiles, object.Name)
+		}
+		return nil
+	}))
+	sort.Strings(foundFiles)
+	return foundFiles
+}
+
+func checkListing(ctx context.Context, t *testing.T, f *Fs, dirName string, expectedFiles []string) {
+	foundFiles := listAllFiles(ctx, t, f, dirName)
+	sort.Strings(expectedFiles)
+	assert.Equal(t, expectedFiles, foundFiles)
+}
+
+func (f *Fs) InternalTestLifecycleRules(t *testing.T) {
+	ctx := context.Background()
+
+	opt := map[string]string{}
+
+	t.Run("InitState", func(t *testing.T) {
+		// There should be no lifecycle rules at the outset
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("DryRun", func(t *testing.T) {
+		// There should still be no lifecycle rules after each dry run operation
+		ctx, ci := fs.AddConfig(ctx)
+		ci.DryRun = true
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("RealThing", func(t *testing.T) {
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+	})
+}
+
 // -run TestIntegration/FsMkdir/FsPutFiles/Internal
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Metadata", f.InternalTestMetadata)
 	t.Run("Versions", f.InternalTestVersions)
+	t.Run("CleanupUnfinished", f.InternalTestCleanupUnfinished)
+	t.Run("LifecycleRules", f.InternalTestLifecycleRules)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/b2/upload.go

@@ -478,17 +478,14 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
 		remaining = up.size
 	)
 	g.SetLimit(up.f.opt.UploadConcurrency)
-	for part := 0; part < up.parts; part++ {
+	for part := range up.parts {
 		// Fail fast, in case an errgroup managed function returns an error
 		// gCtx is cancelled. There is no point in copying all the other parts.
 		if gCtx.Err() != nil {
 			break
 		}
 
-		reqSize := remaining
-		if reqSize >= up.chunkSize {
-			reqSize = up.chunkSize
-		}
+		reqSize := min(remaining, up.chunkSize)
 
 		part := part // for the closure
 		g.Go(func() (err error) {

backend/box/box.go

@@ -237,8 +237,8 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
 	return claims, nil
 }
 
-func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]interface{} {
-	signingHeaders := map[string]interface{}{
+func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]any {
+	signingHeaders := map[string]any{
 		"kid": boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
 	}
 	return signingHeaders
@@ -1343,12 +1343,8 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 	nextStreamPosition = streamPosition
 
 	for {
-		limit := f.opt.ListChunk
-
 		// box only allows a max of 500 events
-		if limit > 500 {
-			limit = 500
-		}
+		limit := min(f.opt.ListChunk, 500)
 
 		opts := rest.Opts{
 			Method:     "GET",

backend/box/upload.go

@@ -105,7 +105,7 @@ func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api
 	const defaultDelay = 10
 	var tries int
 outer:
-	for tries = 0; tries < maxTries; tries++ {
+	for tries = range maxTries {
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
 			if err != nil {
@@ -203,7 +203,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	errs := make(chan error, 1)
 	var wg sync.WaitGroup
 outer:
-	for part := 0; part < session.TotalParts; part++ {
+	for part := range session.TotalParts {
 		// Check any errors
 		select {
 		case err = <-errs:
@@ -211,10 +211,7 @@ outer:
 		default:
 		}
 
-		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
-		}
+		reqSize := min(remaining, chunkSize)
 
 		// Make a block of memory
 		buf := make([]byte, reqSize)

backend/cache/cache.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/atexit"
@@ -1086,13 +1087,13 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	return cachedEntries, nil
 }
 
-func (f *Fs) recurse(ctx context.Context, dir string, list *walk.ListRHelper) error {
+func (f *Fs) recurse(ctx context.Context, dir string, list *list.Helper) error {
 	entries, err := f.List(ctx, dir)
 	if err != nil {
 		return err
 	}
 
-	for i := 0; i < len(entries); i++ {
+	for i := range entries {
 		innerDir, ok := entries[i].(fs.Directory)
 		if ok {
 			err := f.recurse(ctx, innerDir.Remote(), list)
@@ -1138,7 +1139,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	}
 
 	// if we're here, we're gonna do a standard recursive traversal and cache everything
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	err = f.recurse(ctx, dir, list)
 	if err != nil {
 		return err
@@ -1428,7 +1429,7 @@ func (f *Fs) cacheReader(u io.Reader, src fs.ObjectInfo, originalRead func(inn i
 	}()
 
 	// wait until both are done
-	for c := 0; c < 2; c++ {
+	for range 2 {
 		<-done
 	}
 }
@@ -1753,7 +1754,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 }
 
 // Stats returns stats about the cache storage
-func (f *Fs) Stats() (map[string]map[string]interface{}, error) {
+func (f *Fs) Stats() (map[string]map[string]any, error) {
 	return f.cache.Stats()
 }
 
@@ -1933,7 +1934,7 @@ var commandHelp = []fs.CommandHelp{
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (any, error) {
 	switch name {
 	case "stats":
 		return f.Stats()

backend/cache/cache_internal_test.go

@@ -360,7 +360,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
-	for i := 0; i < len(checkSample); i++ {
+	for i := range checkSample {
 		require.Equal(t, testData[i], checkSample[i])
 	}
 }
@@ -387,7 +387,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 	readData, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
 	require.NoError(t, err)
-	for i := 0; i < len(readData); i++ {
+	for i := range readData {
 		require.Equalf(t, testData[i], readData[i], "at byte %v", i)
 	}
 }
@@ -688,7 +688,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 	co, ok := o.(*cache.Object)
 	require.True(t, ok)
 
-	for i := 0; i < 4; i++ { // read first 4
+	for i := range 4 { // read first 4
 		_ = runInstance.readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false)
 	}
 	cfs.CleanUpCache(true)
@@ -971,7 +971,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	f, err := os.CreateTemp("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
-	for i := 0; i < int(cnt); i++ {
+	for range int(cnt) {
 		data := randStringBytes(int(chunk))
 		_, _ = f.Write(data)
 	}
@@ -1085,9 +1085,9 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
 	return err
 }
 
-func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error) {
+func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]any, error) {
 	var err error
-	var l []interface{}
+	var l []any
 	var list fs.DirEntries
 	list, err = f.List(context.Background(), remote)
 	for _, ll := range list {
@@ -1215,7 +1215,7 @@ func (r *run) listenForBackgroundUpload(t *testing.T, f fs.Fs, remote string) ch
 		var err error
 		var state cache.BackgroundUploadState
 
-		for i := 0; i < 2; i++ {
+		for range 2 {
 			select {
 			case state = <-buCh:
 				// continue
@@ -1293,7 +1293,7 @@ func (r *run) completeAllBackgroundUploads(t *testing.T, f fs.Fs, lastRemote str
 
 func (r *run) retryBlock(block func() error, maxRetries int, rate time.Duration) error {
 	var err error
-	for i := 0; i < maxRetries; i++ {
+	for range maxRetries {
 		err = block()
 		if err == nil {
 			return nil

backend/cache/cache_test.go

@@ -17,7 +17,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                      "TestCache:",
 		NilObject:                       (*cache.Object)(nil),
-		UnimplementableFsMethods:        []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter", "DirSetModTime", "MkdirMetadata"},
+		UnimplementableFsMethods:        []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter", "DirSetModTime", "MkdirMetadata", "ListP"},
 		UnimplementableObjectMethods:    []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata", "SetMetadata"},
 		UnimplementableDirectoryMethods: []string{"Metadata", "SetMetadata", "SetModTime"},
 		SkipInvalidUTF8:                 true, // invalid UTF-8 confuses the cache

backend/cache/cache_upload_test.go

@@ -162,7 +162,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	randInstance := rand.New(rand.NewSource(time.Now().Unix()))
 
 	lastFile := ""
-	for i := 0; i < totalFiles; i++ {
+	for i := range totalFiles {
 		size := int64(randInstance.Intn(maxSize-minSize) + minSize)
 		testReader := runInstance.randomReader(t, size)
 		remote := "test/" + strconv.Itoa(i) + ".bin"

backend/cache/handle.go

@@ -182,7 +182,7 @@ func (r *Handle) queueOffset(offset int64) {
 			}
 		}
 
-		for i := 0; i < r.workers; i++ {
+		for i := range r.workers {
 			o := r.preloadOffset + int64(r.cacheFs().opt.ChunkSize)*int64(i)
 			if o < 0 || o >= r.cachedObject.Size() {
 				continue
@@ -222,7 +222,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	if !found {
 		// we're gonna give the workers a chance to pickup the chunk
 		// and retry a couple of times
-		for i := 0; i < r.cacheFs().opt.ReadRetries*8; i++ {
+		for i := range r.cacheFs().opt.ReadRetries * 8 {
 			data, err = r.storage().GetChunk(r.cachedObject, chunkStart)
 			if err == nil {
 				found = true

backend/cache/plex.go

@@ -209,7 +209,7 @@ func (p *plexConnector) authenticate() error {
 	if err != nil {
 		return err
 	}
-	var data map[string]interface{}
+	var data map[string]any
 	err = json.NewDecoder(resp.Body).Decode(&data)
 	if err != nil {
 		return fmt.Errorf("failed to obtain token: %w", err)
@@ -273,11 +273,11 @@ func (p *plexConnector) isPlaying(co *Object) bool {
 }
 
 // adapted from: https://stackoverflow.com/a/28878037 (credit)
-func get(m interface{}, path ...interface{}) (interface{}, bool) {
+func get(m any, path ...any) (any, bool) {
 	for _, p := range path {
 		switch idx := p.(type) {
 		case string:
-			if mm, ok := m.(map[string]interface{}); ok {
+			if mm, ok := m.(map[string]any); ok {
 				if val, found := mm[idx]; found {
 					m = val
 					continue
@@ -285,7 +285,7 @@ func get(m interface{}, path ...interface{}) (interface{}, bool) {
 			}
 			return nil, false
 		case int:
-			if mm, ok := m.([]interface{}); ok {
+			if mm, ok := m.([]any); ok {
 				if len(mm) > idx {
 					m = mm[idx]
 					continue

backend/cache/storage_persistent.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"
 	bolt "go.etcd.io/bbolt"
+	"go.etcd.io/bbolt/errors"
 )
 
 // Constants
@@ -597,7 +598,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
 	})
 
 	if err != nil {
-		if err == bolt.ErrDatabaseNotOpen {
+		if err == errors.ErrDatabaseNotOpen {
 			// we're likely a late janitor and we need to end quietly as there's no guarantee of what exists anymore
 			return
 		}
@@ -606,16 +607,16 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
 }
 
 // Stats returns a go map with the stats key values
-func (b *Persistent) Stats() (map[string]map[string]interface{}, error) {
-	r := make(map[string]map[string]interface{})
-	r["data"] = make(map[string]interface{})
+func (b *Persistent) Stats() (map[string]map[string]any, error) {
+	r := make(map[string]map[string]any)
+	r["data"] = make(map[string]any)
 	r["data"]["oldest-ts"] = time.Now()
 	r["data"]["oldest-file"] = ""
 	r["data"]["newest-ts"] = time.Now()
 	r["data"]["newest-file"] = ""
 	r["data"]["total-chunks"] = 0
 	r["data"]["total-size"] = int64(0)
-	r["files"] = make(map[string]interface{})
+	r["files"] = make(map[string]any)
 	r["files"]["oldest-ts"] = time.Now()
 	r["files"]["oldest-name"] = ""
 	r["files"]["newest-ts"] = time.Now()

backend/chunker/chunker.go

@@ -356,7 +356,8 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		DirModTimeUpdatesOnWrite: true,
 	}).Fill(ctx, f).Mask(ctx, baseFs).WrapsFs(f, baseFs)
 
-	f.features.Disable("ListR") // Recursive listing may cause chunker skip files
+	f.features.ListR = nil // Recursive listing may cause chunker skip files
+	f.features.ListP = nil // ListP not supported yet
 
 	return f, err
 }
@@ -632,7 +633,7 @@ func (f *Fs) parseChunkName(filePath string) (parentPath string, chunkNo int, ct
 
 // forbidChunk prints error message or raises error if file is chunk.
 // First argument sets log prefix, use `false` to suppress message.
-func (f *Fs) forbidChunk(o interface{}, filePath string) error {
+func (f *Fs) forbidChunk(o any, filePath string) error {
 	if parentPath, _, _, _ := f.parseChunkName(filePath); parentPath != "" {
 		if f.opt.FailHard {
 			return fmt.Errorf("chunk overlap with %q", parentPath)
@@ -680,7 +681,7 @@ func (f *Fs) newXactID(ctx context.Context, filePath string) (xactID string, err
 	circleSec := unixSec % closestPrimeZzzzSeconds
 	first4chars := strconv.FormatInt(circleSec, 36)
 
-	for tries := 0; tries < maxTransactionProbes; tries++ {
+	for range maxTransactionProbes {
 		f.xactIDMutex.Lock()
 		randomness := f.xactIDRand.Int63n(maxTwoBase36Digits + 1)
 		f.xactIDMutex.Unlock()
@@ -1189,10 +1190,7 @@ func (f *Fs) put(
 		}
 
 		tempRemote := f.makeChunkName(baseRemote, c.chunkNo, "", xactID)
-		size := c.sizeLeft
-		if size > c.chunkSize {
-			size = c.chunkSize
-		}
+		size := min(c.sizeLeft, c.chunkSize)
 		savedReadCount := c.readCount
 
 		// If a single chunk is expected, avoid the extra rename operation
@@ -1477,10 +1475,7 @@ func (c *chunkingReader) dummyRead(in io.Reader, size int64) error {
 	const bufLen = 1048576 // 1 MiB
 	buf := make([]byte, bufLen)
 	for size > 0 {
-		n := size
-		if n > bufLen {
-			n = bufLen
-		}
+		n := min(size, bufLen)
 		if _, err := io.ReadFull(in, buf[0:n]); err != nil {
 			return err
 		}

backend/chunker/chunker_internal_test.go

@@ -40,7 +40,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
 	})
 }
 
-type settings map[string]interface{}
+type settings map[string]any
 
 func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
 	fsName := strings.Split(f.Name(), "{")[0] // strip off hash

backend/chunker/chunker_test.go

@@ -46,6 +46,7 @@ func TestIntegration(t *testing.T) {
 			"DirCacheFlush",
 			"UserInfo",
 			"Disconnect",
+			"ListP",
 		},
 	}
 	if *fstest.RemoteName == "" {

backend/cloudinary/api/types.go

@@ -18,7 +18,7 @@ type CloudinaryEncoder interface {
 	ToStandardPath(string) string
 	// ToStandardName takes name in this encoding and converts
 	// it in Standard encoding.
-	ToStandardName(string) string
+	ToStandardName(string, string) string
 	// Encoded root of the remote (as passed into NewFs)
 	FromStandardFullPath(string) string
 }

backend/cloudinary/cloudinary.go

@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -103,19 +105,39 @@ func init() {
 				Advanced: true,
 				Help:     "Wait N seconds for eventual consistency of the databases that support the backend operation",
 			},
+			{
+				Name:     "adjust_media_files_extensions",
+				Default:  true,
+				Advanced: true,
+				Help:     "Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems",
+			},
+			{
+				Name: "media_extensions",
+				Default: []string{
+					"3ds", "3g2", "3gp", "ai", "arw", "avi", "avif", "bmp", "bw",
+					"cr2", "cr3", "djvu", "dng", "eps3", "fbx", "flif", "flv", "gif",
+					"glb", "gltf", "hdp", "heic", "heif", "ico", "indd", "jp2", "jpe",
+					"jpeg", "jpg", "jxl", "jxr", "m2ts", "mov", "mp4", "mpeg", "mts",
+					"mxf", "obj", "ogv", "pdf", "ply", "png", "psd", "svg", "tga",
+					"tif", "tiff", "ts", "u3ma", "usdz", "wdp", "webm", "webp", "wmv"},
+				Advanced: true,
+				Help:     "Cloudinary supported media extensions",
+			},
 		},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	CloudName                 string               `config:"cloud_name"`
-	APIKey                    string               `config:"api_key"`
-	APISecret                 string               `config:"api_secret"`
-	UploadPrefix              string               `config:"upload_prefix"`
-	UploadPreset              string               `config:"upload_preset"`
-	Enc                       encoder.MultiEncoder `config:"encoding"`
-	EventuallyConsistentDelay fs.Duration          `config:"eventually_consistent_delay"`
+	CloudName                  string               `config:"cloud_name"`
+	APIKey                     string               `config:"api_key"`
+	APISecret                  string               `config:"api_secret"`
+	UploadPrefix               string               `config:"upload_prefix"`
+	UploadPreset               string               `config:"upload_preset"`
+	Enc                        encoder.MultiEncoder `config:"encoding"`
+	EventuallyConsistentDelay  fs.Duration          `config:"eventually_consistent_delay"`
+	MediaExtensions            []string             `config:"media_extensions"`
+	AdjustMediaFilesExtensions bool                 `config:"adjust_media_files_extensions"`
 }
 
 // Fs represents a remote cloudinary server
@@ -203,6 +225,18 @@ func (f *Fs) FromStandardPath(s string) string {
 
 // FromStandardName implementation of the api.CloudinaryEncoder
 func (f *Fs) FromStandardName(s string) string {
+	if f.opt.AdjustMediaFilesExtensions {
+		parsedURL, err := url.Parse(s)
+		ext := ""
+		if err != nil {
+			fs.Logf(nil, "Error parsing URL: %v", err)
+		} else {
+			ext = path.Ext(parsedURL.Path)
+			if slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) {
+				s = strings.TrimSuffix(parsedURL.Path, ext)
+			}
+		}
+	}
 	return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")
 }
 
@@ -212,8 +246,20 @@ func (f *Fs) ToStandardPath(s string) string {
 }
 
 // ToStandardName implementation of the api.CloudinaryEncoder
-func (f *Fs) ToStandardName(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")
+func (f *Fs) ToStandardName(s string, assetURL string) string {
+	ext := ""
+	if f.opt.AdjustMediaFilesExtensions {
+		parsedURL, err := url.Parse(assetURL)
+		if err != nil {
+			fs.Logf(nil, "Error parsing URL: %v", err)
+		} else {
+			ext = path.Ext(parsedURL.Path)
+			if !slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) {
+				ext = ""
+			}
+		}
+	}
+	return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&") + ext
 }
 
 // FromStandardFullPath encodes a full path to Cloudinary standard
@@ -331,10 +377,7 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
 		}
 
 		for _, asset := range results.Assets {
-			remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)
-			if dir != "" {
-				remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))
-			}
+			remote := path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName, asset.SecureURL))
 			o := &Object{
 				fs:           f,
 				remote:       remote,

backend/combine/combine.go

@@ -20,6 +20,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"golang.org/x/sync/errgroup"
@@ -265,6 +266,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 	}
 
+	// Enable ListP always
+	features.ListP = f.ListP
+
 	// Enable Purge when any upstreams support it
 	if features.Purge == nil {
 		for _, u := range f.upstreams {
@@ -809,24 +813,52 @@ func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.D
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	return list.WithListP(ctx, dir, f)
+}
+
+// ListP lists the objects and directories of the Fs starting
+// from dir non recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
 	// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
 	if f.root == "" && dir == "" {
-		entries = make(fs.DirEntries, 0, len(f.upstreams))
+		entries := make(fs.DirEntries, 0, len(f.upstreams))
 		for combineDir := range f.upstreams {
 			d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when))
 			entries = append(entries, d)
 		}
-		return entries, nil
+		return callback(entries)
 	}
 	u, uRemote, err := f.findUpstream(dir)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	entries, err = u.f.List(ctx, uRemote)
-	if err != nil {
-		return nil, err
+	wrappedCallback := func(entries fs.DirEntries) error {
+		entries, err := u.wrapEntries(ctx, entries)
+		if err != nil {
+			return err
+		}
+		return callback(entries)
 	}
-	return u.wrapEntries(ctx, entries)
+	listP := u.f.Features().ListP
+	if listP == nil {
+		entries, err := u.f.List(ctx, uRemote)
+		if err != nil {
+			return err
+		}
+		return wrappedCallback(entries)
+	}
+	return listP(ctx, dir, wrappedCallback)
 }
 
 // ListR lists the objects and directories of the Fs starting

backend/compress/compress.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
@@ -208,6 +209,8 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	if !operations.CanServerSideMove(wrappedFs) {
 		f.features.Disable("PutStream")
 	}
+	// Enable ListP always
+	f.features.ListP = f.ListP
 
 	return f, err
 }
@@ -352,11 +355,39 @@ func (f *Fs) processEntries(entries fs.DirEntries) (newEntries fs.DirEntries, er
 // found.
 // List entries and process them
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	entries, err = f.Fs.List(ctx, dir)
-	if err != nil {
-		return nil, err
+	return list.WithListP(ctx, dir, f)
+}
+
+// ListP lists the objects and directories of the Fs starting
+// from dir non recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
+	wrappedCallback := func(entries fs.DirEntries) error {
+		entries, err := f.processEntries(entries)
+		if err != nil {
+			return err
+		}
+		return callback(entries)
 	}
-	return f.processEntries(entries)
+	listP := f.Fs.Features().ListP
+	if listP == nil {
+		entries, err := f.Fs.List(ctx, dir)
+		if err != nil {
+			return err
+		}
+		return wrappedCallback(entries)
+	}
+	return listP(ctx, dir, wrappedCallback)
 }
 
 // ListR lists the objects and directories of the Fs starting

backend/crypt/cipher.go

@@ -192,7 +192,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 		dirNameEncrypt:  dirNameEncrypt,
 		encryptedSuffix: ".bin",
 	}
-	c.buffers.New = func() interface{} {
+	c.buffers.New = func() any {
 		return new([blockSize]byte)
 	}
 	err := c.Key(password, salt)
@@ -336,7 +336,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	_, _ = result.WriteString(strconv.Itoa(dir) + ".")
 
 	// but we'll augment it with the nameKey for real calculation
-	for i := 0; i < len(c.nameKey); i++ {
+	for i := range len(c.nameKey) {
 		dir += int(c.nameKey[i])
 	}
 
@@ -418,7 +418,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 	}
 
 	// add the nameKey to get the real rotate distance
-	for i := 0; i < len(c.nameKey); i++ {
+	for i := range len(c.nameKey) {
 		dir += int(c.nameKey[i])
 	}
 
@@ -664,7 +664,7 @@ func (n *nonce) increment() {
 // add a uint64 to the nonce
 func (n *nonce) add(x uint64) {
 	carry := uint16(0)
-	for i := 0; i < 8; i++ {
+	for i := range 8 {
 		digit := (*n)[i]
 		xDigit := byte(x)
 		x >>= 8

backend/crypt/cipher_test.go

@@ -1307,10 +1307,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 	open := func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
 		end := len(ciphertext)
 		if underlyingLimit >= 0 {
-			end = int(underlyingOffset + underlyingLimit)
-			if end > len(ciphertext) {
-				end = len(ciphertext)
-			}
+			end = min(int(underlyingOffset+underlyingLimit), len(ciphertext))
 		}
 		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
@@ -1490,7 +1487,7 @@ func TestDecrypterRead(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Test truncating the file at each possible point
-	for i := 0; i < len(file16)-1; i++ {
+	for i := range len(file16) - 1 {
 		what := fmt.Sprintf("truncating to %d/%d", i, len(file16))
 		cd := newCloseDetector(bytes.NewBuffer(file16[:i]))
 		fh, err := c.newDecrypter(cd)

backend/crypt/crypt.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 )
 
 // Globals
@@ -293,6 +294,9 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		PartialUploads:           true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 
+	// Enable ListP always
+	f.features.ListP = f.ListP
+
 	return f, err
 }
 
@@ -416,11 +420,40 @@ func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntr
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	entries, err = f.Fs.List(ctx, f.cipher.EncryptDirName(dir))
-	if err != nil {
-		return nil, err
+	return list.WithListP(ctx, dir, f)
+}
+
+// ListP lists the objects and directories of the Fs starting
+// from dir non recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
+	wrappedCallback := func(entries fs.DirEntries) error {
+		entries, err := f.encryptEntries(ctx, entries)
+		if err != nil {
+			return err
+		}
+		return callback(entries)
 	}
-	return f.encryptEntries(ctx, entries)
+	listP := f.Fs.Features().ListP
+	encryptedDir := f.cipher.EncryptDirName(dir)
+	if listP == nil {
+		entries, err := f.Fs.List(ctx, encryptedDir)
+		if err != nil {
+			return err
+		}
+		return wrappedCallback(entries)
+	}
+	return listP(ctx, encryptedDir, wrappedCallback)
 }
 
 // ListR lists the objects and directories of the Fs starting
@@ -924,7 +957,7 @@ Usage Example:
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "decode":
 		out := make([]string, 0, len(arg))

backend/crypt/pkcs7/pkcs7.go

@@ -25,7 +25,7 @@ func Pad(n int, buf []byte) []byte {
 	}
 	length := len(buf)
 	padding := n - (length % n)
-	for i := 0; i < padding; i++ {
+	for range padding {
 		buf = append(buf, byte(padding))
 	}
 	if (len(buf) % n) != 0 {
@@ -54,7 +54,7 @@ func Unpad(n int, buf []byte) ([]byte, error) {
 	if padding == 0 {
 		return nil, ErrorPaddingTooShort
 	}
-	for i := 0; i < padding; i++ {
+	for i := range padding {
 		if buf[length-1-i] != byte(padding) {
 			return nil, ErrorPaddingNotAllTheSame
 		}

backend/drive/drive.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -37,8 +38,8 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
@@ -199,12 +200,7 @@ func driveScopes(scopesString string) (scopes []string) {
 
 // Returns true if one of the scopes was "drive.appfolder"
 func driveScopesContainsAppFolder(scopes []string) bool {
-	for _, scope := range scopes {
-		if scope == scopePrefix+"drive.appfolder" {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(scopes, scopePrefix+"drive.appfolder")
 }
 
 func driveOAuthOptions() []fs.Option {
@@ -958,12 +954,7 @@ func parseDrivePath(path string) (root string, err error) {
 type listFn func(*drive.File) bool
 
 func containsString(slice []string, s string) bool {
-	for _, e := range slice {
-		if e == s {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(slice, s)
 }
 
 // getFile returns drive.File for the ID passed and fields passed in
@@ -1152,13 +1143,7 @@ OUTER:
 			// Check the case of items is correct since
 			// the `=` operator is case insensitive.
 			if title != "" && title != item.Name {
-				found := false
-				for _, stem := range stems {
-					if stem == item.Name {
-						found = true
-						break
-					}
-				}
+				found := slices.Contains(stems, item.Name)
 				if !found {
 					continue
 				}
@@ -1561,13 +1546,10 @@ func (f *Fs) getFileFields(ctx context.Context) (fields googleapi.Field) {
 func (f *Fs) newRegularObject(ctx context.Context, remote string, info *drive.File) (obj fs.Object, err error) {
 	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
 	if f.opt.SkipChecksumGphotos {
-		for _, space := range info.Spaces {
-			if space == "photos" {
-				info.Md5Checksum = ""
-				info.Sha1Checksum = ""
-				info.Sha256Checksum = ""
-				break
-			}
+		if slices.Contains(info.Spaces, "photos") {
+			info.Md5Checksum = ""
+			info.Sha1Checksum = ""
+			info.Sha256Checksum = ""
 		}
 	}
 	o := &Object{
@@ -2207,7 +2189,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	wg := sync.WaitGroup{}
 	in := make(chan listREntry, listRInputBuffer)
 	out := make(chan error, f.ci.Checkers)
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	overflow := []listREntry{}
 	listed := 0
 
@@ -2245,7 +2227,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	wg.Add(1)
 	in <- listREntry{directoryID, dir}
 
-	for i := 0; i < f.ci.Checkers; i++ {
+	for range f.ci.Checkers {
 		go f.listRRunner(ctx, &wg, in, out, cb, sendJob)
 	}
 	go func() {
@@ -2254,11 +2236,8 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		// if the input channel overflowed add the collected entries to the channel now
 		for len(overflow) > 0 {
 			mu.Lock()
-			l := len(overflow)
 			// only fill half of the channel to prevent entries being put into overflow again
-			if l > listRInputBuffer/2 {
-				l = listRInputBuffer / 2
-			}
+			l := min(len(overflow), listRInputBuffer/2)
 			wg.Add(l)
 			for _, d := range overflow[:l] {
 				in <- d
@@ -2278,7 +2257,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		mu.Unlock()
 	}()
 	// wait until the all workers to finish
-	for i := 0; i < f.ci.Checkers; i++ {
+	for range f.ci.Checkers {
 		e := <-out
 		mu.Lock()
 		// if one worker returns an error early, close the input so all other workers exit
@@ -3530,14 +3509,14 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
 	return f.unTrash(ctx, dir, directoryID, true)
 }
 
-// copy file with id to dest
-func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
+// copy or move file with id to dest
+func (f *Fs) copyOrMoveID(ctx context.Context, operation string, id, dest string) (err error) {
 	info, err := f.getFile(ctx, id, f.getFileFields(ctx))
 	if err != nil {
 		return fmt.Errorf("couldn't find id: %w", err)
 	}
 	if info.MimeType == driveFolderType {
-		return fmt.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
+		return fmt.Errorf("can't %s directory use: rclone %s --drive-root-folder-id %s %s %s", operation, operation, id, fs.ConfigString(f), dest)
 	}
 	info.Name = f.opt.Enc.ToStandardName(info.Name)
 	o, err := f.newObjectWithInfo(ctx, info.Name, info)
@@ -3558,9 +3537,15 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 	if err != nil {
 		return err
 	}
-	_, err = operations.Copy(ctx, dstFs, nil, destLeaf, o)
-	if err != nil {
-		return fmt.Errorf("copy failed: %w", err)
+
+	var opErr error
+	if operation == "moveid" {
+		_, opErr = operations.Move(ctx, dstFs, nil, destLeaf, o)
+	} else {
+		_, opErr = operations.Copy(ctx, dstFs, nil, destLeaf, o)
+	}
+	if opErr != nil {
+		return fmt.Errorf("%s failed: %w", operation, opErr)
 	}
 	return nil
 }
@@ -3797,6 +3782,28 @@ attempted if possible.
 
 Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
 `,
+}, {
+	Name:  "moveid",
+	Short: "Move files by ID",
+	Long: `This command moves files by ID
+
+Usage:
+
+    rclone backend moveid drive: ID path
+    rclone backend moveid drive: ID1 path1 ID2 path2
+
+It moves the drive file with ID given to the path (an rclone path which
+will be passed internally to rclone moveto).
+
+The path should end with a / to indicate move the file as named to
+this directory. If it doesn't end with a / then the last path
+component will be used as the file name.
+
+If the destination is a drive backend then server-side moving will be
+attempted if possible.
+
+Use the --interactive/-i or --dry-run flag to see what would be moved beforehand.
+`,
 }, {
 	Name:  "exportformats",
 	Short: "Dump the export formats for debug purposes",
@@ -3886,7 +3893,7 @@ Third delete all orphaned files to the trash
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "get":
 		out := make(map[string]string)
@@ -3975,16 +3982,16 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			dir = arg[0]
 		}
 		return f.unTrashDir(ctx, dir, true)
-	case "copyid":
+	case "copyid", "moveid":
 		if len(arg)%2 != 0 {
 			return nil, errors.New("need an even number of arguments")
 		}
 		for len(arg) > 0 {
 			id, dest := arg[0], arg[1]
 			arg = arg[2:]
-			err = f.copyID(ctx, id, dest)
+			err = f.copyOrMoveID(ctx, name, id, dest)
 			if err != nil {
-				return nil, fmt.Errorf("failed copying %q to %q: %w", id, dest, err)
+				return nil, fmt.Errorf("failed %s %q to %q: %w", name, id, dest, err)
 			}
 		}
 		return nil, nil

backend/drive/drive_internal_test.go

@@ -479,8 +479,8 @@ func (f *Fs) InternalTestUnTrash(t *testing.T) {
 	require.NoError(t, f.Purge(ctx, "trashDir"))
 }
 
-// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyID
-func (f *Fs) InternalTestCopyID(t *testing.T) {
+// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyOrMoveID
+func (f *Fs) InternalTestCopyOrMoveID(t *testing.T) {
 	ctx := context.Background()
 	obj, err := f.NewObject(ctx, existingFile)
 	require.NoError(t, err)
@@ -498,7 +498,7 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	}
 
 	t.Run("BadID", func(t *testing.T) {
-		err = f.copyID(ctx, "ID-NOT-FOUND", dir+"/")
+		err = f.copyOrMoveID(ctx, "moveid", "ID-NOT-FOUND", dir+"/")
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "couldn't find id")
 	})
@@ -506,19 +506,31 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	t.Run("Directory", func(t *testing.T) {
 		rootID, err := f.dirCache.RootID(ctx, false)
 		require.NoError(t, err)
-		err = f.copyID(ctx, rootID, dir+"/")
+		err = f.copyOrMoveID(ctx, "moveid", rootID, dir+"/")
 		require.Error(t, err)
-		assert.Contains(t, err.Error(), "can't copy directory")
+		assert.Contains(t, err.Error(), "can't moveid directory")
 	})
 
-	t.Run("WithoutDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/")
+	t.Run("MoveWithoutDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/")
 		require.NoError(t, err)
 		checkFile(path.Base(existingFile))
 	})
 
-	t.Run("WithDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/potato.txt")
+	t.Run("CopyWithoutDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/")
+		require.NoError(t, err)
+		checkFile(path.Base(existingFile))
+	})
+
+	t.Run("MoveWithDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/potato.txt")
+		require.NoError(t, err)
+		checkFile("potato.txt")
+	})
+
+	t.Run("CopyWithDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/potato.txt")
 		require.NoError(t, err)
 		checkFile("potato.txt")
 	})
@@ -647,7 +659,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	})
 	t.Run("Shortcuts", f.InternalTestShortcuts)
 	t.Run("UnTrash", f.InternalTestUnTrash)
-	t.Run("CopyID", f.InternalTestCopyID)
+	t.Run("CopyOrMoveID", f.InternalTestCopyOrMoveID)
 	t.Run("Query", f.InternalTestQuery)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
 	t.Run("ShouldRetry", f.InternalTestShouldRetry)

backend/drive/metadata.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"strconv"
 	"strings"
 	"sync"
@@ -324,9 +325,7 @@ func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err e
 	metadata := make(fs.Metadata, 16)
 
 	// Dump user metadata first as it overrides system metadata
-	for k, v := range info.Properties {
-		metadata[k] = v
-	}
+	maps.Copy(metadata, info.Properties)
 
 	// System metadata
 	metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission)

backend/drive/upload.go

@@ -177,10 +177,7 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 			if start >= rx.ContentLength {
 				break
 			}
-			reqSize = rx.ContentLength - start
-			if reqSize >= int64(rx.f.opt.ChunkSize) {
-				reqSize = int64(rx.f.opt.ChunkSize)
-			}
+			reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
 			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
 		} else {
 			// If size unknown read into buffer

backend/dropbox/dbhash/dbhash.go

@@ -55,10 +55,7 @@ func (d *digest) Write(p []byte) (n int, err error) {
 	n = len(p)
 	for len(p) > 0 {
 		d.writtenMore = true
-		toWrite := bytesPerBlock - d.n
-		if toWrite > len(p) {
-			toWrite = len(p)
-		}
+		toWrite := min(bytesPerBlock-d.n, len(p))
 		_, err = d.blockHash.Write(p[:toWrite])
 		if err != nil {
 			panic(hashReturnedError)

backend/dropbox/dbhash/dbhash_test.go

@@ -11,7 +11,7 @@ import (
 
 func testChunk(t *testing.T, chunk int) {
 	data := make([]byte, chunk)
-	for i := 0; i < chunk; i++ {
+	for i := range chunk {
 		data[i] = 'A'
 	}
 	for _, test := range []struct {

backend/dropbox/dropbox.go

@@ -92,6 +92,9 @@ const (
 	maxFileNameLength = 255
 )
 
+type exportAPIFormat string
+type exportExtension string // dotless
+
 var (
 	// Description of how to auth for this app
 	dropboxConfig = &oauthutil.Config{
@@ -132,6 +135,16 @@ var (
 		DefaultTimeoutAsync:   10 * time.Second,
 		DefaultBatchSizeAsync: 100,
 	}
+
+	exportKnownAPIFormats = map[exportAPIFormat]exportExtension{
+		"markdown": "md",
+		"html":     "html",
+	}
+	// Populated based on exportKnownAPIFormats
+	exportKnownExtensions = map[exportExtension]exportAPIFormat{}
+
+	paperExtension         = ".paper"
+	paperTemplateExtension = ".papert"
 )
 
 // Gets an oauth config with the right scopes
@@ -247,23 +260,61 @@ folders.`,
 			Help:     "Specify a different Dropbox namespace ID to use as the root for all paths.",
 			Default:  "",
 			Advanced: true,
-		}}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
+		}, {
+			Name: "export_formats",
+			Help: `Comma separated list of preferred formats for exporting files
+
+Certain Dropbox files can only be accessed by exporting them to another format.
+These include Dropbox Paper documents.
+
+For each such file, rclone will choose the first format on this list that Dropbox
+considers valid. If none is valid, it will choose Dropbox's default format.
+
+Known formats include: "html", "md" (markdown)`,
+			Default:  fs.CommaSepList{"html", "md"},
+			Advanced: true,
+		}, {
+			Name:     "skip_exports",
+			Help:     "Skip exportable files in all listings.\n\nIf given, exportable files practically become invisible to rclone.",
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:    "show_all_exports",
+			Default: false,
+			Help: `Show all exportable files in listings.
+
+Adding this flag will allow all exportable files to be server side copied.
+Note that rclone doesn't add extensions to the exportable file names in this mode.
+
+Do **not** use this flag when trying to download exportable files - rclone
+will fail to download them.
+`,
+			Advanced: true,
+		},
+		}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
 	})
+
+	for apiFormat, ext := range exportKnownAPIFormats {
+		exportKnownExtensions[ext] = apiFormat
+	}
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize     fs.SizeSuffix        `config:"chunk_size"`
-	Impersonate   string               `config:"impersonate"`
-	SharedFiles   bool                 `config:"shared_files"`
-	SharedFolders bool                 `config:"shared_folders"`
-	BatchMode     string               `config:"batch_mode"`
-	BatchSize     int                  `config:"batch_size"`
-	BatchTimeout  fs.Duration          `config:"batch_timeout"`
-	AsyncBatch    bool                 `config:"async_batch"`
-	PacerMinSleep fs.Duration          `config:"pacer_min_sleep"`
-	Enc           encoder.MultiEncoder `config:"encoding"`
-	RootNsid      string               `config:"root_namespace"`
+	ChunkSize      fs.SizeSuffix        `config:"chunk_size"`
+	Impersonate    string               `config:"impersonate"`
+	SharedFiles    bool                 `config:"shared_files"`
+	SharedFolders  bool                 `config:"shared_folders"`
+	BatchMode      string               `config:"batch_mode"`
+	BatchSize      int                  `config:"batch_size"`
+	BatchTimeout   fs.Duration          `config:"batch_timeout"`
+	AsyncBatch     bool                 `config:"async_batch"`
+	PacerMinSleep  fs.Duration          `config:"pacer_min_sleep"`
+	Enc            encoder.MultiEncoder `config:"encoding"`
+	RootNsid       string               `config:"root_namespace"`
+	ExportFormats  fs.CommaSepList      `config:"export_formats"`
+	SkipExports    bool                 `config:"skip_exports"`
+	ShowAllExports bool                 `config:"show_all_exports"`
 }
 
 // Fs represents a remote dropbox server
@@ -283,8 +334,18 @@ type Fs struct {
 	pacer          *fs.Pacer      // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
 	batcher        *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
+	exportExts     []exportExtension
 }
 
+type exportType int
+
+const (
+	notExport        exportType = iota // a regular file
+	exportHide                         // should be hidden
+	exportListOnly                     // listable, but can't export
+	exportExportable                   // can export
+)
+
 // Object describes a dropbox object
 //
 // Dropbox Objects always have full metadata
@@ -296,6 +357,9 @@ type Object struct {
 	bytes   int64     // size of the object
 	modTime time.Time // time it was last modified
 	hash    string    // content_hash of the object
+
+	exportType      exportType
+	exportAPIFormat exportAPIFormat
 }
 
 // Name of the remote (as passed into NewFs)
@@ -436,6 +500,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		HeaderGenerator: f.headerGenerator,
 	}
 
+	for _, e := range opt.ExportFormats {
+		ext := exportExtension(e)
+		if exportKnownExtensions[ext] == "" {
+			return nil, fmt.Errorf("dropbox: unknown export format '%s'", e)
+		}
+		f.exportExts = append(f.exportExts, ext)
+	}
+
 	// unauthorized config for endpoints that fail with auth
 	ucfg := dropbox.Config{
 		LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
@@ -588,38 +660,126 @@ func (f *Fs) setRoot(root string) {
 	}
 }
 
+type getMetadataResult struct {
+	entry    files.IsMetadata
+	notFound bool
+	err      error
+}
+
 // getMetadata gets the metadata for a file or directory
-func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
-	err = f.pacer.Call(func() (bool, error) {
-		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
+func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
+	res.err = f.pacer.Call(func() (bool, error) {
+		res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
 			Path: f.opt.Enc.FromStandardPath(objPath),
 		})
-		return shouldRetry(ctx, err)
+		return shouldRetry(ctx, res.err)
 	})
-	if err != nil {
-		switch e := err.(type) {
+	if res.err != nil {
+		switch e := res.err.(type) {
 		case files.GetMetadataAPIError:
 			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
-				notFound = true
-				err = nil
+				res.notFound = true
+				res.err = nil
 			}
 		}
 	}
 	return
 }
 
-// getFileMetadata gets the metadata for a file
-func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
-	entry, notFound, err := f.getMetadata(ctx, filePath)
-	if err != nil {
-		return nil, err
+// Get metadata such that the result would be exported with the given extension
+// Return a channel that will eventually receive the metadata
+func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportExtension exportExtension) chan getMetadataResult {
+	ch := make(chan getMetadataResult, 1)
+	wantDownloadable := (wantExportExtension == "")
+	go func() {
+		defer close(ch)
+
+		res := f.getMetadata(ctx, filePath)
+		info, ok := res.entry.(*files.FileMetadata)
+		if !ok { // Can't check anything about file, just return what we have
+			ch <- res
+			return
+		}
+
+		// Return notFound if downloadability or extension doesn't match
+		if wantDownloadable != info.IsDownloadable {
+			ch <- getMetadataResult{notFound: true}
+			return
+		}
+		if !info.IsDownloadable {
+			_, ext := f.chooseExportFormat(info)
+			if ext != wantExportExtension {
+				ch <- getMetadataResult{notFound: true}
+				return
+			}
+		}
+
+		// Return our real result or error
+		ch <- res
+	}()
+	return ch
+}
+
+// For a given rclone-path, figure out what the Dropbox-path may be, in order of preference.
+// Multiple paths might be plausible, due to export path munging.
+func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-chan getMetadataResult) {
+	ret = []<-chan getMetadataResult{}
+
+	// Prefer an exact match
+	ret = append(ret, f.getMetadataForExt(ctx, filePath, ""))
+
+	// Check if we're plausibly an export path, otherwise we're done
+	if f.opt.SkipExports || f.opt.ShowAllExports {
+		return
 	}
-	if notFound {
+	dotted := path.Ext(filePath)
+	if dotted == "" {
+		return
+	}
+	ext := exportExtension(dotted[1:])
+	if exportKnownExtensions[ext] == "" {
+		return
+	}
+
+	// We might be an export path! Try all possibilities
+	base := strings.TrimSuffix(filePath, dotted)
+
+	// `foo.papert.md` will only come from `foo.papert`. Never check something like `foo.papert.paper`
+	if strings.HasSuffix(base, paperTemplateExtension) {
+		ret = append(ret, f.getMetadataForExt(ctx, base, ext))
+		return
+	}
+
+	// Otherwise, try both `foo.md` coming from `foo`, or from `foo.paper`
+	ret = append(ret, f.getMetadataForExt(ctx, base, ext))
+	ret = append(ret, f.getMetadataForExt(ctx, base+paperExtension, ext))
+	return
+}
+
+// getFileMetadata gets the metadata for a file
+func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
+	var res getMetadataResult
+
+	// Try all possible metadatas
+	possibleMetadatas := f.possibleMetadatas(ctx, filePath)
+	for _, ch := range possibleMetadatas {
+		res = <-ch
+
+		if res.err != nil {
+			return nil, res.err
+		}
+		if !res.notFound {
+			break
+		}
+	}
+
+	if res.notFound {
 		return nil, fs.ErrorObjectNotFound
 	}
-	fileInfo, ok := entry.(*files.FileMetadata)
+
+	fileInfo, ok := res.entry.(*files.FileMetadata)
 	if !ok {
-		if _, ok = entry.(*files.FolderMetadata); ok {
+		if _, ok = res.entry.(*files.FolderMetadata); ok {
 			return nil, fs.ErrorIsDir
 		}
 		return nil, fs.ErrorNotAFile
@@ -628,15 +788,15 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *fi
 }
 
 // getDirMetadata gets the metadata for a directory
-func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
-	entry, notFound, err := f.getMetadata(ctx, dirPath)
-	if err != nil {
-		return nil, err
+func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (*files.FolderMetadata, error) {
+	res := f.getMetadata(ctx, dirPath)
+	if res.err != nil {
+		return nil, res.err
 	}
-	if notFound {
+	if res.notFound {
 		return nil, fs.ErrorDirNotFound
 	}
-	dirInfo, ok := entry.(*files.FolderMetadata)
+	dirInfo, ok := res.entry.(*files.FolderMetadata)
 	if !ok {
 		return nil, fs.ErrorIsFile
 	}
@@ -836,16 +996,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var res *files.ListFolderResult
 	for {
 		if !started {
-			arg := files.ListFolderArg{
-				Path:      f.opt.Enc.FromStandardPath(root),
-				Recursive: false,
-				Limit:     1000,
-			}
+			arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(root))
+			arg.Recursive = false
+			arg.Limit = 1000
+
 			if root == "/" {
 				arg.Path = "" // Specify root folder as empty string
 			}
 			err = f.pacer.Call(func() (bool, error) {
-				res, err = f.srv.ListFolder(&arg)
+				res, err = f.srv.ListFolder(arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -898,7 +1057,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if err != nil {
 					return nil, err
 				}
-				entries = append(entries, o)
+				if o.(*Object).exportType.listable() {
+					entries = append(entries, o)
+				}
 			}
 		}
 		if !res.HasMore {
@@ -984,16 +1145,14 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		}
 
 		// check directory empty
-		arg := files.ListFolderArg{
-			Path:      encRoot,
-			Recursive: false,
-		}
+		arg := files.NewListFolderArg(encRoot)
+		arg.Recursive = false
 		if root == "/" {
 			arg.Path = "" // Specify root folder as empty string
 		}
 		var res *files.ListFolderResult
 		err = f.pacer.Call(func() (bool, error) {
-			res, err = f.srv.ListFolder(&arg)
+			res, err = f.srv.ListFolder(arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1174,6 +1333,16 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		return shouldRetry(ctx, err)
 	})
 
+	if err != nil && createArg.Settings.Expires != nil && strings.Contains(err.Error(), sharing.SharedLinkSettingsErrorNotAuthorized) {
+		// Some plans can't create links with expiry
+		fs.Debugf(absPath, "can't create link with expiry, trying without")
+		createArg.Settings.Expires = nil
+		err = f.pacer.Call(func() (bool, error) {
+			linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
+			return shouldRetry(ctx, err)
+		})
+	}
+
 	if err != nil && strings.Contains(err.Error(),
 		sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
 		fs.Debugf(absPath, "has a public link already, attempting to retrieve it")
@@ -1338,16 +1507,14 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
 	var startCursor *files.ListFolderGetLatestCursorResult
 
 	err = f.pacer.Call(func() (bool, error) {
-		arg := files.ListFolderArg{
-			Path:      f.opt.Enc.FromStandardPath(f.slashRoot),
-			Recursive: true,
-		}
+		arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(f.slashRoot))
+		arg.Recursive = true
 
 		if arg.Path == "/" {
 			arg.Path = ""
 		}
 
-		startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
+		startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
 
 		return shouldRetry(ctx, err)
 	})
@@ -1451,8 +1618,50 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+func (f *Fs) chooseExportFormat(info *files.FileMetadata) (exportAPIFormat, exportExtension) {
+	// Find API export formats Dropbox supports for this file
+	// Sometimes Dropbox lists a format in ExportAs but not ExportOptions, so check both
+	ei := info.ExportInfo
+	dropboxFormatStrings := append([]string{ei.ExportAs}, ei.ExportOptions...)
+
+	// Find which extensions these correspond to
+	exportExtensions := map[exportExtension]exportAPIFormat{}
+	var dropboxPreferredAPIFormat exportAPIFormat
+	var dropboxPreferredExtension exportExtension
+	for _, format := range dropboxFormatStrings {
+		apiFormat := exportAPIFormat(format)
+		// Only consider formats we know about
+		if ext, ok := exportKnownAPIFormats[apiFormat]; ok {
+			if dropboxPreferredAPIFormat == "" {
+				dropboxPreferredAPIFormat = apiFormat
+				dropboxPreferredExtension = ext
+			}
+			exportExtensions[ext] = apiFormat
+		}
+	}
+
+	// See if the user picked a valid extension
+	for _, ext := range f.exportExts {
+		if apiFormat, ok := exportExtensions[ext]; ok {
+			return apiFormat, ext
+		}
+	}
+
+	// If no matches, prefer the first valid format Dropbox lists
+	return dropboxPreferredAPIFormat, dropboxPreferredExtension
+}
+
 // ------------------------------------------------------------
 
+func (et exportType) listable() bool {
+	return et != exportHide
+}
+
+// something we should _try_ to export
+func (et exportType) exportable() bool {
+	return et == exportExportable || et == exportListOnly
+}
+
 // Fs returns the parent Fs
 func (o *Object) Fs() fs.Info {
 	return o.fs
@@ -1496,6 +1705,32 @@ func (o *Object) Size() int64 {
 	return o.bytes
 }
 
+func (o *Object) setMetadataForExport(info *files.FileMetadata) {
+	o.bytes = -1
+	o.hash = ""
+
+	if o.fs.opt.SkipExports {
+		o.exportType = exportHide
+		return
+	}
+	if o.fs.opt.ShowAllExports {
+		o.exportType = exportListOnly
+		return
+	}
+
+	var exportExt exportExtension
+	o.exportAPIFormat, exportExt = o.fs.chooseExportFormat(info)
+	if o.exportAPIFormat == "" {
+		o.exportType = exportHide
+	} else {
+		o.exportType = exportExportable
+		// get rid of any paper extension, if present
+		o.remote = strings.TrimSuffix(o.remote, paperExtension)
+		// add the export extension
+		o.remote += "." + string(exportExt)
+	}
+}
+
 // setMetadataFromEntry sets the fs data from a files.FileMetadata
 //
 // This isn't a complete set of metadata and has an inaccurate date
@@ -1504,6 +1739,10 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
 	o.bytes = int64(info.Size)
 	o.modTime = info.ClientModified
 	o.hash = info.ContentHash
+
+	if !info.IsDownloadable {
+		o.setMetadataForExport(info)
+	}
 	return nil
 }
 
@@ -1567,6 +1806,27 @@ func (o *Object) Storable() bool {
 	return true
 }
 
+func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
+	if o.exportType == exportListOnly || o.exportAPIFormat == "" {
+		fs.Debugf(o.remote, "No export format found")
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
+	var exportResult *files.ExportResult
+	err = o.fs.pacer.Call(func() (bool, error) {
+		exportResult, in, err = o.fs.srv.Export(&arg)
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	o.bytes = int64(exportResult.ExportMetadata.Size)
+	o.hash = exportResult.ExportMetadata.ExportHash
+	return
+}
+
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	if o.fs.opt.SharedFiles {
@@ -1586,6 +1846,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return
 	}
 
+	if o.exportType.exportable() {
+		return o.export(ctx)
+	}
+
 	fs.FixRangeOption(options, o.bytes)
 	headers := fs.OpenOptionHeaders(options)
 	arg := files.DownloadArg{

backend/dropbox/dropbox_internal_test.go

@@ -1,9 +1,16 @@
 package dropbox
 
 import (
+	"context"
+	"io"
+	"strings"
 	"testing"
 
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestInternalCheckPathLength(t *testing.T) {
@@ -42,3 +49,54 @@ func TestInternalCheckPathLength(t *testing.T) {
 		assert.Equal(t, test.ok, err == nil, test.in)
 	}
 }
+
+func (f *Fs) importPaperForTest(t *testing.T) {
+	content := `# test doc
+
+Lorem ipsum __dolor__ sit amet
+[link](http://google.com)
+`
+
+	arg := files.PaperCreateArg{
+		Path:         f.slashRootSlash + "export.paper",
+		ImportFormat: &files.ImportFormat{Tagged: dropbox.Tagged{Tag: files.ImportFormatMarkdown}},
+	}
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		reader := strings.NewReader(content)
+		_, err = f.srv.PaperCreate(&arg, reader)
+		return shouldRetry(context.Background(), err)
+	})
+	require.NoError(t, err)
+}
+
+func (f *Fs) InternalTestPaperExport(t *testing.T) {
+	ctx := context.Background()
+	f.importPaperForTest(t)
+
+	f.exportExts = []exportExtension{"html"}
+
+	obj, err := f.NewObject(ctx, "export.html")
+	require.NoError(t, err)
+
+	rc, err := obj.Open(ctx)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, rc.Close()) }()
+
+	buf, err := io.ReadAll(rc)
+	require.NoError(t, err)
+	text := string(buf)
+
+	for _, excerpt := range []string{
+		"Lorem ipsum",
+		"<b>dolor</b>",
+		`href="http://google.com"`,
+	} {
+		require.Contains(t, text, excerpt)
+	}
+}
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("PaperExport", f.InternalTestPaperExport)
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)

backend/filefabric/api/types.go

@@ -216,11 +216,11 @@ var ItemFields = mustFields(Item{})
 
 // fields returns the JSON fields in use by opt as a | separated
 // string.
-func fields(opt interface{}) (pipeTags string, err error) {
+func fields(opt any) (pipeTags string, err error) {
 	var tags []string
 	def := reflect.ValueOf(opt)
 	defType := def.Type()
-	for i := 0; i < def.NumField(); i++ {
+	for i := range def.NumField() {
 		field := defType.Field(i)
 		tag, ok := field.Tag.Lookup("json")
 		if !ok {
@@ -239,7 +239,7 @@ func fields(opt interface{}) (pipeTags string, err error) {
 
 // mustFields returns the JSON fields in use by opt as a | separated
 // string. It panics on failure.
-func mustFields(opt interface{}) string {
+func mustFields(opt any) string {
 	tags, err := fields(opt)
 	if err != nil {
 		panic(err)
@@ -351,12 +351,12 @@ type SpaceInfo struct {
 // DeleteResponse is returned from doDeleteFile
 type DeleteResponse struct {
 	Status
-	Deleted        []string      `json:"deleted"`
-	Errors         []interface{} `json:"errors"`
-	ID             string        `json:"fi_id"`
-	BackgroundTask int           `json:"backgroundtask"`
-	UsSize         string        `json:"us_size"`
-	PaSize         string        `json:"pa_size"`
+	Deleted        []string `json:"deleted"`
+	Errors         []any    `json:"errors"`
+	ID             string   `json:"fi_id"`
+	BackgroundTask int      `json:"backgroundtask"`
+	UsSize         string   `json:"us_size"`
+	PaSize         string   `json:"pa_size"`
 	//SpaceInfo      SpaceInfo     `json:"spaceinfo"`
 }
 

backend/filefabric/filefabric.go

@@ -371,7 +371,7 @@ func (f *Fs) getToken(ctx context.Context) (token string, err error) {
 }
 
 // params for rpc
-type params map[string]interface{}
+type params map[string]any
 
 // rpc calls the rpc.php method of the SME file fabric
 //

backend/filescom/filescom.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"slices"
 	"strings"
 	"time"
 
@@ -169,11 +170,9 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
 	}
 
 	if apiErr, ok := err.(files_sdk.ResponseError); ok {
-		for _, e := range retryErrorCodes {
-			if apiErr.HttpCode == e {
-				fs.Debugf(nil, "Retrying API error %v", err)
-				return true, err
-			}
+		if slices.Contains(retryErrorCodes, apiErr.HttpCode) {
+			fs.Debugf(nil, "Retrying API error %v", err)
+			return true, err
 		}
 	}
 

backend/ftp/ftp_internal_test.go

@@ -17,7 +17,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-type settings map[string]interface{}
+type settings map[string]any
 
 func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 	fsName := strings.Split(f.Name(), "{")[0] // strip off hash

backend/gofile/gofile.go

@@ -25,7 +25,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
@@ -734,7 +734,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 }
 
 // implementation of ListR
-func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err error) {
+func (f *Fs) listR(ctx context.Context, dir string, list *list.Helper) (err error) {
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -820,7 +820,7 @@ func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err
 // Don't implement this unless you have a more efficient way
 // of listing recursively than doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	err = f.listR(ctx, dir, list)
 	if err != nil {
 		return err

backend/googlecloudstorage/googlecloudstorage.go

@@ -35,7 +35,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
@@ -845,7 +845,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // of listing recursively that doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
 	bucket, directory := f.split(dir)
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	listR := func(bucket, directory, prefix string, addBucket bool) error {
 		return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *storage.Object, isDirectory bool) error {
 			entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)

backend/googlephotos/albums.go

@@ -4,6 +4,7 @@ package googlephotos
 
 import (
 	"path"
+	"slices"
 	"strings"
 	"sync"
 
@@ -119,7 +120,7 @@ func (as *albums) _del(album *api.Album) {
 		dirs := as.path[dir]
 		for i, dir := range dirs {
 			if dir == leaf {
-				dirs = append(dirs[:i], dirs[i+1:]...)
+				dirs = slices.Delete(dirs, i, i+1)
 				break
 			}
 		}

backend/googlephotos/googlephotos.go

@@ -167,7 +167,7 @@ listings and won't be transferred.`,
 The Google API will deliver images and video which aren't full
 resolution, and/or have EXIF data missing.
 
-However if you ue the gphotosdl proxy tnen you can download original,
+However if you use the gphotosdl proxy then you can download original,
 unchanged images.
 
 This runs a headless browser in the background.
@@ -388,7 +388,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
 		Method:  "GET",
 		RootURL: "https://accounts.google.com/.well-known/openid-configuration",
 	}
-	var openIDconfig map[string]interface{}
+	var openIDconfig map[string]any
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
 		return shouldRetry(ctx, resp, err)
@@ -448,7 +448,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
 			"token_type_hint": []string{"access_token"},
 		},
 	}
-	var res interface{}
+	var res any
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
 		return shouldRetry(ctx, resp, err)

backend/hasher/commands.go

@@ -24,7 +24,7 @@ import (
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "drop":
 		return nil, f.db.Stop(true)

backend/hasher/hasher.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/kv"
 )
 
@@ -182,6 +183,9 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
+	// Enable ListP always
+	f.features.ListP = f.ListP
+
 	cache.PinUntilFinalized(f.Fs, f)
 	return f, err
 }
@@ -237,10 +241,39 @@ func (f *Fs) wrapEntries(baseEntries fs.DirEntries) (hashEntries fs.DirEntries,
 
 // List the objects and directories in dir into entries.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	if entries, err = f.Fs.List(ctx, dir); err != nil {
-		return nil, err
+	return list.WithListP(ctx, dir, f)
+}
+
+// ListP lists the objects and directories of the Fs starting
+// from dir non recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
+	wrappedCallback := func(entries fs.DirEntries) error {
+		entries, err := f.wrapEntries(entries)
+		if err != nil {
+			return err
+		}
+		return callback(entries)
 	}
-	return f.wrapEntries(entries)
+	listP := f.Fs.Features().ListP
+	if listP == nil {
+		entries, err := f.Fs.List(ctx, dir)
+		if err != nil {
+			return err
+		}
+		return wrappedCallback(entries)
+	}
+	return listP(ctx, dir, wrappedCallback)
 }
 
 // ListR lists the objects and directories recursively into out.

backend/hasher/kv.go

@@ -6,6 +6,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"maps"
 	"strings"
 	"time"
 
@@ -195,9 +196,7 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
 		r.Fp = op.fp
 	}
 
-	for hashType, hashVal := range op.hashes {
-		r.Hashes[hashType] = hashVal
-	}
+	maps.Copy(r.Hashes, op.hashes)
 	if data, err = r.encode(op.key); err != nil {
 		return fmt.Errorf("marshal failed: %w", err)
 	}

backend/hidrive/hidrivehash/hidrivehash.go

@@ -52,10 +52,7 @@ func writeByBlock(p []byte, writer io.Writer, blockSize uint32, bytesInBlock *ui
 	total := len(p)
 	nullBytes := make([]byte, blockSize)
 	for len(p) > 0 {
-		toWrite := int(blockSize - *bytesInBlock)
-		if toWrite > len(p) {
-			toWrite = len(p)
-		}
+		toWrite := min(int(blockSize-*bytesInBlock), len(p))
 		c, err := writer.Write(p[:toWrite])
 		*bytesInBlock += uint32(c)
 		*onlyNullBytesInBlock = *onlyNullBytesInBlock && bytes.Equal(nullBytes[:toWrite], p[:toWrite])
@@ -276,7 +273,7 @@ func (h *hidriveHash) Sum(b []byte) []byte {
 	}
 
 	checksum := zeroSum
-	for i := 0; i < len(h.levels); i++ {
+	for i := range h.levels {
 		level := h.levels[i]
 		if i < len(h.levels)-1 {
 			// Aggregate non-empty non-final levels.

backend/hidrive/hidrivehash/hidrivehash_test.go

@@ -216,7 +216,7 @@ func TestLevelWrite(t *testing.T) {
 func TestLevelIsFull(t *testing.T) {
 	content := [hidrivehash.Size]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
 	l := hidrivehash.NewLevel()
-	for i := 0; i < 256; i++ {
+	for range 256 {
 		assert.False(t, l.(internal.LevelHash).IsFull())
 		written, err := l.Write(content[:])
 		assert.Equal(t, len(content), written)

backend/http/http.go

@@ -180,7 +180,6 @@ func getFsEndpoint(ctx context.Context, client *http.Client, url string, opt *Op
 	}
 	addHeaders(req, opt)
 	res, err := noRedir.Do(req)
-
 	if err != nil {
 		fs.Debugf(nil, "Assuming path is a file as HEAD request could not be sent: %v", err)
 		return createFileResult()
@@ -249,6 +248,14 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 	f.httpClient = client
 	f.endpoint = u
 	f.endpointURL = u.String()
+
+	if isFile {
+		// Correct root if definitely pointing to a file
+		f.root = path.Dir(f.root)
+		if f.root == "." || f.root == "/" {
+			f.root = ""
+		}
+	}
 	return isFile, nil
 }
 
@@ -505,7 +512,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		entries = append(entries, entry)
 		entriesMu.Unlock()
 	}
-	for i := 0; i < checkers; i++ {
+	for range checkers {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
@@ -740,7 +747,7 @@ It doesn't return anything.
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "set":
 		newOpt := f.opt

backend/iclouddrive/api/client.go

@@ -76,7 +76,7 @@ func (c *Client) DriveService() (*DriveService, error) {
 // This function is the main entry point for making requests to the iCloud
 // API. If the initial request returns a 401 (Unauthorized), it will try to
 // reauthenticate and retry the request.
-func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
 	resp, err = c.Session.Request(ctx, opts, request, response)
 	if err != nil && resp != nil {
 		// try to reauth
@@ -100,7 +100,7 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{
 // This function is useful when you have a session that is already
 // authenticated, but you need to make a request without triggering
 // a re-authentication.
-func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
 	// Make the request without re-authenticating
 	resp, err = c.Session.Request(ctx, opts, request, response)
 	return resp, err
@@ -161,6 +161,6 @@ func newRequestError(Status string, Text string) *RequestError {
 }
 
 // newErr orf makes a new error from sprintf parameters.
-func newRequestErrorf(Status string, Text string, Parameters ...interface{}) *RequestError {
+func newRequestErrorf(Status string, Text string, Parameters ...any) *RequestError {
 	return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))
 }

backend/iclouddrive/api/drive.go

@@ -476,7 +476,7 @@ func (d *DriveService) MoveItemByDriveID(ctx context.Context, id, etag, dstID st
 
 // CopyDocByItemID copies a document by its item ID.
 func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*DriveItemRaw, *http.Response, error) {
-	// putting name in info doesnt work. extension does work so assume this is a bug in the endpoint
+	// putting name in info doesn't work. extension does work so assume this is a bug in the endpoint
 	values := map[string]any{
 		"info_to_update": map[string]any{},
 	}
@@ -631,7 +631,7 @@ func NewUpdateFileInfo() UpdateFileInfo {
 		FileFlags: FileFlags{
 			IsExecutable: true,
 			IsHidden:     false,
-			IsWritable:   false,
+			IsWritable:   true,
 		},
 	}
 }
@@ -733,8 +733,8 @@ type DocumentUpdateResponse struct {
 			StatusCode   int    `json:"status_code"`
 			ErrorMessage string `json:"error_message"`
 		} `json:"status"`
-		OperationID interface{} `json:"operation_id"`
-		Document    *Document   `json:"document"`
+		OperationID any       `json:"operation_id"`
+		Document    *Document `json:"document"`
 	} `json:"results"`
 }
 
@@ -765,9 +765,9 @@ type Document struct {
 		IsWritable   bool `json:"is_writable"`
 		IsHidden     bool `json:"is_hidden"`
 	} `json:"file_flags"`
-	LastOpenedTime   int64       `json:"lastOpenedTime"`
-	RestorePath      interface{} `json:"restorePath"`
-	HasChainedParent bool        `json:"hasChainedParent"`
+	LastOpenedTime   int64 `json:"lastOpenedTime"`
+	RestorePath      any   `json:"restorePath"`
+	HasChainedParent bool  `json:"hasChainedParent"`
 }
 
 // DriveID returns the drive ID of the Document.

backend/iclouddrive/api/session.go

@@ -3,13 +3,13 @@ package api
 import (
 	"context"
 	"fmt"
+	"maps"
 	"net/http"
 	"net/url"
 	"slices"
 	"strings"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
-
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/lib/rest"
 )
@@ -35,7 +35,7 @@ type Session struct {
 // }
 
 // Request makes a request
-func (s *Session) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (*http.Response, error) {
+func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) {
 	resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)
 
 	if err != nil {
@@ -129,7 +129,7 @@ func (s *Session) AuthWithToken(ctx context.Context) error {
 
 // Validate2FACode validates the 2FA code
 func (s *Session) Validate2FACode(ctx context.Context, code string) error {
-	values := map[string]interface{}{"securityCode": map[string]string{"code": code}}
+	values := map[string]any{"securityCode": map[string]string{"code": code}}
 	body, err := IntoReader(values)
 	if err != nil {
 		return err
@@ -220,9 +220,7 @@ func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string
 		"Referer":                          fmt.Sprintf("%s/", homeEndpoint),
 		"User-Agent":                       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
 	}
-	for k, v := range overwrite {
-		headers[k] = v
-	}
+	maps.Copy(headers, overwrite)
 	return headers
 }
 
@@ -230,9 +228,7 @@ func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string
 func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {
 	headers := GetCommonHeaders(map[string]string{})
 	headers["Cookie"] = s.GetCookieString()
-	for k, v := range overwrite {
-		headers[k] = v
-	}
+	maps.Copy(headers, overwrite)
 	return headers
 }
 
@@ -254,9 +250,7 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
 		"Referer":      fmt.Sprintf("%s/", baseEndpoint),
 		"User-Agent":   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
 	}
-	for k, v := range overwrite {
-		headers[k] = v
-	}
+	maps.Copy(headers, overwrite)
 	return headers
 }
 
@@ -338,33 +332,33 @@ type AccountInfo struct {
 
 // ValidateDataDsInfo represents an validation info
 type ValidateDataDsInfo struct {
-	HsaVersion                         int           `json:"hsaVersion"`
-	LastName                           string        `json:"lastName"`
-	ICDPEnabled                        bool          `json:"iCDPEnabled"`
-	TantorMigrated                     bool          `json:"tantorMigrated"`
-	Dsid                               string        `json:"dsid"`
-	HsaEnabled                         bool          `json:"hsaEnabled"`
-	IsHideMyEmailSubscriptionActive    bool          `json:"isHideMyEmailSubscriptionActive"`
-	IroncadeMigrated                   bool          `json:"ironcadeMigrated"`
-	Locale                             string        `json:"locale"`
-	BrZoneConsolidated                 bool          `json:"brZoneConsolidated"`
-	ICDRSCapableDeviceList             string        `json:"ICDRSCapableDeviceList"`
-	IsManagedAppleID                   bool          `json:"isManagedAppleID"`
-	IsCustomDomainsFeatureAvailable    bool          `json:"isCustomDomainsFeatureAvailable"`
-	IsHideMyEmailFeatureAvailable      bool          `json:"isHideMyEmailFeatureAvailable"`
-	ContinueOnDeviceEligibleDeviceInfo []string      `json:"ContinueOnDeviceEligibleDeviceInfo"`
-	Gilligvited                        bool          `json:"gilligvited"`
-	AppleIDAliases                     []interface{} `json:"appleIdAliases"`
-	UbiquityEOLEnabled                 bool          `json:"ubiquityEOLEnabled"`
-	IsPaidDeveloper                    bool          `json:"isPaidDeveloper"`
-	CountryCode                        string        `json:"countryCode"`
-	NotificationID                     string        `json:"notificationId"`
-	PrimaryEmailVerified               bool          `json:"primaryEmailVerified"`
-	ADsID                              string        `json:"aDsID"`
-	Locked                             bool          `json:"locked"`
-	ICDRSCapableDeviceCount            int           `json:"ICDRSCapableDeviceCount"`
-	HasICloudQualifyingDevice          bool          `json:"hasICloudQualifyingDevice"`
-	PrimaryEmail                       string        `json:"primaryEmail"`
+	HsaVersion                         int      `json:"hsaVersion"`
+	LastName                           string   `json:"lastName"`
+	ICDPEnabled                        bool     `json:"iCDPEnabled"`
+	TantorMigrated                     bool     `json:"tantorMigrated"`
+	Dsid                               string   `json:"dsid"`
+	HsaEnabled                         bool     `json:"hsaEnabled"`
+	IsHideMyEmailSubscriptionActive    bool     `json:"isHideMyEmailSubscriptionActive"`
+	IroncadeMigrated                   bool     `json:"ironcadeMigrated"`
+	Locale                             string   `json:"locale"`
+	BrZoneConsolidated                 bool     `json:"brZoneConsolidated"`
+	ICDRSCapableDeviceList             string   `json:"ICDRSCapableDeviceList"`
+	IsManagedAppleID                   bool     `json:"isManagedAppleID"`
+	IsCustomDomainsFeatureAvailable    bool     `json:"isCustomDomainsFeatureAvailable"`
+	IsHideMyEmailFeatureAvailable      bool     `json:"isHideMyEmailFeatureAvailable"`
+	ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"`
+	Gilligvited                        bool     `json:"gilligvited"`
+	AppleIDAliases                     []any    `json:"appleIdAliases"`
+	UbiquityEOLEnabled                 bool     `json:"ubiquityEOLEnabled"`
+	IsPaidDeveloper                    bool     `json:"isPaidDeveloper"`
+	CountryCode                        string   `json:"countryCode"`
+	NotificationID                     string   `json:"notificationId"`
+	PrimaryEmailVerified               bool     `json:"primaryEmailVerified"`
+	ADsID                              string   `json:"aDsID"`
+	Locked                             bool     `json:"locked"`
+	ICDRSCapableDeviceCount            int      `json:"ICDRSCapableDeviceCount"`
+	HasICloudQualifyingDevice          bool     `json:"hasICloudQualifyingDevice"`
+	PrimaryEmail                       string   `json:"primaryEmail"`
 	AppleIDEntries                     []struct {
 		IsPrimary bool   `json:"isPrimary"`
 		Type      string `json:"type"`

backend/imagekit/util.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"slices"
 	"strconv"
 	"time"
 
@@ -142,12 +143,7 @@ func shouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
 	if resp == nil {
 		return false
 	}
-	for _, e := range retryErrorCodes {
-		if resp.StatusCode == e {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(retryErrorCodes, resp.StatusCode)
 }
 
 func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {

backend/internetarchive/internetarchive.go

@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"path"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -151,6 +152,19 @@ Owner is able to add custom keys. Metadata feature grabs all the keys including
 			Help:     "Host of InternetArchive Frontend.\n\nLeave blank for default value.",
 			Default:  "https://archive.org",
 			Advanced: true,
+		}, {
+			Name: "item_metadata",
+			Help: `Metadata to be set on the IA item, this is different from file-level metadata that can be set using --metadata-set.
+Format is key=value and the 'x-archive-meta-' prefix is automatically added.`,
+			Default:  []string{},
+			Hide:     fs.OptionHideConfigurator,
+			Advanced: true,
+		}, {
+			Name: "item_derive",
+			Help: `Whether to trigger derive on the IA item or not. If set to false, the item will not be derived by IA upon upload.
+The derive process produces a number of secondary files from an upload to make an upload more usable on the web.
+Setting this to false is useful for uploading files that are already in a format that IA can display or reduce burden on IA's infrastructure.`,
+			Default: true,
 		}, {
 			Name: "disable_checksum",
 			Help: `Don't ask the server to test against MD5 checksum calculated by rclone.
@@ -187,7 +201,7 @@ Only enable if you need to be guaranteed to be reflected after write operations.
 const iaItemMaxSize int64 = 1099511627776
 
 // metadata keys that are not writeable
-var roMetadataKey = map[string]interface{}{
+var roMetadataKey = map[string]any{
 	// do not add mtime here, it's a documented exception
 	"name": nil, "source": nil, "size": nil, "md5": nil,
 	"crc32": nil, "sha1": nil, "format": nil, "old_version": nil,
@@ -201,6 +215,8 @@ type Options struct {
 	Endpoint        string               `config:"endpoint"`
 	FrontEndpoint   string               `config:"front_endpoint"`
 	DisableChecksum bool                 `config:"disable_checksum"`
+	ItemMetadata    []string             `config:"item_metadata"`
+	ItemDerive      bool                 `config:"item_derive"`
 	WaitArchive     fs.Duration          `config:"wait_archive"`
 	Enc             encoder.MultiEncoder `config:"encoding"`
 }
@@ -790,17 +806,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		"x-amz-filemeta-rclone-update-track": updateTracker,
 
 		// we add some more headers for intuitive actions
-		"x-amz-auto-make-bucket":     "1",    // create an item if does not exist, do nothing if already
-		"x-archive-auto-make-bucket": "1",    // same as above in IAS3 original way
-		"x-archive-keep-old-version": "0",    // do not keep old versions (a.k.a. trashes in other clouds)
-		"x-archive-meta-mediatype":   "data", // mark media type of the uploading file as "data"
-		"x-archive-queue-derive":     "0",    // skip derivation process (e.g. encoding to smaller files, OCR on PDFs)
-		"x-archive-cascade-delete":   "1",    // enable "cascate delete" (delete all derived files in addition to the file itself)
+		"x-amz-auto-make-bucket":     "1", // create an item if does not exist, do nothing if already
+		"x-archive-auto-make-bucket": "1", // same as above in IAS3 original way
+		"x-archive-keep-old-version": "0", // do not keep old versions (a.k.a. trashes in other clouds)
+		"x-archive-cascade-delete":   "1", // enable "cascate delete" (delete all derived files in addition to the file itself)
 	}
+
 	if size >= 0 {
 		headers["Content-Length"] = fmt.Sprintf("%d", size)
 		headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
 	}
+
+	// This is IA's ITEM metadata, not file metadata
+	headers, err = o.appendItemMetadataHeaders(headers, o.fs.opt)
+	if err != nil {
+		return err
+	}
+
 	var mdata fs.Metadata
 	mdata, err = fs.GetMetadataOptions(ctx, o.fs, src, options)
 	if err == nil && mdata != nil {
@@ -863,6 +885,51 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return err
 }
 
+func (o *Object) appendItemMetadataHeaders(headers map[string]string, options Options) (newHeaders map[string]string, err error) {
+	metadataCounter := make(map[string]int)
+	metadataValues := make(map[string][]string)
+
+	// First pass: count occurrences and collect values
+	for _, v := range options.ItemMetadata {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) != 2 {
+			return newHeaders, errors.New("item metadata key=value should be in the form key=value")
+		}
+		key, value := parts[0], parts[1]
+		metadataCounter[key]++
+		metadataValues[key] = append(metadataValues[key], value)
+	}
+
+	// Second pass: add headers with appropriate prefixes
+	for key, count := range metadataCounter {
+		if count == 1 {
+			// Only one occurrence, use x-archive-meta-
+			headers[fmt.Sprintf("x-archive-meta-%s", key)] = metadataValues[key][0]
+		} else {
+			// Multiple occurrences, use x-archive-meta01-, x-archive-meta02-, etc.
+			for i, value := range metadataValues[key] {
+				headers[fmt.Sprintf("x-archive-meta%02d-%s", i+1, key)] = value
+			}
+		}
+	}
+
+	if o.fs.opt.ItemDerive {
+		headers["x-archive-queue-derive"] = "1"
+	} else {
+		headers["x-archive-queue-derive"] = "0"
+	}
+
+	fs.Debugf(o, "Setting IA item derive: %t", o.fs.opt.ItemDerive)
+
+	for k, v := range headers {
+		if strings.HasPrefix(k, "x-archive-meta") {
+			fs.Debugf(o, "Setting IA item metadata: %s=%s", k, v)
+		}
+	}
+
+	return headers, nil
+}
+
 // Remove an object
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
@@ -925,10 +992,8 @@ func (o *Object) Metadata(ctx context.Context) (m fs.Metadata, err error) {
 
 func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	if resp != nil {
-		for _, e := range retryErrorCodes {
-			if resp.StatusCode == e {
-				return true, err
-			}
+		if slices.Contains(retryErrorCodes, resp.StatusCode) {
+			return true, err
 		}
 	}
 	// Ok, not an awserr, check for generic failure conditions
@@ -1081,13 +1146,7 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath, tracker string, newSiz
 			}
 
 			fileTrackers, _ := listOrString(iaFile.UpdateTrack)
-			trackerMatch := false
-			for _, v := range fileTrackers {
-				if v == tracker {
-					trackerMatch = true
-					break
-				}
-			}
+			trackerMatch := slices.Contains(fileTrackers, tracker)
 			if !trackerMatch {
 				continue
 			}

backend/jottacloud/api/types.go

@@ -70,7 +70,7 @@ func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 // MarshalJSON turns a Rfc3339Time into JSON
 func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
+	return fmt.Appendf(nil, "\"%s\"", t.String()), nil
 }
 
 // LoginToken is struct representing the login token generated in the WebUI
@@ -165,25 +165,25 @@ type DeviceRegistrationResponse struct {
 
 // CustomerInfo provides general information about the account. Required for finding the correct internal username.
 type CustomerInfo struct {
-	Username          string      `json:"username"`
-	Email             string      `json:"email"`
-	Name              string      `json:"name"`
-	CountryCode       string      `json:"country_code"`
-	LanguageCode      string      `json:"language_code"`
-	CustomerGroupCode string      `json:"customer_group_code"`
-	BrandCode         string      `json:"brand_code"`
-	AccountType       string      `json:"account_type"`
-	SubscriptionType  string      `json:"subscription_type"`
-	Usage             int64       `json:"usage"`
-	Quota             int64       `json:"quota"`
-	BusinessUsage     int64       `json:"business_usage"`
-	BusinessQuota     int64       `json:"business_quota"`
-	WriteLocked       bool        `json:"write_locked"`
-	ReadLocked        bool        `json:"read_locked"`
-	LockedCause       interface{} `json:"locked_cause"`
-	WebHash           string      `json:"web_hash"`
-	AndroidHash       string      `json:"android_hash"`
-	IOSHash           string      `json:"ios_hash"`
+	Username          string `json:"username"`
+	Email             string `json:"email"`
+	Name              string `json:"name"`
+	CountryCode       string `json:"country_code"`
+	LanguageCode      string `json:"language_code"`
+	CustomerGroupCode string `json:"customer_group_code"`
+	BrandCode         string `json:"brand_code"`
+	AccountType       string `json:"account_type"`
+	SubscriptionType  string `json:"subscription_type"`
+	Usage             int64  `json:"usage"`
+	Quota             int64  `json:"quota"`
+	BusinessUsage     int64  `json:"business_usage"`
+	BusinessQuota     int64  `json:"business_quota"`
+	WriteLocked       bool   `json:"write_locked"`
+	ReadLocked        bool   `json:"read_locked"`
+	LockedCause       any    `json:"locked_cause"`
+	WebHash           string `json:"web_hash"`
+	AndroidHash       string `json:"android_hash"`
+	IOSHash           string `json:"ios_hash"`
 }
 
 // TrashResponse is returned when emptying the Trash

backend/jottacloud/jottacloud.go

@@ -31,7 +31,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
@@ -1264,7 +1264,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		Parameters: url.Values{},
 	}
 	opts.Parameters.Set("mode", "liststream")
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {

backend/linkbox/linkbox.go

@@ -193,7 +193,7 @@ func (o *Object) set(e *entity) {
 // Call linkbox with the query in opts and return result
 //
 // This will be checked for error and an error will be returned if Status != 1
-func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result interface{}) error {
+func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result any) error {
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, opts, nil, &result)
 		return f.shouldRetry(ctx, resp, err)

backend/local/local.go

@@ -1046,7 +1046,7 @@ you can try to change the output.`,
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (any, error) {
 	switch name {
 	case "noop":
 		if txt, ok := opt["error"]; ok {
@@ -1056,7 +1056,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			return nil, errors.New(txt)
 		}
 		if _, ok := opt["echo"]; ok {
-			out := map[string]interface{}{}
+			out := map[string]any{}
 			out["name"] = name
 			out["arg"] = arg
 			out["opt"] = opt

backend/local/local_internal_test.go

@@ -86,7 +86,7 @@ func TestVerifyCopy(t *testing.T) {
 	require.NoError(t, err)
 	src.(*Object).fs.opt.NoCheckUpdated = true
 
-	for i := 0; i < 100; i++ {
+	for i := range 100 {
 		go r.WriteFile(src.Remote(), fmt.Sprintf("some new content %d", i), src.ModTime(context.Background()))
 	}
 	_, err = operations.Copy(context.Background(), r.Fremote, nil, filePath+"2", src)

backend/mailru/api/m1.go

@@ -63,8 +63,8 @@ type UserInfoResponse struct {
 				Prolong      bool   `json:"prolong"`
 				Promocodes   struct {
 				} `json:"promocodes"`
-				Subscription []interface{} `json:"subscription"`
-				Version      string        `json:"version"`
+				Subscription []any  `json:"subscription"`
+				Version      string `json:"version"`
 			} `json:"billing"`
 			Bonuses struct {
 				CameraUpload bool `json:"camera_upload"`

backend/mailru/mailru.go

@@ -901,7 +901,7 @@ func (t *treeState) NextRecord() (fs.DirEntry, error) {
 		return nil, nil
 	case api.ListParseUnknown15:
 		skip := int(r.ReadPu32())
-		for i := 0; i < skip; i++ {
+		for range skip {
 			r.ReadPu32()
 			r.ReadPu32()
 		}
@@ -1768,7 +1768,7 @@ func (f *Fs) eligibleForSpeedup(remote string, size int64, options ...fs.OpenOpt
 func (f *Fs) parseSpeedupPatterns(patternString string) (err error) {
 	f.speedupGlobs = nil
 	f.speedupAny = false
-	uniqueValidPatterns := make(map[string]interface{})
+	uniqueValidPatterns := make(map[string]any)
 
 	for _, pattern := range strings.Split(patternString, ",") {
 		pattern = strings.ToLower(strings.TrimSpace(pattern))
@@ -2131,10 +2131,7 @@ func getTransferRange(size int64, options ...fs.OpenOption) (start int64, end in
 	if limit < 0 {
 		limit = size - offset
 	}
-	end = offset + limit
-	if end > size {
-		end = size
-	}
+	end = min(offset+limit, size)
 	partial = !(offset == 0 && end == size)
 	return offset, end, partial
 }

backend/mailru/mrhash/mrhash_test.go

@@ -11,7 +11,7 @@ import (
 
 func testChunk(t *testing.T, chunk int) {
 	data := make([]byte, chunk)
-	for i := 0; i < chunk; i++ {
+	for i := range chunk {
 		data[i] = 'A'
 	}
 	for _, test := range []struct {

backend/mega/mega.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -218,11 +219,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		srv = mega.New().SetClient(fshttp.NewClient(ctx))
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
 		srv.SetHTTPS(opt.UseHTTPS)
-		srv.SetLogger(func(format string, v ...interface{}) {
+		srv.SetLogger(func(format string, v ...any) {
 			fs.Infof("*go-mega*", format, v...)
 		})
 		if opt.Debug {
-			srv.SetDebugger(func(format string, v ...interface{}) {
+			srv.SetDebugger(func(format string, v ...any) {
 				fs.Debugf("*go-mega*", format, v...)
 			})
 		}
@@ -498,11 +499,8 @@ func (f *Fs) list(ctx context.Context, dir *mega.Node, fn listFn) (found bool, e
 	if err != nil {
 		return false, fmt.Errorf("list failed: %w", err)
 	}
-	for _, item := range nodes {
-		if fn(item) {
-			found = true
-			break
-		}
+	if slices.ContainsFunc(nodes, fn) {
+		found = true
 	}
 	return
 }
@@ -1156,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Upload the chunks
 	// FIXME do this in parallel
-	for id := 0; id < u.Chunks(); id++ {
+	for id := range u.Chunks() {
 		_, chunkSize, err := u.ChunkLocation(id)
 		if err != nil {
 			return fmt.Errorf("upload failed to read chunk location: %w", err)

backend/memory/memory.go

@@ -17,7 +17,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/bucket"
 )
 
@@ -383,7 +383,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // of listing recursively that doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
 	bucket, directory := f.split(dir)
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	entries := fs.DirEntries{}
 	listR := func(bucket, directory, prefix string, addBucket bool) error {
 		err = f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error {

backend/memory/memory_internal_test.go

@@ -29,7 +29,7 @@ func testPurgeListDeadlock(t *testing.T) {
 	r.Fremote.Features().Disable("Purge") // force fallback-purge
 
 	// make a lot of files to prevent it from finishing too quickly
-	for i := 0; i < 100; i++ {
+	for i := range 100 {
 		dst := "file" + fmt.Sprint(i) + ".txt"
 		r.WriteObject(ctx, dst, "hello", t1)
 	}

backend/netstorage/netstorage.go

@@ -28,7 +28,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
 )
@@ -274,7 +274,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 }
 
 // Command the backend to run a named commands: du and symlink
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "du":
 		// No arg parsing needed, the path is passed in the fs
@@ -516,7 +516,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		return fs.ErrorDirNotFound
 	}
 
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	for resumeStart := u.Path; resumeStart != ""; {
 		var files []File
 		files, resumeStart, err = f.netStorageListRequest(ctx, URL, u.Path)
@@ -858,7 +858,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 
 // callBackend calls NetStorage API using either rest.Call or rest.CallXML function,
 // depending on whether the response is required
-func (f *Fs) callBackend(ctx context.Context, URL, method, actionHeader string, noResponse bool, response interface{}, options []fs.OpenOption) (io.ReadCloser, error) {
+func (f *Fs) callBackend(ctx context.Context, URL, method, actionHeader string, noResponse bool, response any, options []fs.OpenOption) (io.ReadCloser, error) {
 	opts := rest.Opts{
 		Method:     method,
 		RootURL:    URL,
@@ -1080,7 +1080,7 @@ func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.Ope
 }
 
 // netStorageDuRequest performs a NetStorage du request
-func (f *Fs) netStorageDuRequest(ctx context.Context) (interface{}, error) {
+func (f *Fs) netStorageDuRequest(ctx context.Context) (any, error) {
 	URL := f.url("")
 	const actionHeader = "version=1&action=du&format=xml&encoding=utf-8"
 	duResp := &Du{}
@@ -1100,7 +1100,7 @@ func (f *Fs) netStorageDuRequest(ctx context.Context) (interface{}, error) {
 }
 
 // netStorageDuRequest performs a NetStorage symlink request
-func (f *Fs) netStorageSymlinkRequest(ctx context.Context, URL string, dst string, modTime *int64) (interface{}, error) {
+func (f *Fs) netStorageSymlinkRequest(ctx context.Context, URL string, dst string, modTime *int64) (any, error) {
 	target := url.QueryEscape(strings.TrimSuffix(dst, "/"))
 	actionHeader := "version=1&action=symlink&target=" + target
 	if modTime != nil {

backend/onedrive/metadata.go

@@ -396,10 +396,57 @@ func (m *Metadata) WritePermissions(ctx context.Context) (err error) {
 	return nil
 }
 
+// Order the permissions so that any with users come first.
+//
+// This is to work around a quirk with Graph:
+//
+// 1. You are adding permissions for both a group and a user.
+// 2. The user is a member of the group.
+// 3. The permissions for the group and user are the same.
+// 4. You are adding the group permission before the user permission.
+//
+// When all of the above are true, Graph indicates it has added the
+// user permission, but it immediately drops it
+//
+// See: https://github.com/rclone/rclone/issues/8465
+func (m *Metadata) orderPermissions(xs []*api.PermissionsType) {
+	// Return true if identity has any user permissions
+	hasUserIdentity := func(identity *api.IdentitySet) bool {
+		if identity == nil {
+			return false
+		}
+		return identity.User.ID != "" || identity.User.DisplayName != "" || identity.User.Email != "" || identity.User.LoginName != ""
+	}
+	// Return true if p has any user permissions
+	hasUser := func(p *api.PermissionsType) bool {
+		if hasUserIdentity(p.GetGrantedTo(m.fs.driveType)) {
+			return true
+		}
+		for _, identity := range p.GetGrantedToIdentities(m.fs.driveType) {
+			if hasUserIdentity(identity) {
+				return true
+			}
+		}
+		return false
+	}
+	// Put Permissions with a user first, leaving unsorted otherwise
+	slices.SortStableFunc(xs, func(a, b *api.PermissionsType) int {
+		aHasUser := hasUser(a)
+		bHasUser := hasUser(b)
+		if aHasUser && !bHasUser {
+			return -1
+		} else if !aHasUser && bHasUser {
+			return 1
+		}
+		return 0
+	})
+}
+
 // sortPermissions sorts the permissions (to be written) into add, update, and remove queues
 func (m *Metadata) sortPermissions() (add, update, remove []*api.PermissionsType) {
 	new, old := m.queuedPermissions, m.permissions
 	if len(old) == 0 || m.permsAddOnly {
+		m.orderPermissions(new)
 		return new, nil, nil // they must all be "add"
 	}
 
@@ -447,6 +494,9 @@ func (m *Metadata) sortPermissions() (add, update, remove []*api.PermissionsType
 			remove = append(remove, o)
 		}
 	}
+	m.orderPermissions(add)
+	m.orderPermissions(update)
+	m.orderPermissions(remove)
 	return add, update, remove
 }
 

backend/onedrive/metadata_test.go

@@ -0,0 +1,125 @@
+package onedrive
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/rclone/rclone/backend/onedrive/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOrderPermissions(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []*api.PermissionsType
+		expected []string
+	}{
+		{
+			name:     "empty",
+			input:    []*api.PermissionsType{},
+			expected: []string(nil),
+		},
+		{
+			name: "users first, then group, then none",
+			input: []*api.PermissionsType{
+				{ID: "1", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group1"}}},
+				{ID: "2", GrantedToIdentities: []*api.IdentitySet{{User: api.Identity{DisplayName: "Alice"}}}},
+				{ID: "3", GrantedTo: &api.IdentitySet{User: api.Identity{DisplayName: "Alice"}}},
+				{ID: "4"},
+			},
+			expected: []string{"2", "3", "1", "4"},
+		},
+		{
+			name: "same type unsorted",
+			input: []*api.PermissionsType{
+				{ID: "b", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group B"}}},
+				{ID: "a", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group A"}}},
+				{ID: "c", GrantedToIdentities: []*api.IdentitySet{{Group: api.Identity{DisplayName: "Group A"}}, {User: api.Identity{DisplayName: "Alice"}}}},
+			},
+			expected: []string{"c", "b", "a"},
+		},
+		{
+			name: "all user identities",
+			input: []*api.PermissionsType{
+				{ID: "c", GrantedTo: &api.IdentitySet{User: api.Identity{DisplayName: "Bob"}}},
+				{ID: "a", GrantedTo: &api.IdentitySet{User: api.Identity{Email: "alice@example.com"}}},
+				{ID: "b", GrantedToIdentities: []*api.IdentitySet{{User: api.Identity{LoginName: "user3"}}}},
+			},
+			expected: []string{"c", "a", "b"},
+		},
+		{
+			name: "no user or group info",
+			input: []*api.PermissionsType{
+				{ID: "z"},
+				{ID: "x"},
+				{ID: "y"},
+			},
+			expected: []string{"z", "x", "y"},
+		},
+	}
+
+	for _, driveType := range []string{driveTypePersonal, driveTypeBusiness} {
+		t.Run(driveType, func(t *testing.T) {
+			for _, tt := range tests {
+				m := &Metadata{fs: &Fs{driveType: driveType}}
+				t.Run(tt.name, func(t *testing.T) {
+					if driveType == driveTypeBusiness {
+						for i := range tt.input {
+							tt.input[i].GrantedToV2 = tt.input[i].GrantedTo
+							tt.input[i].GrantedTo = nil
+							tt.input[i].GrantedToIdentitiesV2 = tt.input[i].GrantedToIdentities
+							tt.input[i].GrantedToIdentities = nil
+						}
+					}
+					m.orderPermissions(tt.input)
+					var gotIDs []string
+					for _, p := range tt.input {
+						gotIDs = append(gotIDs, p.ID)
+					}
+					assert.Equal(t, tt.expected, gotIDs)
+				})
+			}
+		})
+	}
+}
+
+func TestOrderPermissionsJSON(t *testing.T) {
+	testJSON := `[
+  {
+    "id": "1",
+    "grantedToV2": {
+      "group": {
+        "id": "group@example.com"
+      }
+    },
+    "roles": [
+      "write"
+    ]
+  },
+  {
+    "id": "2",
+    "grantedToV2": {
+      "user": {
+        "id": "user@example.com"
+      }
+    },
+    "roles": [
+      "write"
+    ]
+  }
+]`
+
+	var testPerms []*api.PermissionsType
+	err := json.Unmarshal([]byte(testJSON), &testPerms)
+	require.NoError(t, err)
+
+	m := &Metadata{fs: &Fs{driveType: driveTypeBusiness}}
+	m.orderPermissions(testPerms)
+	var gotIDs []string
+	for _, p := range testPerms {
+		gotIDs = append(gotIDs, p.ID)
+	}
+	assert.Equal(t, []string{"2", "1"}, gotIDs)
+
+}

backend/onedrive/onedrive.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
@@ -1396,7 +1397,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	// So we have to filter things outside of the root which is
 	// inefficient.
 
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 
 	// list a folder conventionally - used for shared folders
 	var listFolder func(dir string) error
@@ -2532,10 +2533,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
 	remaining := size
 	position := int64(0)
 	for remaining > 0 {
-		n := int64(o.fs.opt.ChunkSize)
-		if remaining < n {
-			n = remaining
-		}
+		n := min(remaining, int64(o.fs.opt.ChunkSize))
 		seg := readers.NewRepeatableReader(io.LimitReader(in, n))
 		fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
 		info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)

backend/onedrive/quickxorhash/quickxorhash.go

@@ -86,7 +86,7 @@ func (q *quickXorHash) Write(p []byte) (n int, err error) {
 
 // Calculate the current checksum
 func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
-	for i := 0; i < dataSize; i++ {
+	for i := range dataSize {
 		shift := (i * 11) % 160
 		shiftBytes := shift / 8
 		shiftBits := shift % 8

backend/onedrive/quickxorhash/quickxorhash_test.go

@@ -130,10 +130,7 @@ func TestQuickXorHashByBlock(t *testing.T) {
 			require.NoError(t, err, what)
 			h := New()
 			for i := 0; i < len(in); i += blockSize {
-				end := i + blockSize
-				if end > len(in) {
-					end = len(in)
-				}
+				end := min(i+blockSize, len(in))
 				n, err := h.Write(in[i:end])
 				require.Equal(t, end-i, n, what)
 				require.NoError(t, err, what)

backend/opendrive/opendrive.go

@@ -92,6 +92,21 @@ Note that these chunks are buffered in memory so increasing them will
 increase memory use.`,
 			Default:  10 * fs.Mebi,
 			Advanced: true,
+		}, {
+			Name:     "access",
+			Help:     "Files and folders will be uploaded with this access permission (default private)",
+			Default:  "private",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "private",
+				Help:  "The file or folder access can be granted in a way that will allow select users to view, read or write what is absolutely essential for them.",
+			}, {
+				Value: "public",
+				Help:  "The file or folder can be downloaded by anyone from a web browser. The link can be shared in any way,",
+			}, {
+				Value: "hidden",
+				Help:  "The file or folder can be accessed has the same restrictions as  Public if the user knows the URL of the file or folder link in order to access the contents",
+			}},
 		}},
 	})
 }
@@ -102,6 +117,7 @@ type Options struct {
 	Password  string               `config:"password"`
 	Enc       encoder.MultiEncoder `config:"encoding"`
 	ChunkSize fs.SizeSuffix        `config:"chunk_size"`
+	Access    string               `config:"access"`
 }
 
 // Fs represents a remote server
@@ -475,7 +491,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Method: "POST",
 		Path:   "/file/move_copy.json",
 	}
-	var request interface{} = moveCopyFileData
+	var request any = moveCopyFileData
 
 	// use /file/rename.json if moving within the same directory
 	_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
@@ -548,7 +564,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		Method: "POST",
 		Path:   "/folder/move_copy.json",
 	}
-	var request interface{} = moveFolderData
+	var request any = moveFolderData
 
 	// use /folder/rename.json if moving within the same parent directory
 	if srcDirectoryID == dstDirectoryID {
@@ -735,6 +751,23 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
+// getAccessLevel is a helper function to determine access level integer
+func getAccessLevel(access string) int64 {
+	var accessLevel int64
+	switch access {
+	case "private":
+		accessLevel = 0
+	case "public":
+		accessLevel = 1
+	case "hidden":
+		accessLevel = 2
+	default:
+		accessLevel = 0
+		fs.Errorf(nil, "Invalid access: %s, defaulting to private", access)
+	}
+	return accessLevel
+}
+
 // DirCacher methods
 
 // CreateDir makes a directory with pathID as parent and name leaf
@@ -747,7 +780,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			SessionID:           f.session.SessionID,
 			FolderName:          f.opt.Enc.FromStandardName(leaf),
 			FolderSubParent:     pathID,
-			FolderIsPublic:      0,
+			FolderIsPublic:      getAccessLevel(f.opt.Access),
 			FolderPublicUpl:     0,
 			FolderPublicDisplay: 0,
 			FolderPublicDnl:     0,
@@ -1009,10 +1042,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	chunkCounter := 0
 
 	for remainingBytes > 0 {
-		currentChunkSize := int64(o.fs.opt.ChunkSize)
-		if currentChunkSize > remainingBytes {
-			currentChunkSize = remainingBytes
-		}
+		currentChunkSize := min(int64(o.fs.opt.ChunkSize), remainingBytes)
 		remainingBytes -= currentChunkSize
 		fs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes)
 
@@ -1080,7 +1110,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Set permissions
 	err = o.fs.pacer.Call(func() (bool, error) {
-		update := permissions{SessionID: o.fs.session.SessionID, FileID: o.id, FileIsPublic: 0}
+		update := permissions{SessionID: o.fs.session.SessionID, FileID: o.id, FileIsPublic: getAccessLevel(o.fs.opt.Access)}
 		// fs.Debugf(nil, "Permissions : %#v", update)
 		opts := rest.Opts{
 			Method:     "POST",

backend/oracleobjectstorage/command.go

@@ -131,7 +131,7 @@ If it is a string or a []string it will be shown to the user
 otherwise it will be JSON encoded and shown to the user like that
 */
 func (f *Fs) Command(ctx context.Context, commandName string, args []string,
-	opt map[string]string) (result interface{}, err error) {
+	opt map[string]string) (result any, err error) {
 	// fs.Debugf(f, "command %v, args: %v, opts:%v", commandName, args, opt)
 	switch commandName {
 	case operationRename:
@@ -159,7 +159,7 @@ func (f *Fs) Command(ctx context.Context, commandName string, args []string,
 	}
 }
 
-func (f *Fs) rename(ctx context.Context, remote, newName string) (interface{}, error) {
+func (f *Fs) rename(ctx context.Context, remote, newName string) (any, error) {
 	if remote == "" {
 		return nil, fmt.Errorf("path to object file cannot be empty")
 	}
@@ -332,7 +332,7 @@ func (f *Fs) listMultipartUploadParts(ctx context.Context, bucketName, bucketPat
 	return uploadedParts, nil
 }
 
-func (f *Fs) restore(ctx context.Context, opt map[string]string) (interface{}, error) {
+func (f *Fs) restore(ctx context.Context, opt map[string]string) (any, error) {
 	req := objectstorage.RestoreObjectsRequest{
 		NamespaceName:         common.String(f.opt.Namespace),
 		RestoreObjectsDetails: objectstorage.RestoreObjectsDetails{},

backend/oracleobjectstorage/copy.go

@@ -112,7 +112,7 @@ func copyObjectWaitForWorkRequest(ctx context.Context, wID *string, entityType s
 			string(objectstorage.WorkRequestSummaryStatusCanceled),
 			string(objectstorage.WorkRequestStatusFailed),
 		},
-		Refresh: func() (interface{}, string, error) {
+		Refresh: func() (any, string, error) {
 			getWorkRequestRequest := objectstorage.GetWorkRequestRequest{}
 			getWorkRequestRequest.WorkRequestId = wID
 			workRequestResponse, err := client.GetWorkRequest(context.Background(), getWorkRequestRequest)

backend/oracleobjectstorage/object.go

@@ -131,7 +131,7 @@ func (o *Object) setMetaData(
 	contentMd5 *string,
 	contentType *string,
 	lastModified *common.SDKTime,
-	storageTier interface{},
+	storageTier any,
 	meta map[string]string) error {
 
 	if contentLength != nil {

backend/oracleobjectstorage/oracleobjectstorage.go

@@ -18,8 +18,8 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/pacer"
 )
@@ -649,7 +649,7 @@ of listing recursively that doing a directory traversal.
 */
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
 	bucketName, directory := f.split(dir)
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	listR := func(bucket, directory, prefix string, addBucket bool) error {
 		return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
 			entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)

backend/oracleobjectstorage/waiter.go

@@ -5,6 +5,7 @@ package oracleobjectstorage
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 
@@ -23,7 +24,7 @@ var refreshGracePeriod = 30 * time.Second
 //
 // `state` is the latest state of that object. And `err` is any error that
 // may have happened while refreshing the state.
-type StateRefreshFunc func() (result interface{}, state string, err error)
+type StateRefreshFunc func() (result any, state string, err error)
 
 // StateChangeConf is the configuration struct used for `WaitForState`.
 type StateChangeConf struct {
@@ -56,7 +57,7 @@ type StateChangeConf struct {
 // reach the target state.
 //
 // Cancellation from the passed in context will cancel the refresh loop
-func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (interface{}, error) {
+func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (any, error) {
 	// fs.Debugf(entityType, "Waiting for state to become: %s", conf.Target)
 
 	notfoundTick := 0
@@ -72,7 +73,7 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
 	}
 
 	type Result struct {
-		Result interface{}
+		Result any
 		State  string
 		Error  error
 		Done   bool
@@ -165,12 +166,9 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
 					}
 				}
 
-				for _, allowed := range conf.Pending {
-					if currentState == allowed {
-						found = true
-						targetOccurrence = 0
-						break
-					}
+				if slices.Contains(conf.Pending, currentState) {
+					found = true
+					targetOccurrence = 0
 				}
 
 				if !found && len(conf.Pending) > 0 {
@@ -278,8 +276,8 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
 // NotFoundError resource not found error
 type NotFoundError struct {
 	LastError    error
-	LastRequest  interface{}
-	LastResponse interface{}
+	LastRequest  any
+	LastResponse any
 	Message      string
 	Retries      int
 }

backend/pcloud/pcloud.go

@@ -27,7 +27,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
@@ -631,7 +631,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // ListR lists the objects and directories of the Fs starting
 // from dir recursively into out.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	err = f.listHelper(ctx, dir, true, func(o fs.DirEntry) error {
 		return list.Add(o)
 	})
@@ -990,10 +990,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	if err != nil {
 		return nil, err
 	}
-	free := q.Quota - q.UsedQuota
-	if free < 0 {
-		free = 0
-	}
+	free := max(q.Quota-q.UsedQuota, 0)
 	usage = &fs.Usage{
 		Total: fs.NewUsageValue(q.Quota),     // quota of bytes that can be used
 		Used:  fs.NewUsageValue(q.UsedQuota), // bytes in use
@@ -1324,7 +1321,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		// sometimes pcloud leaves a half complete file on
 		// error, so delete it if it exists, trying a few times
-		for i := 0; i < 5; i++ {
+		for range 5 {
 			delObj, delErr := o.fs.NewObject(ctx, o.remote)
 			if delErr == nil && delObj != nil {
 				_ = delObj.Remove(ctx)

backend/pcloud/writer_at.go

@@ -37,7 +37,7 @@ func (c *writerAt) Close() error {
 	}
 	sizeOk := false
 	sizeLastSeen := int64(0)
-	for retry := 0; retry < 5; retry++ {
+	for retry := range 5 {
 		fs.Debugf(c.remote, "checking file size: try %d/5", retry)
 		obj, err := c.fs.NewObject(c.ctx, c.remote)
 		if err != nil {

backend/pikpak/api/types.go

@@ -71,14 +71,14 @@ type Error struct {
 
 // ErrorDetails contains further details of api error
 type ErrorDetails struct {
-	Type         string        `json:"@type,omitempty"`
-	Reason       string        `json:"reason,omitempty"`
-	Domain       string        `json:"domain,omitempty"`
-	Metadata     struct{}      `json:"metadata,omitempty"` // TODO: undiscovered yet
-	Locale       string        `json:"locale,omitempty"`   // e.g. "en"
-	Message      string        `json:"message,omitempty"`
-	StackEntries []interface{} `json:"stack_entries,omitempty"` // TODO: undiscovered yet
-	Detail       string        `json:"detail,omitempty"`
+	Type         string   `json:"@type,omitempty"`
+	Reason       string   `json:"reason,omitempty"`
+	Domain       string   `json:"domain,omitempty"`
+	Metadata     struct{} `json:"metadata,omitempty"` // TODO: undiscovered yet
+	Locale       string   `json:"locale,omitempty"`   // e.g. "en"
+	Message      string   `json:"message,omitempty"`
+	StackEntries []any    `json:"stack_entries,omitempty"` // TODO: undiscovered yet
+	Detail       string   `json:"detail,omitempty"`
 }
 
 // Error returns a string for the error and satisfies the error interface
@@ -168,44 +168,44 @@ type FileList struct {
 // for a single file, i.e. supports for higher `--multi-thread-streams=N`.
 // However, it is not generally applicable as it is only for media.
 type File struct {
-	Apps              []*FileApp    `json:"apps,omitempty"`
-	Audit             *FileAudit    `json:"audit,omitempty"`
-	Collection        string        `json:"collection,omitempty"` // TODO
-	CreatedTime       Time          `json:"created_time,omitempty"`
-	DeleteTime        Time          `json:"delete_time,omitempty"`
-	FileCategory      string        `json:"file_category,omitempty"` // "AUDIO", "VIDEO"
-	FileExtension     string        `json:"file_extension,omitempty"`
-	FolderType        string        `json:"folder_type,omitempty"`
-	Hash              string        `json:"hash,omitempty"` // custom hash with a form of sha1sum
-	IconLink          string        `json:"icon_link,omitempty"`
-	ID                string        `json:"id,omitempty"`
-	Kind              string        `json:"kind,omitempty"` // "drive#file"
-	Links             *FileLinks    `json:"links,omitempty"`
-	Md5Checksum       string        `json:"md5_checksum,omitempty"`
-	Medias            []*Media      `json:"medias,omitempty"`
-	MimeType          string        `json:"mime_type,omitempty"`
-	ModifiedTime      Time          `json:"modified_time,omitempty"` // updated when renamed or moved
-	Name              string        `json:"name,omitempty"`
-	OriginalFileIndex int           `json:"original_file_index,omitempty"` // TODO
-	OriginalURL       string        `json:"original_url,omitempty"`
-	Params            *FileParams   `json:"params,omitempty"`
-	ParentID          string        `json:"parent_id,omitempty"`
-	Phase             string        `json:"phase,omitempty"`
-	Revision          int           `json:"revision,omitempty,string"`
-	ReferenceEvents   []interface{} `json:"reference_events"`
-	ReferenceResource interface{}   `json:"reference_resource"`
-	Size              int64         `json:"size,omitempty,string"`
-	SortName          string        `json:"sort_name,omitempty"`
-	Space             string        `json:"space,omitempty"`
-	SpellName         []interface{} `json:"spell_name,omitempty"` // TODO maybe list of something?
-	Starred           bool          `json:"starred,omitempty"`
-	Tags              []interface{} `json:"tags"`
-	ThumbnailLink     string        `json:"thumbnail_link,omitempty"`
-	Trashed           bool          `json:"trashed,omitempty"`
-	UserID            string        `json:"user_id,omitempty"`
-	UserModifiedTime  Time          `json:"user_modified_time,omitempty"`
-	WebContentLink    string        `json:"web_content_link,omitempty"`
-	Writable          bool          `json:"writable,omitempty"`
+	Apps              []*FileApp  `json:"apps,omitempty"`
+	Audit             *FileAudit  `json:"audit,omitempty"`
+	Collection        string      `json:"collection,omitempty"` // TODO
+	CreatedTime       Time        `json:"created_time,omitempty"`
+	DeleteTime        Time        `json:"delete_time,omitempty"`
+	FileCategory      string      `json:"file_category,omitempty"` // "AUDIO", "VIDEO"
+	FileExtension     string      `json:"file_extension,omitempty"`
+	FolderType        string      `json:"folder_type,omitempty"`
+	Hash              string      `json:"hash,omitempty"` // custom hash with a form of sha1sum
+	IconLink          string      `json:"icon_link,omitempty"`
+	ID                string      `json:"id,omitempty"`
+	Kind              string      `json:"kind,omitempty"` // "drive#file"
+	Links             *FileLinks  `json:"links,omitempty"`
+	Md5Checksum       string      `json:"md5_checksum,omitempty"`
+	Medias            []*Media    `json:"medias,omitempty"`
+	MimeType          string      `json:"mime_type,omitempty"`
+	ModifiedTime      Time        `json:"modified_time,omitempty"` // updated when renamed or moved
+	Name              string      `json:"name,omitempty"`
+	OriginalFileIndex int         `json:"original_file_index,omitempty"` // TODO
+	OriginalURL       string      `json:"original_url,omitempty"`
+	Params            *FileParams `json:"params,omitempty"`
+	ParentID          string      `json:"parent_id,omitempty"`
+	Phase             string      `json:"phase,omitempty"`
+	Revision          int         `json:"revision,omitempty,string"`
+	ReferenceEvents   []any       `json:"reference_events"`
+	ReferenceResource any         `json:"reference_resource"`
+	Size              int64       `json:"size,omitempty,string"`
+	SortName          string      `json:"sort_name,omitempty"`
+	Space             string      `json:"space,omitempty"`
+	SpellName         []any       `json:"spell_name,omitempty"` // TODO maybe list of something?
+	Starred           bool        `json:"starred,omitempty"`
+	Tags              []any       `json:"tags"`
+	ThumbnailLink     string      `json:"thumbnail_link,omitempty"`
+	Trashed           bool        `json:"trashed,omitempty"`
+	UserID            string      `json:"user_id,omitempty"`
+	UserModifiedTime  Time        `json:"user_modified_time,omitempty"`
+	WebContentLink    string      `json:"web_content_link,omitempty"`
+	Writable          bool        `json:"writable,omitempty"`
 }
 
 // FileLinks includes links to file at backend
@@ -235,18 +235,18 @@ type Media struct {
 		VideoType  string `json:"video_type,omitempty"`  // "mpegts"
 		HdrType    string `json:"hdr_type,omitempty"`
 	} `json:"video,omitempty"`
-	Link           *Link         `json:"link,omitempty"`
-	NeedMoreQuota  bool          `json:"need_more_quota,omitempty"`
-	VipTypes       []interface{} `json:"vip_types,omitempty"` // TODO maybe list of something?
-	RedirectLink   string        `json:"redirect_link,omitempty"`
-	IconLink       string        `json:"icon_link,omitempty"`
-	IsDefault      bool          `json:"is_default,omitempty"`
-	Priority       int           `json:"priority,omitempty"`
-	IsOrigin       bool          `json:"is_origin,omitempty"`
-	ResolutionName string        `json:"resolution_name,omitempty"`
-	IsVisible      bool          `json:"is_visible,omitempty"`
-	Category       string        `json:"category,omitempty"` // "category_origin"
-	Audio          interface{}   `json:"audio"`              // TODO: undiscovered yet
+	Link           *Link  `json:"link,omitempty"`
+	NeedMoreQuota  bool   `json:"need_more_quota,omitempty"`
+	VipTypes       []any  `json:"vip_types,omitempty"` // TODO maybe list of something?
+	RedirectLink   string `json:"redirect_link,omitempty"`
+	IconLink       string `json:"icon_link,omitempty"`
+	IsDefault      bool   `json:"is_default,omitempty"`
+	Priority       int    `json:"priority,omitempty"`
+	IsOrigin       bool   `json:"is_origin,omitempty"`
+	ResolutionName string `json:"resolution_name,omitempty"`
+	IsVisible      bool   `json:"is_visible,omitempty"`
+	Category       string `json:"category,omitempty"` // "category_origin"
+	Audio          any    `json:"audio"`              // TODO: undiscovered yet
 }
 
 // FileParams includes parameters for instant open
@@ -263,20 +263,20 @@ type FileParams struct {
 
 // FileApp includes parameters for instant open
 type FileApp struct {
-	ID            string        `json:"id,omitempty"`   // "decompress" for rar files
-	Name          string        `json:"name,omitempty"` // decompress" for rar files
-	Access        []interface{} `json:"access,omitempty"`
-	Link          string        `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
-	RedirectLink  string        `json:"redirect_link,omitempty"`
-	VipTypes      []interface{} `json:"vip_types,omitempty"`
-	NeedMoreQuota bool          `json:"need_more_quota,omitempty"`
-	IconLink      string        `json:"icon_link,omitempty"`
-	IsDefault     bool          `json:"is_default,omitempty"`
-	Params        struct{}      `json:"params,omitempty"` // TODO
-	CategoryIDs   []interface{} `json:"category_ids,omitempty"`
-	AdSceneType   int           `json:"ad_scene_type,omitempty"`
-	Space         string        `json:"space,omitempty"`
-	Links         struct{}      `json:"links,omitempty"` // TODO
+	ID            string   `json:"id,omitempty"`   // "decompress" for rar files
+	Name          string   `json:"name,omitempty"` // decompress" for rar files
+	Access        []any    `json:"access,omitempty"`
+	Link          string   `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
+	RedirectLink  string   `json:"redirect_link,omitempty"`
+	VipTypes      []any    `json:"vip_types,omitempty"`
+	NeedMoreQuota bool     `json:"need_more_quota,omitempty"`
+	IconLink      string   `json:"icon_link,omitempty"`
+	IsDefault     bool     `json:"is_default,omitempty"`
+	Params        struct{} `json:"params,omitempty"` // TODO
+	CategoryIDs   []any    `json:"category_ids,omitempty"`
+	AdSceneType   int      `json:"ad_scene_type,omitempty"`
+	Space         string   `json:"space,omitempty"`
+	Links         struct{} `json:"links,omitempty"` // TODO
 }
 
 // ------------------------------------------------------------
@@ -290,27 +290,27 @@ type TaskList struct {
 
 // Task is a basic element representing a single task such as offline download and upload
 type Task struct {
-	Kind              string        `json:"kind,omitempty"` // "drive#task"
-	ID                string        `json:"id,omitempty"`   // task id?
-	Name              string        `json:"name,omitempty"` // torrent name?
-	Type              string        `json:"type,omitempty"` // "offline"
-	UserID            string        `json:"user_id,omitempty"`
-	Statuses          []interface{} `json:"statuses,omitempty"`    // TODO
-	StatusSize        int           `json:"status_size,omitempty"` // TODO
-	Params            *TaskParams   `json:"params,omitempty"`      // TODO
-	FileID            string        `json:"file_id,omitempty"`
-	FileName          string        `json:"file_name,omitempty"`
-	FileSize          string        `json:"file_size,omitempty"`
-	Message           string        `json:"message,omitempty"` // e.g. "Saving"
-	CreatedTime       Time          `json:"created_time,omitempty"`
-	UpdatedTime       Time          `json:"updated_time,omitempty"`
-	ThirdTaskID       string        `json:"third_task_id,omitempty"` // TODO
-	Phase             string        `json:"phase,omitempty"`         // e.g. "PHASE_TYPE_RUNNING"
-	Progress          int           `json:"progress,omitempty"`
-	IconLink          string        `json:"icon_link,omitempty"`
-	Callback          string        `json:"callback,omitempty"`
-	ReferenceResource interface{}   `json:"reference_resource,omitempty"` // TODO
-	Space             string        `json:"space,omitempty"`
+	Kind              string      `json:"kind,omitempty"` // "drive#task"
+	ID                string      `json:"id,omitempty"`   // task id?
+	Name              string      `json:"name,omitempty"` // torrent name?
+	Type              string      `json:"type,omitempty"` // "offline"
+	UserID            string      `json:"user_id,omitempty"`
+	Statuses          []any       `json:"statuses,omitempty"`    // TODO
+	StatusSize        int         `json:"status_size,omitempty"` // TODO
+	Params            *TaskParams `json:"params,omitempty"`      // TODO
+	FileID            string      `json:"file_id,omitempty"`
+	FileName          string      `json:"file_name,omitempty"`
+	FileSize          string      `json:"file_size,omitempty"`
+	Message           string      `json:"message,omitempty"` // e.g. "Saving"
+	CreatedTime       Time        `json:"created_time,omitempty"`
+	UpdatedTime       Time        `json:"updated_time,omitempty"`
+	ThirdTaskID       string      `json:"third_task_id,omitempty"` // TODO
+	Phase             string      `json:"phase,omitempty"`         // e.g. "PHASE_TYPE_RUNNING"
+	Progress          int         `json:"progress,omitempty"`
+	IconLink          string      `json:"icon_link,omitempty"`
+	Callback          string      `json:"callback,omitempty"`
+	ReferenceResource any         `json:"reference_resource,omitempty"` // TODO
+	Space             string      `json:"space,omitempty"`
 }
 
 // TaskParams includes parameters informing status of Task

backend/pikpak/helper.go

@@ -638,7 +638,7 @@ func (c *pikpakClient) SetCaptchaTokener(ctx context.Context, m configmap.Mapper
 	return c
 }
 
-func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request any, response any) (resp *http.Response, err error) {
 	if c.captcha != nil {
 		token, err := c.captcha.Token(opts)
 		if err != nil || token == "" {

backend/pikpak/pikpak.go

@@ -1232,7 +1232,7 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
 	params := url.Values{}
 	iVal := reflect.ValueOf(&form.MultiParts).Elem()
 	iTyp := iVal.Type()
-	for i := 0; i < iVal.NumField(); i++ {
+	for i := range iVal.NumField() {
 		params.Set(iTyp.Field(i).Tag.Get("json"), iVal.Field(i).String())
 	}
 	formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, params, "file", name)
@@ -1520,7 +1520,7 @@ Result:
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "addurl":
 		if len(arg) != 1 {

backend/putio/error.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"slices"
 	"strconv"
 	"time"
 
@@ -13,10 +14,8 @@ import (
 )
 
 func checkStatusCode(resp *http.Response, expected ...int) error {
-	for _, code := range expected {
-		if resp.StatusCode == code {
-			return nil
-		}
+	if slices.Contains(expected, resp.StatusCode) {
+		return nil
 	}
 	return &statusCodeError{response: resp}
 }

backend/putio/fs.go

@@ -332,10 +332,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 	var offsetMismatch bool
 	buf := make([]byte, defaultChunkSize)
 	for clientOffset < size {
-		chunkSize := size - clientOffset
-		if chunkSize >= int64(defaultChunkSize) {
-			chunkSize = int64(defaultChunkSize)
-		}
+		chunkSize := min(size-clientOffset, int64(defaultChunkSize))
 		chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
 		chunkStart := clientOffset
 		reqSize := chunkSize

backend/qingstor/qingstor.go

@@ -22,7 +22,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	qsConfig "github.com/yunify/qingstor-sdk-go/v3/config"
@@ -704,7 +704,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // of listing recursively that doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
 	bucket, directory := f.split(dir)
-	list := walk.NewListRHelper(callback)
+	list := list.NewHelper(callback)
 	listR := func(bucket, directory, prefix string, addBucket bool) error {
 		return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *qs.KeyType, isDirectory bool) error {
 			entry, err := f.itemToDirEntry(remote, object, isDirectory)

backend/qingstor/upload.go

@@ -358,7 +358,7 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) (err error) {
 	})()
 
 	ch := make(chan chunk, mu.cfg.concurrency)
-	for i := 0; i < mu.cfg.concurrency; i++ {
+	for range mu.cfg.concurrency {
 		mu.wg.Add(1)
 		go mu.readChunk(ch)
 	}

backend/quatrix/quatrix.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -643,10 +644,8 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 		return err
 	}
 
-	for _, removedID := range result.IDs {
-		if removedID == id {
-			return nil
-		}
+	if slices.Contains(result.IDs, id) {
+		return nil
 	}
 
 	return fmt.Errorf("file %s was not deleted successfully", id)

backend/quatrix/upload_memory.go

@@ -59,11 +59,7 @@ func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed f
 
 	defer func() { u.fileUsage[fileID] = borrowed }()
 
-	effectiveChunkSize := int64(speed * u.effectiveTime.Seconds())
-
-	if effectiveChunkSize < u.reserved {
-		effectiveChunkSize = u.reserved
-	}
+	effectiveChunkSize := max(int64(speed*u.effectiveTime.Seconds()), u.reserved)
 
 	if neededMemory < effectiveChunkSize {
 		effectiveChunkSize = neededMemory

backend/s3/ibm_signer.go

@@ -0,0 +1,59 @@
+package s3
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/IBM/go-sdk-core/v5/core"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+)
+
+// Authenticator defines an interface for obtaining an IAM token.
+type Authenticator interface {
+	GetToken() (string, error)
+}
+
+// IbmIamSigner is a structure for signing requests using IBM IAM.
+// Requeres APIKey and Resource InstanceID
+type IbmIamSigner struct {
+	APIKey     string
+	InstanceID string
+	Auth       Authenticator
+}
+
+// SignHTTP signs requests using IBM IAM token.
+func (signer *IbmIamSigner) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4signer.SignerOptions)) error {
+	var authenticator Authenticator
+	if signer.Auth != nil {
+		authenticator = signer.Auth
+	} else {
+		authenticator = &core.IamAuthenticator{ApiKey: signer.APIKey}
+	}
+	token, err := authenticator.GetToken()
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("ibm-service-instance-id", signer.InstanceID)
+	return nil
+}
+
+// NoOpCredentialsProvider is needed since S3 SDK requires having credentials, even though authentication is happening via IBM IAM.
+type NoOpCredentialsProvider struct{}
+
+// Retrieve returns mock credentials for the NoOpCredentialsProvider.
+func (n *NoOpCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	return aws.Credentials{
+		AccessKeyID:     "NoOpAccessKey",
+		SecretAccessKey: "NoOpSecretKey",
+		SessionToken:    "",
+		Source:          "NoOpCredentialsProvider",
+	}, nil
+}
+
+// IsExpired always returns false
+func (n *NoOpCredentialsProvider) IsExpired() bool {
+	return false
+}

backend/s3/ibm_signer_test.go

@@ -0,0 +1,47 @@
+package s3
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/stretchr/testify/assert"
+)
+
+type MockAuthenticator struct {
+	Token string
+	Error error
+}
+
+func (m *MockAuthenticator) GetToken() (string, error) {
+	return m.Token, m.Error
+}
+
+func TestSignHTTP(t *testing.T) {
+	apiKey := "mock-api-key"
+	instanceID := "mock-instance-id"
+	token := "mock-iam-token"
+	mockAuth := &MockAuthenticator{
+		Token: token,
+		Error: nil,
+	}
+	signer := &IbmIamSigner{
+		APIKey:     apiKey,
+		InstanceID: instanceID,
+		Auth:       mockAuth,
+	}
+	req, err := http.NewRequest("GET", "https://example.com", nil)
+	if err != nil {
+		t.Fatalf("Failed to create HTTP request: %v", err)
+	}
+	credentials := aws.Credentials{
+		AccessKeyID:     "mock-access-key",
+		SecretAccessKey: "mock-secret-key",
+	}
+	err = signer.SignHTTP(context.TODO(), credentials, req, "payload-hash", "service", "region", time.Now())
+	assert.NoError(t, err, "Expected no error")
+	assert.Equal(t, "Bearer "+token, req.Header.Get("Authorization"), "Authorization header should be set correctly")
+	assert.Equal(t, instanceID, req.Header.Get("ibm-service-instance-id"), "ibm-service-instance-id header should be set correctly")
+}