diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go
index 18dfcda6c..a4403fcce 100644
--- a/cmd/serve/ftp/ftp.go
+++ b/cmd/serve/ftp/ftp.go
@@ -175,7 +175,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (*driver, error) {
 		opt: *opt,
 	}
 	if proxyflags.Opt.AuthProxy != "" {
-		d.proxy = proxy.New(ctx, &proxyflags.Opt)
+		d.proxy = proxy.New(ctx, &proxyflags.Opt, &vfscommon.Opt)
 		d.userPass = make(map[string]string, 16)
 	} else {
 		d.globalVFS = vfs.New(f, &vfscommon.Opt)
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index f946d415d..1b829e3c9 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -146,7 +146,7 @@ func run(ctx context.Context, f fs.Fs, opt Options) (s *HTTP, err error) {
 	}
 
 	if proxyflags.Opt.AuthProxy != "" {
-		s.proxy = proxy.New(ctx, &proxyflags.Opt)
+		s.proxy = proxy.New(ctx, &proxyflags.Opt, &vfscommon.Opt)
 		// override auth
 		s.opt.Auth.CustomAuthFn = s.auth
 	} else {
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go
index c8dcd6f17..ca1d040a5 100644
--- a/cmd/serve/proxy/proxy.go
+++ b/cmd/serve/proxy/proxy.go
@@ -122,6 +122,7 @@ type Proxy struct {
 	vfsCache *libcache.Cache
 	ctx      context.Context // for global config
 	Opt      Options
+	vfsOpt   vfscommon.Options
 }
 
 // cacheEntry is what is stored in the vfsCache
@@ -131,12 +132,15 @@ type cacheEntry struct {
 }
 
 // New creates a new proxy with the Options passed in
-func New(ctx context.Context, opt *Options) *Proxy {
+//
+// Any VFS are created with the vfsOpt passed in.
+func New(ctx context.Context, opt *Options, vfsOpt *vfscommon.Options) *Proxy {
 	return &Proxy{
 		ctx:      ctx,
 		Opt:      *opt,
 		cmdLine:  strings.Fields(opt.AuthProxy),
 		vfsCache: libcache.New(),
+		vfsOpt:   *vfsOpt,
 	}
 }
 
@@ -242,7 +246,7 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)
 		// need to in memory. An attacker would find it easier to go
 		// after the unencrypted password in memory most likely.
 		entry := cacheEntry{
-			vfs:    vfs.New(f, &vfscommon.Opt),
+			vfs:    vfs.New(f, &p.vfsOpt),
 			pwHash: sha256.Sum256([]byte(auth)),
 		}
 		return entry, true, nil
diff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.go
index 539468d44..1495c2dff 100644
--- a/cmd/serve/proxy/proxy_test.go
+++ b/cmd/serve/proxy/proxy_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/ssh"
@@ -22,7 +23,7 @@ func TestRun(t *testing.T) {
 	opt := DefaultOpt
 	cmd := "go run proxy_code.go"
 	opt.AuthProxy = cmd
-	p := New(context.Background(), &opt)
+	p := New(context.Background(), &opt, &vfscommon.Opt)
 
 	t.Run("Normal", func(t *testing.T) {
 		config, err := p.run(map[string]string{
diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go
index 88252d2cb..fad51c91f 100644
--- a/cmd/serve/s3/server.go
+++ b/cmd/serve/s3/server.go
@@ -81,7 +81,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
 	w.handler = w.faker.Server()
 
 	if proxyflags.Opt.AuthProxy != "" {
-		w.proxy = proxy.New(ctx, &proxyflags.Opt)
+		w.proxy = proxy.New(ctx, &proxyflags.Opt, &vfscommon.Opt)
 		// proxy auth middleware
 		w.handler = proxyAuthMiddleware(w.handler, w)
 		w.handler = authPairMiddleware(w.handler, w)
diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.go
index e41d8cee0..e26f0fb0c 100644
--- a/cmd/serve/sftp/server.go
+++ b/cmd/serve/sftp/server.go
@@ -53,7 +53,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) *server {
 		waitChan: make(chan struct{}),
 	}
 	if proxyflags.Opt.AuthProxy != "" {
-		s.proxy = proxy.New(ctx, &proxyflags.Opt)
+		s.proxy = proxy.New(ctx, &proxyflags.Opt, &vfscommon.Opt)
 	} else {
 		s.vfs = vfs.New(f, &vfscommon.Opt)
 	}
diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go
index d62da7abf..2dc3e03a8 100644
--- a/cmd/serve/webdav/webdav.go
+++ b/cmd/serve/webdav/webdav.go
@@ -205,7 +205,7 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options) (w *WebDAV, err error
 		opt: *opt,
 	}
 	if proxyflags.Opt.AuthProxy != "" {
-		w.proxy = proxy.New(ctx, &proxyflags.Opt)
+		w.proxy = proxy.New(ctx, &proxyflags.Opt, &vfscommon.Opt)
 		// override auth
 		w.opt.Auth.CustomAuthFn = w.auth
 	} else {