Merge bf6781350ddabc1fc239103dc655c330a00f7051 into 0b9671313b14ffe839ecbd7dd2ae5ac7f6f05db8

backend/http/http.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"mime"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"path"
 	"strings"
@@ -121,11 +122,18 @@ type Fs struct {
 
 // Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
 type Object struct {
-	fs          *Fs
-	remote      string
-	size        int64
-	modTime     time.Time
-	contentType string
+	fs       *Fs
+	remote   string
+	size     int64     // Size of the object
+	modTime  time.Time // Last modified
+	mimeType string    // MimeType of object - may be ""
+
+	// Metadata as pointers to strings as they often won't be present
+	contentDisposition         *string // Content-Disposition: header
+	contentDispositionFilename *string // Filename retrieved from Content-Disposition: header
+	cacheControl               *string // Cache-Control: header
+	contentEncoding            *string // Content-Encoding: header
+	contentLanguage            *string // Content-Language: header
 }
 
 // statusError returns an error if the res contained an error
@@ -438,6 +446,29 @@ func addHeaders(req *http.Request, opt *Options) {
 	}
 }
 
+// parseFilename extracts the filename from a Content-Disposition header
+func parseFilename(contentDisposition string) (string, error) {
+	// Normalize the contentDisposition to canonical MIME format
+	mediaType, params, err := mime.ParseMediaType(contentDisposition)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse contentDisposition: %v", err)
+	}
+
+	// Check if the contentDisposition is an attachment
+	if strings.ToLower(mediaType) != "attachment" {
+		return "", fmt.Errorf("not an attachment: %s", mediaType)
+	}
+
+	// Extract the filename from the parameters
+	filename, ok := params["filename"]
+	if !ok {
+		return "", fmt.Errorf("filename not found in contentDisposition")
+	}
+
+	// Decode filename if it contains special encoding
+	return textproto.TrimString(filename), nil
+}
+
 // Adds the configured headers to the request if any
 func (f *Fs) addHeaders(req *http.Request) {
 	addHeaders(req, &f.opt)
@@ -577,6 +608,9 @@ func (o *Object) String() string {
 
 // Remote the name of the remote HTTP file, relative to the fs root
 func (o *Object) Remote() string {
+	if o.contentDispositionFilename != nil {
+		return *o.contentDispositionFilename
+	}
 	return o.remote
 }
 
@@ -605,7 +639,7 @@ func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
 		o.size = -1
 		o.modTime = timeUnset
-		o.contentType = fs.MimeType(ctx, o)
+		o.mimeType = fs.MimeType(ctx, o)
 		return nil
 	}
 	url := o.url()
@@ -627,19 +661,59 @@ func (o *Object) head(ctx context.Context) error {
 
 // decodeMetadata updates info fields in the Object according to HTTP response headers
 func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
+
+	// Parse from Content-Length and Content-Range headers
+	o.size = rest.ParseSizeFromHeaders(res.Header)
+
+	// Parse Last-Modified header
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
 	o.modTime = t
-	o.contentType = res.Header.Get("Content-Type")
-	o.size = rest.ParseSizeFromHeaders(res.Header)
+
+	// Parse Content-Type header
+	o.mimeType = res.Header.Get("Content-Type")
+
+	// Parse Content-Disposition header
+	contentDisposition := res.Header.Get("Content-Disposition")
+	if contentDisposition != "" {
+		o.contentDisposition = &contentDisposition
+	}
+
+	// Get contentDispositionFilename
+	if o.contentDisposition != nil {
+		var filename string
+		filename, err = parseFilename(*o.contentDisposition)
+		if err == nil && filename != "" {
+			o.contentDispositionFilename = &filename
+		}
+	}
+
+	// Parse Cache-Control header
+	cacheControl := res.Header.Get("Cache-Control")
+	if cacheControl != "" {
+		o.cacheControl = &cacheControl
+	}
+
+	// Parse Content-Encoding header
+	contentEncoding := res.Header.Get("Content-Encoding")
+	if contentEncoding != "" {
+		o.contentEncoding = &contentEncoding
+	}
+
+	// Parse Content-Language header
+	contentLanguage := res.Header.Get("Content-Language")
+	if contentLanguage != "" {
+		o.contentLanguage = &contentLanguage
+	}
 
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
-		mediaType, _, err := mime.ParseMediaType(o.contentType)
+		var mediaType string
+		mediaType, _, err = mime.ParseMediaType(o.mimeType)
 		if err != nil {
-			return fmt.Errorf("failed to parse Content-Type: %q: %w", o.contentType, err)
+			return fmt.Errorf("failed to parse Content-Type: %q: %w", o.mimeType, err)
 		}
 		if mediaType == "text/html" {
 			return fs.ErrorNotAFile
@@ -713,7 +787,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // MimeType of an Object if known, "" otherwise
 func (o *Object) MimeType(ctx context.Context) string {
-	return o.contentType
+	return o.mimeType
 }
 
 var commandHelp = []fs.CommandHelp{{
@@ -751,7 +825,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	switch name {
 	case "set":
 		newOpt := f.opt
-		err := configstruct.Set(configmap.Simple(opt), &newOpt)
+		err = configstruct.Set(configmap.Simple(opt), &newOpt)
 		if err != nil {
 			return nil, fmt.Errorf("reading config: %w", err)
 		}
@@ -771,6 +845,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	}
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+	metadata = make(fs.Metadata, 6)
+	if o.mimeType != "" {
+		metadata["content-type"] = o.mimeType
+	}
+
+	// Set system metadata
+	setMetadata := func(k string, v *string) {
+		if v == nil || *v == "" {
+			return
+		}
+		metadata[k] = *v
+	}
+	setMetadata("content-disposition", o.contentDisposition)
+	setMetadata("content-disposition-filename", o.contentDispositionFilename)
+	setMetadata("cache-control", o.cacheControl)
+	setMetadata("content-language", o.contentLanguage)
+	setMetadata("content-encoding", o.contentEncoding)
+	return metadata, nil
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs          = &Fs{}
@@ -778,4 +876,5 @@ var (
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
 	_ fs.Commander   = &Fs{}
+	_ fs.Metadataer  = &Object{}
 )

backend/http/http_internal_test.go

@@ -60,6 +60,17 @@ func prepareServer(t *testing.T) configmap.Simple {
 		what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path)
 		assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0])
 		assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2])
+
+		// Set the content disposition header for the fifth file
+		// later we will check if it is set using the metadata method
+		if r.URL.Path == "/five.txt.gz" {
+			w.Header().Set("Content-Disposition", "attachment; filename=\"five.txt.gz\"")
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.Header().Set("Cache-Control", "no-cache")
+			w.Header().Set("Content-Language", "en-US")
+			w.Header().Set("Content-Encoding", "gzip")
+		}
+
 		fileServer.ServeHTTP(w, r)
 	})
 
@@ -102,27 +113,33 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 
 	sort.Sort(entries)
 
-	require.Equal(t, 4, len(entries))
+	require.Equal(t, 5, len(entries))
 
 	e := entries[0]
-	assert.Equal(t, "four", e.Remote())
+	assert.Equal(t, "five.txt.gz", e.Remote())
 	assert.Equal(t, int64(-1), e.Size())
-	_, ok := e.(fs.Directory)
+	_, ok := e.(fs.Object)
 	assert.True(t, ok)
 
 	e = entries[1]
+	assert.Equal(t, "four", e.Remote())
+	assert.Equal(t, int64(-1), e.Size())
+	_, ok = e.(fs.Directory)
+	assert.True(t, ok)
+
+	e = entries[2]
 	assert.Equal(t, "one%.txt", e.Remote())
 	assert.Equal(t, int64(5+lineEndSize), e.Size())
 	_, ok = e.(*Object)
 	assert.True(t, ok)
 
-	e = entries[2]
+	e = entries[3]
 	assert.Equal(t, "three", e.Remote())
 	assert.Equal(t, int64(-1), e.Size())
 	_, ok = e.(fs.Directory)
 	assert.True(t, ok)
 
-	e = entries[3]
+	e = entries[4]
 	assert.Equal(t, "two.html", e.Remote())
 	if noSlash {
 		assert.Equal(t, int64(-1), e.Size())
@@ -218,6 +235,23 @@ func TestNewObjectWithLeadingSlash(t *testing.T) {
 	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
+func TestNewObjectWithMetadata(t *testing.T) {
+	f := prepare(t)
+	o, err := f.NewObject(context.Background(), "/five.txt.gz")
+	require.NoError(t, err)
+	assert.Equal(t, "five.txt.gz", o.Remote())
+	ho, ok := o.(*Object)
+	assert.True(t, ok)
+	metadata, err := ho.Metadata(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, "text/plain; charset=utf-8", metadata["content-type"])
+	assert.Equal(t, "attachment; filename=\"five.txt.gz\"", metadata["content-disposition"])
+	assert.Equal(t, "five.txt.gz", metadata["content-disposition-filename"])
+	assert.Equal(t, "no-cache", metadata["cache-control"])
+	assert.Equal(t, "en-US", metadata["content-language"])
+	assert.Equal(t, "gzip", metadata["content-encoding"])
+}
+
 func TestOpen(t *testing.T) {
 	m := prepareServer(t)
 

fs/object/object.go

@@ -191,11 +191,12 @@ var _ fs.Fs = MemoryFs
 
 // MemoryObject is an in memory object
 type MemoryObject struct {
-	remote  string
-	modTime time.Time
-	content []byte
-	meta    fs.Metadata
-	fs      fs.Fs
+	remote   string
+	modTime  time.Time
+	content  []byte
+	meta     fs.Metadata
+	fs       fs.Fs
+	mimeType string
 }
 
 // NewMemoryObject returns an in memory Object with the modTime and content passed in
@@ -214,6 +215,12 @@ func (o *MemoryObject) WithMetadata(meta fs.Metadata) *MemoryObject {
 	return o
 }
 
+// WithMimeType adds mimeType to the MemoryObject
+func (o *MemoryObject) WithMimeType(mimeType string) *MemoryObject {
+	o.mimeType = mimeType
+	return o
+}
+
 // Content returns the underlying buffer
 func (o *MemoryObject) Content() []byte {
 	return o.content
@@ -329,8 +336,14 @@ func (o *MemoryObject) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return o.meta, nil
 }
 
+// MimeType on the object
+func (o *MemoryObject) MimeType(ctx context.Context) string {
+	return o.mimeType
+}
+
 // Check interfaces
 var (
 	_ fs.Object     = (*MemoryObject)(nil)
+	_ fs.MimeTyper  = (*MemoryObject)(nil)
 	_ fs.Metadataer = (*MemoryObject)(nil)
 )

lib/http/serve/serve.go

@@ -39,6 +39,26 @@ func Object(w http.ResponseWriter, r *http.Request, o fs.Object) {
 	modTime := o.ModTime(r.Context())
 	w.Header().Set("Last-Modified", modTime.UTC().Format(http.TimeFormat))
 
+	// Set metadata headers if present
+	metadata, err := fs.GetMetadata(r.Context(), o)
+	if err != nil {
+		fs.Debugf(o, "Request get metadata error: %v", err)
+	}
+	if metadata != nil {
+		if metadata["content-disposition"] != "" {
+			w.Header().Set("Content-Disposition", metadata["content-disposition"])
+		}
+		if metadata["cache-control"] != "" {
+			w.Header().Set("Cache-Control", metadata["cache-control"])
+		}
+		if metadata["content-language"] != "" {
+			w.Header().Set("Content-Language", metadata["content-language"])
+		}
+		if metadata["content-encoding"] != "" {
+			w.Header().Set("Content-Encoding", metadata["content-encoding"])
+		}
+	}
+
 	if r.Method == "HEAD" {
 		return
 	}

lib/http/serve/serve_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest/mockobject"
 	"github.com/stretchr/testify/assert"
 )
@@ -82,3 +84,23 @@ func TestObjectBadRange(t *testing.T) {
 	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "Bad Request\n", string(body))
 }
+
+func TestObjectHEADMetadata(t *testing.T) {
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("HEAD", "http://example.com/aFile", nil)
+	m := fs.Metadata{
+		"content-disposition": "inline",
+		"cache-control":       "no-cache",
+		"content-language":    "en",
+		"content-encoding":    "gzip",
+	}
+	o := object.NewMemoryObject("aFile", time.Now(), []byte("")).
+		WithMetadata(m).WithMimeType("text/plain; charset=utf-8")
+	Object(w, r, o)
+	resp := w.Result()
+	assert.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "inline", resp.Header.Get("Content-Disposition"))
+	assert.Equal(t, "no-cache", resp.Header.Get("Cache-Control"))
+	assert.Equal(t, "en", resp.Header.Get("Content-Language"))
+	assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+}