encode,staticfiles: Content negotiation, precompressed files (#4045)

* encode: implement prefer setting

* encode: minimum_length configurable via caddyfile

* encode: configurable content-types which to encode

* file_server: support precompressed files

* encode: use ReponseMatcher for conditional encoding of content

* linting error & documentation of encode.PrecompressedOrder

* encode: allow just one response matcher

also change the namespace of the encoders back, I accidently changed to precompressed >.>
default matchers include a *  to match to any charset, that may be appended

* rounding of the PR

* added integration tests for new caddyfile directives
* improved various doc strings (punctuation and typos)
* added json tag for file_server precompress order and encode matcher

* file_server: add vary header, remove accept-ranges when serving precompressed files

* encode: move Suffix implementation to precompressed modules

modules/caddyhttp/fileserver/staticfiles.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
 	"go.uber.org/zap"
 )
 
@@ -79,6 +80,16 @@ type FileServer struct {
 	// a 404 error. By default, this is false (disabled).
 	PassThru bool `json:"pass_thru,omitempty"`
 
+	// Selection of encoders to use to check for precompressed files.
+	PrecompressedRaw caddy.ModuleMap `json:"precompressed,omitempty" caddy:"namespace=http.precompressed"`
+
+	// If the client has no strong preference (q-factor), choose these encodings in order.
+	// If no order specified here, the first encoding from the Accept-Encoding header
+	// that both client and server support is used
+	PrecompressedOrder []string `json:"precompressed_order,omitempty"`
+
+	precompressors map[string]encode.Precompressed
+
 	logger *zap.Logger
 }
 
@@ -129,6 +140,32 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 		}
 	}
 
+	mods, err := ctx.LoadModule(fsrv, "PrecompressedRaw")
+	if err != nil {
+		return fmt.Errorf("loading encoder modules: %v", err)
+	}
+	for modName, modIface := range mods.(map[string]interface{}) {
+		p, ok := modIface.(encode.Precompressed)
+		if !ok {
+			return fmt.Errorf("module %s is not precompressor", modName)
+		}
+		ae := p.AcceptEncoding()
+		if ae == "" {
+			return fmt.Errorf("precompressor does not specify an Accept-Encoding value")
+		}
+		suffix := p.Suffix()
+		if suffix == "" {
+			return fmt.Errorf("precompressor does not specify a Suffix value")
+		}
+		if _, ok := fsrv.precompressors[ae]; ok {
+			return fmt.Errorf("precompressor already added: %s", ae)
+		}
+		if fsrv.precompressors == nil {
+			fsrv.precompressors = make(map[string]encode.Precompressed)
+		}
+		fsrv.precompressors[ae] = p
+	}
+
 	return nil
 }
 
@@ -205,8 +242,6 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		return fsrv.notFound(w, r, next)
 	}
 
-	// TODO: content negotiation (brotli sidecar files, etc...)
-
 	// one last check to ensure the file isn't hidden (we might
 	// have changed the filename from when we last checked)
 	if fileHidden(filename, filesToHide) {
@@ -230,18 +265,51 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		}
 	}
 
-	fsrv.logger.Debug("opening file", zap.String("filename", filename))
+	var file *os.File
 
-	// open the file
-	file, err := fsrv.openFile(filename, w)
-	if err != nil {
-		if herr, ok := err.(caddyhttp.HandlerError); ok &&
-			herr.StatusCode == http.StatusNotFound {
-			return fsrv.notFound(w, r, next)
+	// check for precompressed files
+	for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) {
+		precompress, ok := fsrv.precompressors[ae]
+		if !ok {
+			continue
 		}
-		return err // error is already structured
+		compressedFilename := filename + precompress.Suffix()
+		compressedInfo, err := os.Stat(compressedFilename)
+		if err != nil || compressedInfo.IsDir() {
+			fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err))
+			continue
+		}
+		fsrv.logger.Debug("opening compressed sidecar file", zap.String("filename", compressedFilename), zap.Error(err))
+		file, err = fsrv.openFile(compressedFilename, w)
+		if err != nil {
+			fsrv.logger.Warn("opening precompressed file failed", zap.String("filename", compressedFilename), zap.Error(err))
+			if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable {
+				return err
+			}
+			continue
+		}
+		defer file.Close()
+		w.Header().Set("Content-Encoding", ae)
+		w.Header().Del("Accept-Ranges")
+		w.Header().Add("Vary", "Accept-Encoding")
+		break
+	}
+
+	// no precompressed file found, use the actual file
+	if file == nil {
+		fsrv.logger.Debug("opening file", zap.String("filename", filename))
+
+		// open the file
+		file, err = fsrv.openFile(filename, w)
+		if err != nil {
+			if herr, ok := err.(caddyhttp.HandlerError); ok &&
+				herr.StatusCode == http.StatusNotFound {
+				return fsrv.notFound(w, r, next)
+			}
+			return err // error is already structured
+		}
+		defer file.Close()
 	}
-	defer file.Close()
 
 	// set the ETag - note that a conditional If-None-Match request is handled
 	// by http.ServeContent below, which checks against this ETag value