encode,staticfiles: Content negotiation, precompressed files (#4045)

* encode: implement prefer setting

* encode: minimum_length configurable via caddyfile

* encode: configurable content-types which to encode

* file_server: support precompressed files

* encode: use ReponseMatcher for conditional encoding of content

* linting error & documentation of encode.PrecompressedOrder

* encode: allow just one response matcher

also change the namespace of the encoders back, I accidently changed to precompressed >.>
default matchers include a *  to match to any charset, that may be appended

* rounding of the PR

* added integration tests for new caddyfile directives
* improved various doc strings (punctuation and typos)
* added json tag for file_server precompress order and encode matcher

* file_server: add vary header, remove accept-ranges when serving precompressed files

* encode: move Suffix implementation to precompressed modules
This commit is contained in:
Steffen Brüheim
2021-03-30 02:47:19 +02:00
committed by GitHub
parent 75f797debd
commit f35a7fa466
12 changed files with 768 additions and 49 deletions

View File

@ -0,0 +1,31 @@
package caddybrotli
import (
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
)
func init() {
caddy.RegisterModule(BrotliPrecompressed{})
}
// BrotliPrecompressed provides the file extension for files precompressed with brotli encoding.
type BrotliPrecompressed struct{}
// CaddyModule returns the Caddy module information.
func (BrotliPrecompressed) CaddyModule() caddy.ModuleInfo {
return caddy.ModuleInfo{
ID: "http.precompressed.br",
New: func() caddy.Module { return new(BrotliPrecompressed) },
}
}
// AcceptEncoding returns the name of the encoding as
// used in the Accept-Encoding request headers.
func (BrotliPrecompressed) AcceptEncoding() string { return "br" }
// Suffix returns the filename suffix of precompressed files.
func (BrotliPrecompressed) Suffix() string { return ".br" }
// Interface guards
var _ encode.Precompressed = (*BrotliPrecompressed)(nil)

View File

@ -15,7 +15,9 @@
package encode
import (
"fmt"
"net/http"
"strconv"
"strings"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
@ -40,21 +42,31 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
//
// encode [<matcher>] <formats...> {
// gzip [<level>]
// gzip [<level>]
// zstd
// minimum_length <length>
// prefer <formats...>
// # response matcher block
// match {
// status <code...>
// header <field> [<value>]
// }
// # or response matcher single line syntax
// match [header <field> [<value>]] | [status <code...>]
// }
//
// Specifying the formats on the first line will use those formats' defaults.
func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
responseMatchers := make(map[string]caddyhttp.ResponseMatcher)
for d.Next() {
for _, arg := range d.RemainingArgs() {
mod, err := caddy.GetModule("http.encoders." + arg)
if err != nil {
return fmt.Errorf("finding encoder module '%s': %v", mod, err)
return d.Errf("finding encoder module '%s': %v", mod, err)
}
encoding, ok := mod.New().(Encoding)
if !ok {
return fmt.Errorf("module %s is not an HTTP encoding", mod)
return d.Errf("module %s is not an HTTP encoding", mod)
}
if enc.EncodingsRaw == nil {
enc.EncodingsRaw = make(caddy.ModuleMap)
@ -63,25 +75,118 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
}
for d.NextBlock(0) {
name := d.Val()
modID := "http.encoders." + name
unm, err := caddyfile.UnmarshalModule(d, modID)
if err != nil {
return err
switch d.Val() {
case "minimum_length":
if !d.NextArg() {
return d.ArgErr()
}
minLength, err := strconv.Atoi(d.Val())
if err != nil {
return err
}
enc.MinLength = minLength
case "prefer":
var encs []string
for d.NextArg() {
encs = append(encs, d.Val())
}
if len(encs) == 0 {
return d.ArgErr()
}
enc.Prefer = encs
case "match":
err := enc.parseNamedResponseMatcher(d.NewFromNextSegment(), responseMatchers)
if err != nil {
return err
}
matcher := responseMatchers["match"]
enc.Matcher = &matcher
default:
name := d.Val()
modID := "http.encoders." + name
unm, err := caddyfile.UnmarshalModule(d, modID)
if err != nil {
return err
}
encoding, ok := unm.(Encoding)
if !ok {
return d.Errf("module %s is not an HTTP encoding; is %T", modID, unm)
}
if enc.EncodingsRaw == nil {
enc.EncodingsRaw = make(caddy.ModuleMap)
}
enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)
}
encoding, ok := unm.(Encoding)
if !ok {
return fmt.Errorf("module %s is not an HTTP encoding; is %T", modID, unm)
}
if enc.EncodingsRaw == nil {
enc.EncodingsRaw = make(caddy.ModuleMap)
}
enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)
}
}
return nil
}
// Parse the tokens of a named response matcher.
//
// match {
// header <field> [<value>]
// status <code...>
// }
//
// Or, single line syntax:
//
// match [header <field> [<value>]] | [status <code...>]
//
func (enc *Encode) parseNamedResponseMatcher(d *caddyfile.Dispenser, matchers map[string]caddyhttp.ResponseMatcher) error {
for d.Next() {
definitionName := d.Val()
if _, ok := matchers[definitionName]; ok {
return d.Errf("matcher is defined more than once: %s", definitionName)
}
matcher := caddyhttp.ResponseMatcher{}
for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
switch d.Val() {
case "header":
if matcher.Headers == nil {
matcher.Headers = http.Header{}
}
// reuse the header request matcher's unmarshaler
headerMatcher := caddyhttp.MatchHeader(matcher.Headers)
err := headerMatcher.UnmarshalCaddyfile(d.NewFromNextSegment())
if err != nil {
return err
}
matcher.Headers = http.Header(headerMatcher)
case "status":
if matcher.StatusCode == nil {
matcher.StatusCode = []int{}
}
args := d.RemainingArgs()
if len(args) == 0 {
return d.ArgErr()
}
for _, arg := range args {
if len(arg) == 3 && strings.HasSuffix(arg, "xx") {
arg = arg[:1]
}
statusNum, err := strconv.Atoi(arg)
if err != nil {
return d.Errf("bad status value '%s': %v", arg, err)
}
matcher.StatusCode = append(matcher.StatusCode, statusNum)
}
default:
return d.Errf("unrecognized response matcher %s", d.Val())
}
}
matchers[definitionName] = matcher
}
return nil
}
// Interface guard
var _ caddyfile.Unmarshaler = (*Encode)(nil)

View File

@ -23,6 +23,7 @@ import (
"bytes"
"fmt"
"io"
"math"
"net/http"
"sort"
"strconv"
@ -43,12 +44,16 @@ type Encode struct {
// will be chosen based on the client's Accept-Encoding header.
EncodingsRaw caddy.ModuleMap `json:"encodings,omitempty" caddy:"namespace=http.encoders"`
// If the client has no strong preference, choose this encoding. TODO: Not yet implemented
// Prefer []string `json:"prefer,omitempty"`
// If the client has no strong preference, choose these encodings in order.
Prefer []string `json:"prefer,omitempty"`
// Only encode responses that are at least this many bytes long.
MinLength int `json:"minimum_length,omitempty"`
// Only encode responses that match against this ResponseMmatcher.
// The default is a collection of text-based Content-Type headers.
Matcher *caddyhttp.ResponseMatcher `json:"match,omitempty"`
writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads...
}
@ -75,11 +80,46 @@ func (enc *Encode) Provision(ctx caddy.Context) error {
if enc.MinLength == 0 {
enc.MinLength = defaultMinLength
}
if enc.Matcher == nil {
// common text-based content types
enc.Matcher = &caddyhttp.ResponseMatcher{
Headers: http.Header{
"Content-Type": []string{
"text/*",
"application/json*",
"application/javascript*",
"application/xhtml+xml*",
"application/atom+xml*",
"application/rss+xml*",
"image/svg+xml*",
},
},
}
}
return nil
}
// Validate ensures that enc's configuration is valid.
func (enc *Encode) Validate() error {
check := make(map[string]bool)
for _, encName := range enc.Prefer {
if _, ok := enc.writerPools[encName]; !ok {
return fmt.Errorf("encoding %s not enabled", encName)
}
if _, ok := check[encName]; ok {
return fmt.Errorf("encoding %s is duplicated in prefer", encName)
}
check[encName] = true
}
return nil
}
func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
for _, encName := range acceptedEncodings(r) {
for _, encName := range AcceptedEncodings(r, enc.Prefer) {
if _, ok := enc.writerPools[encName]; !ok {
continue // encoding not offered
}
@ -150,6 +190,11 @@ func (rw *responseWriter) WriteHeader(status int) {
rw.statusCode = status
}
// Match determines, if encoding should be done based on the ResponseMatcher.
func (enc *Encode) Match(rw *responseWriter) bool {
return enc.Matcher.Match(rw.statusCode, rw.Header())
}
// Write writes to the response. If the response qualifies,
// it is encoded using the encoder, which is initialized
// if not done so already.
@ -240,7 +285,10 @@ func (rw *responseWriter) Close() error {
// init should be called before we write a response, if rw.buf has contents.
func (rw *responseWriter) init() {
if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength {
if rw.Header().Get("Content-Encoding") == "" &&
rw.buf.Len() >= rw.config.MinLength &&
rw.config.Match(rw) {
rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)
rw.w.Reset(rw.ResponseWriter)
rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
@ -250,12 +298,14 @@ func (rw *responseWriter) init() {
rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
}
// acceptedEncodings returns the list of encodings that the
// client supports, in descending order of preference. If
// AcceptedEncodings returns the list of encodings that the
// client supports, in descending order of preference.
// The client preference via q-factor and the server
// preference via Prefer setting are taken into account. If
// the Sec-WebSocket-Key header is present then non-identity
// encodings are not considered. See
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.
func acceptedEncodings(r *http.Request) []string {
func AcceptedEncodings(r *http.Request, preferredOrder []string) []string {
acceptEncHeader := r.Header.Get("Accept-Encoding")
websocketKey := r.Header.Get("Sec-WebSocket-Key")
if acceptEncHeader == "" {
@ -292,18 +342,29 @@ func acceptedEncodings(r *http.Request) []string {
continue
}
// set server preference
prefOrder := -1
for i, p := range preferredOrder {
if encName == p {
prefOrder = len(preferredOrder) - i
break
}
}
prefs = append(prefs, encodingPreference{
encoding: encName,
q: qFactor,
encoding: encName,
q: qFactor,
preferOrder: prefOrder,
})
}
// sort preferences by descending q-factor
sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q })
// TODO: If no preference, or same pref for all encodings,
// and not websocket, use default encoding ordering (enc.Prefer)
// for those which are accepted by the client
// sort preferences by descending q-factor first, then by preferOrder
sort.Slice(prefs, func(i, j int) bool {
if math.Abs(prefs[i].q-prefs[j].q) < 0.00001 {
return prefs[i].preferOrder > prefs[j].preferOrder
}
return prefs[i].q > prefs[j].q
})
prefEncNames := make([]string, len(prefs))
for i := range prefs {
@ -315,8 +376,9 @@ func acceptedEncodings(r *http.Request) []string {
// encodingPreference pairs an encoding with its q-factor.
type encodingPreference struct {
encoding string
q float64
encoding string
q float64
preferOrder int
}
// Encoder is a type which can encode a stream of data.
@ -332,6 +394,13 @@ type Encoding interface {
NewEncoder() Encoder
}
// Precompressed is a type which returns filename suffix of precompressed
// file and Accept-Encoding header to use when serving this file.
type Precompressed interface {
AcceptEncoding() string
Suffix() string
}
var bufPool = sync.Pool{
New: func() interface{} {
return new(bytes.Buffer)
@ -344,6 +413,7 @@ const defaultMinLength = 512
// Interface guards
var (
_ caddy.Provisioner = (*Encode)(nil)
_ caddy.Validator = (*Encode)(nil)
_ caddyhttp.MiddlewareHandler = (*Encode)(nil)
_ caddyhttp.HTTPInterfaces = (*responseWriter)(nil)
)

View File

@ -1,6 +1,8 @@
package encode
import (
"net/http"
"sync"
"testing"
)
@ -10,3 +12,252 @@ func BenchmarkOpenResponseWriter(b *testing.B) {
enc.openResponseWriter("test", nil)
}
}
func TestPreferOrder(t *testing.T) {
testCases := []struct {
name string
accept string
prefer []string
expected []string
}{
{
name: "PreferOrder(): 4 accept, 3 prefer",
accept: "deflate, gzip, br, zstd",
prefer: []string{"zstd", "br", "gzip"},
expected: []string{"zstd", "br", "gzip", "deflate"},
},
{
name: "PreferOrder(): 2 accept, 3 prefer",
accept: "deflate, zstd",
prefer: []string{"zstd", "br", "gzip"},
expected: []string{"zstd", "deflate"},
},
{
name: "PreferOrder(): 2 accept (1 empty), 3 prefer",
accept: "gzip,,zstd",
prefer: []string{"zstd", "br", "gzip"},
expected: []string{"zstd", "gzip", ""},
},
{
name: "PreferOrder(): 1 accept, 2 prefer",
accept: "gzip",
prefer: []string{"zstd", "gzip"},
expected: []string{"gzip"},
},
{
name: "PreferOrder(): 4 accept (1 duplicate), 1 prefer",
accept: "deflate, gzip, br, br",
prefer: []string{"br"},
expected: []string{"br", "br", "deflate", "gzip"},
},
{
name: "PreferOrder(): empty accept, 0 prefer",
accept: "",
prefer: []string{},
expected: []string{},
},
{
name: "PreferOrder(): empty accept, 1 prefer",
accept: "",
prefer: []string{"gzip"},
expected: []string{},
},
{
name: "PreferOrder(): with q-factor",
accept: "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd",
prefer: []string{"gzip"},
expected: []string{"zstd", "deflate", "gzip", "br"},
},
{
name: "PreferOrder(): with q-factor, no prefer",
accept: "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd",
prefer: []string{},
expected: []string{"zstd", "deflate", "gzip", "br"},
},
{
name: "PreferOrder(): q-factor=0 filtered out",
accept: "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0",
prefer: []string{"gzip"},
expected: []string{"br", "gzip", "deflate"},
},
{
name: "PreferOrder(): q-factor=0 filtered out, no prefer",
accept: "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0",
prefer: []string{},
expected: []string{"br", "gzip", "deflate"},
},
{
name: "PreferOrder(): with invalid q-factor",
accept: "br, deflate, gzip;q=2, zstd;q=0.1",
prefer: []string{"zstd", "gzip"},
expected: []string{"gzip", "br", "deflate", "zstd"},
},
{
name: "PreferOrder(): with invalid q-factor, no prefer",
accept: "br, deflate, gzip;q=2, zstd;q=0.1",
prefer: []string{},
expected: []string{"br", "deflate", "gzip", "zstd"},
},
}
enc := new(Encode)
r, _ := http.NewRequest("", "", nil)
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
if test.accept == "" {
r.Header.Del("Accept-Encoding")
} else {
r.Header.Set("Accept-Encoding", test.accept)
}
enc.Prefer = test.prefer
result := AcceptedEncodings(r, enc.Prefer)
if !sliceEqual(result, test.expected) {
t.Errorf("AcceptedEncodings() actual: %s expected: %s",
result,
test.expected)
}
})
}
}
func sliceEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func TestValidate(t *testing.T) {
type testCase struct {
name string
prefer []string
wantErr bool
}
var err error
var testCases []testCase
enc := new(Encode)
enc.writerPools = map[string]*sync.Pool{
"zstd": nil,
"gzip": nil,
"br": nil,
}
testCases = []testCase{
{
name: "ValidatePrefer (zstd, gzip & br enabled): valid order with all encoder",
prefer: []string{"zstd", "br", "gzip"},
wantErr: false,
},
{
name: "ValidatePrefer (zstd, gzip & br enabled): valid order with 2 out of 3 encoders",
prefer: []string{"br", "gzip"},
wantErr: false,
},
{
name: "ValidatePrefer (zstd, gzip & br enabled): valid order with 1 out of 3 encoders",
prefer: []string{"gzip"},
wantErr: false,
},
{
name: "ValidatePrefer (zstd, gzip & br enabled): 1 duplicated (once) encoder",
prefer: []string{"gzip", "zstd", "gzip"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd, gzip & br enabled): 1 not enabled encoder in prefer list",
prefer: []string{"br", "zstd", "gzip", "deflate"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd, gzip & br enabled): no prefer list",
prefer: []string{},
wantErr: false,
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
enc.Prefer = test.prefer
err = enc.Validate()
if (err != nil) != test.wantErr {
t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr)
}
})
}
enc.writerPools = map[string]*sync.Pool{
"zstd": nil,
"gzip": nil,
}
testCases = []testCase{
{
name: "ValidatePrefer (zstd & gzip enabled): 1 not enabled encoder in prefer list",
prefer: []string{"zstd", "br", "gzip"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): 2 not enabled encoder in prefer list",
prefer: []string{"br", "zstd", "gzip", "deflate"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): only not enabled encoder in prefer list",
prefer: []string{"deflate", "br", "gzip"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated (once) encoder in prefer list",
prefer: []string{"gzip", "zstd", "gzip"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated (twice) encoder in prefer list",
prefer: []string{"gzip", "zstd", "gzip", "gzip"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated encoder in prefer list",
prefer: []string{"zstd", "zstd", "gzip", "gzip"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated not enabled encoder in prefer list",
prefer: []string{"br", "br", "gzip"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): 2 duplicated not enabled encoder in prefer list",
prefer: []string{"br", "deflate", "br", "deflate"},
wantErr: true,
},
{
name: "ValidatePrefer (zstd & gzip enabled): valid order zstd first",
prefer: []string{"zstd", "gzip"},
wantErr: false,
},
{
name: "ValidatePrefer (zstd & gzip enabled): valid order gzip first",
prefer: []string{"gzip", "zstd"},
wantErr: false,
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
enc.Prefer = test.prefer
err = enc.Validate()
if (err != nil) != test.wantErr {
t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr)
}
})
}
}

View File

@ -0,0 +1,28 @@
package caddygzip
import (
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
)
func init() {
caddy.RegisterModule(GzipPrecompressed{})
}
// GzipPrecompressed provides the file extension for files precompressed with gzip encoding.
type GzipPrecompressed struct {
Gzip
}
// CaddyModule returns the Caddy module information.
func (GzipPrecompressed) CaddyModule() caddy.ModuleInfo {
return caddy.ModuleInfo{
ID: "http.precompressed.gzip",
New: func() caddy.Module { return new(GzipPrecompressed) },
}
}
// Suffix returns the filename suffix of precompressed files.
func (GzipPrecompressed) Suffix() string { return ".gz" }
var _ encode.Precompressed = (*GzipPrecompressed)(nil)

View File

@ -0,0 +1,28 @@
package caddyzstd
import (
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
)
func init() {
caddy.RegisterModule(ZstdPrecompressed{})
}
// ZstdPrecompressed provides the file extension for files precompressed with zstandard encoding.
type ZstdPrecompressed struct {
Zstd
}
// CaddyModule returns the Caddy module information.
func (ZstdPrecompressed) CaddyModule() caddy.ModuleInfo {
return caddy.ModuleInfo{
ID: "http.precompressed.zstd",
New: func() caddy.Module { return new(ZstdPrecompressed) },
}
}
// Suffix returns the filename suffix of precompressed files.
func (ZstdPrecompressed) Suffix() string { return ".zst" }
var _ encode.Precompressed = (*ZstdPrecompressed)(nil)