tracing: New OpenTelemetry module (#4361)

* opentelemetry: create a new module

* fix imports

* fix test

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update modules/caddyhttp/opentelemetry/tracer.go

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* rename error ErrUnsupportedTracesProtocol

* replace spaces with tabs in the test data

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

* replace spaces with tabs in the README.md

* use default values for a propagation and exporter protocol

* set http attributes with helper

* simplify code

* Cleanup modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update link in README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update documentation in README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update link to naming spec in README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Rename module from opentelemetry to tracing

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Rename span_name to span

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Rename span_name to span

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Simplify otel resource creation

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* handle extra attributes

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* update go.opentelemetry.io/otel/semconv to 1.7.0

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* update go.opentelemetry.io/otel version

* remove environment variable handling

* always use tracecontext,baggage as propagators

* extract tracer name into variable

* rename OpenTelemetry to Tracing

* simplify resource creation

* update go.mod

* rename package from opentelemetry to tracing

* cleanup tests

* update Caddyfile example in README.md

* update README.md

* fix test

* fix module name in README.md

* fix module name in README.md

* change names in README.md and tests

* order imports

* remove redundant tests

* Update documentation README.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Fix grammar

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update comments

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* Update comments

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* update go.sum

* update go.sum

* Add otelhttp instrumentation, update OpenTelemetry libraries.

* Use otelhttp instrumentation for instrumenting HTTP requests.

This change uses context.WithValue to inject the next handler into the
request context via a "nextCall" carrier struct, and pass it on to a
standard Go HTTP handler returned by otelhttp.NewHandler. The
underlying handler will extract the next handler from the context,
call it and pass the returned error to the carrier struct.

* use zap.Error() for the error log

* remove README.md

* update dependencies

* clean up the code

* change comment

* move serveHTTP method from separate file

* add syntax to the UnmarshalCaddyfile comment

* go import the file

* admin: Write proper status on invalid requests (#4569) (fix #4561)

* update dependencies

Co-authored-by: Dave Henderson <dhenderson@gmail.com>
Co-authored-by: Francis Lavoie <lavofr@gmail.com>
Co-authored-by: Vibhav Pant <vibhavp@gmail.com>
Co-authored-by: Alok Naushad <alokme123@gmail.com>
Co-authored-by: Cedric Ziel <cedric@cedric-ziel.com>
This commit is contained in:
Andrii Kushch
2022-03-08 20:18:32 +01:00
committed by GitHub
parent d9b1d46325
commit d0b608af31
11 changed files with 632 additions and 2 deletions

View File

@ -0,0 +1,121 @@
package tracing
import (
"fmt"
"net/http"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"go.uber.org/zap"
)
func init() {
caddy.RegisterModule(Tracing{})
httpcaddyfile.RegisterHandlerDirective("tracing", parseCaddyfile)
}
// Tracing implements an HTTP handler that adds support for distributed tracing,
// using OpenTelemetry. This module is responsible for the injection and
// propagation of the trace context. Configure this module via environment
// variables (see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md).
// Some values can be overwritten in the configuration file.
type Tracing struct {
// SpanName is a span name. It should follow the naming guidelines here:
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span
SpanName string `json:"span"`
// otel implements opentelemetry related logic.
otel openTelemetryWrapper
logger *zap.Logger
}
// CaddyModule returns the Caddy module information.
func (Tracing) CaddyModule() caddy.ModuleInfo {
return caddy.ModuleInfo{
ID: "http.handlers.tracing",
New: func() caddy.Module { return new(Tracing) },
}
}
// Provision implements caddy.Provisioner.
func (ot *Tracing) Provision(ctx caddy.Context) error {
ot.logger = ctx.Logger(ot)
var err error
ot.otel, err = newOpenTelemetryWrapper(ctx, ot.SpanName)
return err
}
// Cleanup implements caddy.CleanerUpper and closes any idle connections. It
// calls Shutdown method for a trace provider https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#shutdown.
func (ot *Tracing) Cleanup() error {
if err := ot.otel.cleanup(ot.logger); err != nil {
return fmt.Errorf("tracerProvider shutdown: %w", err)
}
return nil
}
// ServeHTTP implements caddyhttp.MiddlewareHandler.
func (ot *Tracing) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
return ot.otel.ServeHTTP(w, r, next)
}
// UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax:
//
// tracing {
// [span <span_name>]
// }
//
func (ot *Tracing) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
setParameter := func(d *caddyfile.Dispenser, val *string) error {
if d.NextArg() {
*val = d.Val()
} else {
return d.ArgErr()
}
if d.NextArg() {
return d.ArgErr()
}
return nil
}
// paramsMap is a mapping between "string" parameter from the Caddyfile and its destination within the module
paramsMap := map[string]*string{
"span": &ot.SpanName,
}
for d.Next() {
args := d.RemainingArgs()
if len(args) > 0 {
return d.ArgErr()
}
for d.NextBlock(0) {
if dst, ok := paramsMap[d.Val()]; ok {
if err := setParameter(d, dst); err != nil {
return err
}
} else {
return d.ArgErr()
}
}
}
return nil
}
func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
var m Tracing
err := m.UnmarshalCaddyfile(h.Dispenser)
return &m, err
}
// Interface guards
var (
_ caddy.Provisioner = (*Tracing)(nil)
_ caddyhttp.MiddlewareHandler = (*Tracing)(nil)
_ caddyfile.Unmarshaler = (*Tracing)(nil)
)

View File

@ -0,0 +1,182 @@
package tracing
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
)
func TestTracing_UnmarshalCaddyfile(t *testing.T) {
tests := []struct {
name string
spanName string
d *caddyfile.Dispenser
wantErr bool
}{
{
name: "Full config",
spanName: "my-span",
d: caddyfile.NewTestDispenser(`
tracing {
span my-span
}`),
wantErr: false,
},
{
name: "Only span name in the config",
spanName: "my-span",
d: caddyfile.NewTestDispenser(`
tracing {
span my-span
}`),
wantErr: false,
},
{
name: "Empty config",
d: caddyfile.NewTestDispenser(`
tracing {
}`),
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ot := &Tracing{}
if err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr {
t.Errorf("UnmarshalCaddyfile() error = %v, wantErrType %v", err, tt.wantErr)
}
if ot.SpanName != tt.spanName {
t.Errorf("UnmarshalCaddyfile() SpanName = %v, want SpanName %v", ot.SpanName, tt.spanName)
}
})
}
}
func TestTracing_UnmarshalCaddyfile_Error(t *testing.T) {
tests := []struct {
name string
d *caddyfile.Dispenser
wantErr bool
}{
{
name: "Unknown parameter",
d: caddyfile.NewTestDispenser(`
tracing {
foo bar
}`),
wantErr: true,
},
{
name: "Missed argument",
d: caddyfile.NewTestDispenser(`
tracing {
span
}`),
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ot := &Tracing{}
if err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr {
t.Errorf("UnmarshalCaddyfile() error = %v, wantErrType %v", err, tt.wantErr)
}
})
}
}
func TestTracing_ServeHTTP_Propagation_Without_Initial_Headers(t *testing.T) {
ot := &Tracing{
SpanName: "mySpan",
}
req := httptest.NewRequest("GET", "https://example.com/foo", nil)
w := httptest.NewRecorder()
var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {
traceparent := request.Header.Get("Traceparent")
if traceparent == "" || strings.HasPrefix(traceparent, "00-00000000000000000000000000000000-0000000000000000") {
t.Errorf("Invalid traceparent: %v", traceparent)
}
return nil
}
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
defer cancel()
if err := ot.Provision(ctx); err != nil {
t.Errorf("Provision error: %v", err)
t.FailNow()
}
if err := ot.ServeHTTP(w, req, handler); err != nil {
t.Errorf("ServeHTTP error: %v", err)
}
}
func TestTracing_ServeHTTP_Propagation_With_Initial_Headers(t *testing.T) {
ot := &Tracing{
SpanName: "mySpan",
}
req := httptest.NewRequest("GET", "https://example.com/foo", nil)
req.Header.Set("traceparent", "00-11111111111111111111111111111111-1111111111111111-01")
w := httptest.NewRecorder()
var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {
traceparent := request.Header.Get("Traceparent")
if !strings.HasPrefix(traceparent, "00-11111111111111111111111111111111") {
t.Errorf("Invalid traceparent: %v", traceparent)
}
return nil
}
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
defer cancel()
if err := ot.Provision(ctx); err != nil {
t.Errorf("Provision error: %v", err)
t.FailNow()
}
if err := ot.ServeHTTP(w, req, handler); err != nil {
t.Errorf("ServeHTTP error: %v", err)
}
}
func TestTracing_ServeHTTP_Next_Error(t *testing.T) {
ot := &Tracing{
SpanName: "mySpan",
}
req := httptest.NewRequest("GET", "https://example.com/foo", nil)
w := httptest.NewRecorder()
expectErr := errors.New("test error")
var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {
return expectErr
}
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
defer cancel()
if err := ot.Provision(ctx); err != nil {
t.Errorf("Provision error: %v", err)
t.FailNow()
}
if err := ot.ServeHTTP(w, req, handler); err == nil || !errors.Is(err, expectErr) {
t.Errorf("expected error, got: %v", err)
}
}

View File

@ -0,0 +1,108 @@
package tracing
import (
"context"
"fmt"
"net/http"
"github.com/caddyserver/caddy/v2"
caddycmd "github.com/caddyserver/caddy/v2/cmd"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
"go.uber.org/zap"
)
const (
webEngineName = "Caddy"
defaultSpanName = "handler"
nextCallCtxKey caddy.CtxKey = "nextCall"
)
// nextCall store the next handler, and the error value return on calling it (if any)
type nextCall struct {
next caddyhttp.Handler
err error
}
// openTelemetryWrapper is responsible for the tracing injection, extraction and propagation.
type openTelemetryWrapper struct {
propagators propagation.TextMapPropagator
handler http.Handler
spanName string
}
// newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration.
func newOpenTelemetryWrapper(
ctx context.Context,
spanName string,
) (openTelemetryWrapper, error) {
if spanName == "" {
spanName = defaultSpanName
}
ot := openTelemetryWrapper{
spanName: spanName,
}
res, err := ot.newResource(webEngineName, caddycmd.CaddyVersion())
if err != nil {
return ot, fmt.Errorf("creating resource error: %w", err)
}
traceExporter, err := otlptracegrpc.New(ctx)
if err != nil {
return ot, fmt.Errorf("creating trace exporter error: %w", err)
}
ot.propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
tracerProvider := globalTracerProvider.getTracerProvider(
sdktrace.WithBatcher(traceExporter),
sdktrace.WithResource(res),
)
ot.handler = otelhttp.NewHandler(http.HandlerFunc(ot.serveHTTP), ot.spanName, otelhttp.WithTracerProvider(tracerProvider), otelhttp.WithPropagators(ot.propagators))
return ot, nil
}
// serveHTTP injects a tracing context and call the next handler.
func (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) {
ot.propagators.Inject(r.Context(), propagation.HeaderCarrier(r.Header))
next := r.Context().Value(nextCallCtxKey).(*nextCall)
next.err = next.next.ServeHTTP(w, r)
}
// ServeHTTP propagates call to the by wrapped by `otelhttp` next handler.
func (ot *openTelemetryWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
n := &nextCall{
next: next,
err: nil,
}
ot.handler.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), nextCallCtxKey, n)))
return n.err
}
// cleanup flush all remaining data and shutdown a tracerProvider
func (ot *openTelemetryWrapper) cleanup(logger *zap.Logger) error {
return globalTracerProvider.cleanupTracerProvider(logger)
}
// newResource creates a resource that describe current handler instance and merge it with a default attributes value.
func (ot *openTelemetryWrapper) newResource(
webEngineName,
webEngineVersion string,
) (*resource.Resource, error) {
return resource.Merge(resource.Default(), resource.NewSchemaless(
semconv.WebEngineNameKey.String(webEngineName),
semconv.WebEngineVersionKey.String(webEngineVersion),
))
}

View File

@ -0,0 +1,27 @@
package tracing
import (
"context"
"testing"
"github.com/caddyserver/caddy/v2"
)
func TestOpenTelemetryWrapper_newOpenTelemetryWrapper(t *testing.T) {
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
defer cancel()
var otw openTelemetryWrapper
var err error
if otw, err = newOpenTelemetryWrapper(ctx,
"",
); err != nil {
t.Errorf("newOpenTelemetryWrapper() error = %v", err)
t.FailNow()
}
if otw.propagators == nil {
t.Errorf("Propagators should not be empty")
}
}

View File

@ -0,0 +1,63 @@
package tracing
import (
"context"
"fmt"
"sync"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.uber.org/zap"
)
// globalTracerProvider stores global tracer provider and is responsible for graceful shutdown when nobody is using it.
var globalTracerProvider = &tracerProvider{}
type tracerProvider struct {
mu sync.Mutex
tracerProvider *sdktrace.TracerProvider
tracerProvidersCounter int
}
// getTracerProvider create or return an existing global TracerProvider
func (t *tracerProvider) getTracerProvider(opts ...sdktrace.TracerProviderOption) *sdktrace.TracerProvider {
t.mu.Lock()
defer t.mu.Unlock()
t.tracerProvidersCounter++
if t.tracerProvider == nil {
t.tracerProvider = sdktrace.NewTracerProvider(
opts...,
)
}
return t.tracerProvider
}
// cleanupTracerProvider gracefully shutdown a TracerProvider
func (t *tracerProvider) cleanupTracerProvider(logger *zap.Logger) error {
t.mu.Lock()
defer t.mu.Unlock()
if t.tracerProvidersCounter > 0 {
t.tracerProvidersCounter--
}
if t.tracerProvidersCounter == 0 {
if t.tracerProvider != nil {
// tracerProvider.ForceFlush SHOULD be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#forceflush
if err := t.tracerProvider.ForceFlush(context.Background()); err != nil {
logger.Error("forcing flush", zap.Error(err))
}
// tracerProvider.Shutdown MUST be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#shutdown
if err := t.tracerProvider.Shutdown(context.Background()); err != nil {
return fmt.Errorf("tracerProvider shutdown error: %w", err)
}
}
t.tracerProvider = nil
}
return nil
}

View File

@ -0,0 +1,43 @@
package tracing
import (
"testing"
"go.uber.org/zap"
)
func Test_tracersProvider_getTracerProvider(t *testing.T) {
tp := tracerProvider{}
tp.getTracerProvider()
tp.getTracerProvider()
if tp.tracerProvider == nil {
t.Errorf("There should be tracer provider")
}
if tp.tracerProvidersCounter != 2 {
t.Errorf("Tracer providers counter should equal to 2")
}
}
func Test_tracersProvider_cleanupTracerProvider(t *testing.T) {
tp := tracerProvider{}
tp.getTracerProvider()
tp.getTracerProvider()
err := tp.cleanupTracerProvider(zap.NewNop())
if err != nil {
t.Errorf("There should be no error: %v", err)
}
if tp.tracerProvider == nil {
t.Errorf("There should be tracer provider")
}
if tp.tracerProvidersCounter != 1 {
t.Errorf("Tracer providers counter should equal to 1")
}
}