package tracing import ( "context" "errors" "fmt" "math" "net/http" "os" "runtime" "strings" "github.com/rs/zerolog" otelContrib "go.opentelemetry.io/contrib/propagators/jaeger" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" tracesdk "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.7.0" "go.opentelemetry.io/otel/trace" ) const ( service = "cloudflared" tracerInstrumentName = "origin" TracerContextName = "cf-trace-id" TracerContextNameOverride = "uber-trace-id" IntCloudflaredTracingHeader = "cf-int-cloudflared-tracing" MaxErrorDescriptionLen = 100 traceHttpStatusCodeKey = "upstreamStatusCode" traceID128bitsWidth = 128 / 4 separator = ":" ) var ( CanonicalCloudflaredTracingHeader = http.CanonicalHeaderKey(IntCloudflaredTracingHeader) Http2TransportAttribute = trace.WithAttributes(transportAttributeKey.String("http2")) QuicTransportAttribute = trace.WithAttributes(transportAttributeKey.String("quic")) HostOSAttribute = semconv.HostTypeKey.String(runtime.GOOS) HostArchAttribute = semconv.HostArchKey.String(runtime.GOARCH) otelVersionAttribute attribute.KeyValue hostnameAttribute attribute.KeyValue cloudflaredVersionAttribute attribute.KeyValue serviceAttribute = semconv.ServiceNameKey.String(service) transportAttributeKey = attribute.Key("transport") otelVersionAttributeKey = attribute.Key("jaeger.version") errNoopTracerProvider = errors.New("noop tracer provider records no spans") ) func init() { // Register the jaeger propagator globally. otel.SetTextMapPropagator(otelContrib.Jaeger{}) otelVersionAttribute = otelVersionAttributeKey.String(fmt.Sprintf("go-otel-%s", otel.Version())) if hostname, err := os.Hostname(); err == nil { hostnameAttribute = attribute.String("hostname", hostname) } } func Init(version string) { cloudflaredVersionAttribute = semconv.ProcessRuntimeVersionKey.String(version) } type TracedHTTPRequest struct { *http.Request *cfdTracer ConnIndex uint8 // The connection index used to proxy the request } // NewTracedHTTPRequest creates a new tracer for the current HTTP request context. func NewTracedHTTPRequest(req *http.Request, connIndex uint8, log *zerolog.Logger) *TracedHTTPRequest { ctx, exists := extractTrace(req) if !exists { return &TracedHTTPRequest{req, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}, connIndex} } return &TracedHTTPRequest{req.WithContext(ctx), newCfdTracer(ctx, log), connIndex} } func (tr *TracedHTTPRequest) ToTracedContext() *TracedContext { return &TracedContext{tr.Context(), tr.cfdTracer} } type TracedContext struct { context.Context *cfdTracer } // NewTracedContext creates a new tracer for the current context. func NewTracedContext(ctx context.Context, traceContext string, log *zerolog.Logger) *TracedContext { ctx, exists := extractTraceFromString(ctx, traceContext) if !exists { return &TracedContext{ctx, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}} } return &TracedContext{ctx, newCfdTracer(ctx, log)} } type cfdTracer struct { trace.TracerProvider exporter InMemoryClient log *zerolog.Logger } // NewCfdTracer creates a new tracer for the current request context. func newCfdTracer(ctx context.Context, log *zerolog.Logger) *cfdTracer { mc := new(InMemoryOtlpClient) exp, err := otlptrace.New(ctx, mc) if err != nil { return &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log} } tp := tracesdk.NewTracerProvider( // We want to dump to in-memory exporter immediately tracesdk.WithSyncer(exp), // Record information about this application in a Resource. tracesdk.WithResource(resource.NewWithAttributes( semconv.SchemaURL, serviceAttribute, otelVersionAttribute, hostnameAttribute, cloudflaredVersionAttribute, HostOSAttribute, HostArchAttribute, )), ) return &cfdTracer{tp, mc, log} } func (cft *cfdTracer) Tracer() trace.Tracer { return cft.TracerProvider.Tracer(tracerInstrumentName) } // GetSpans returns the spans as base64 encoded string of protobuf otlp traces. func (cft *cfdTracer) GetSpans() (enc string) { enc, err := cft.exporter.Spans() switch err { case nil: break case errNoTraces: cft.log.Trace().Err(err).Msgf("expected traces to be available") return case errNoopTracer: return // noop tracer has no traces default: cft.log.Debug().Err(err) return } return } // GetProtoSpans returns the spans as the otlp traces in protobuf byte array. func (cft *cfdTracer) GetProtoSpans() (proto []byte) { proto, err := cft.exporter.ExportProtoSpans() switch err { case nil: break case errNoTraces: cft.log.Trace().Err(err).Msgf("expected traces to be available") return case errNoopTracer: return // noop tracer has no traces default: cft.log.Debug().Err(err) return } return } // AddSpans assigns spans as base64 encoded protobuf otlp traces to provided // HTTP headers. func (cft *cfdTracer) AddSpans(headers http.Header) { if headers == nil { return } enc := cft.GetSpans() // No need to add header if no traces if enc == "" { return } headers[CanonicalCloudflaredTracingHeader] = []string{enc} } // End will set the OK status for the span and then end it. func End(span trace.Span) { endSpan(span, -1, codes.Ok, nil) } // EndWithErrorStatus will set a status for the span and then end it. func EndWithErrorStatus(span trace.Span, err error) { endSpan(span, -1, codes.Error, err) } // EndWithStatusCode will set a status for the span and then end it. func EndWithStatusCode(span trace.Span, statusCode int) { endSpan(span, statusCode, codes.Ok, nil) } // EndWithErrorStatus will set a status for the span and then end it. func endSpan(span trace.Span, upstreamStatusCode int, spanStatusCode codes.Code, err error) { if span == nil { return } if upstreamStatusCode > 0 { span.SetAttributes(attribute.Int(traceHttpStatusCodeKey, upstreamStatusCode)) } // add error to status buf cap description errDescription := "" if err != nil { errDescription = err.Error() l := int(math.Min(float64(len(errDescription)), MaxErrorDescriptionLen)) errDescription = errDescription[:l] } span.SetStatus(spanStatusCode, errDescription) span.End() } // extractTraceFromString will extract the trace information from the provided // propagated trace string context. func extractTraceFromString(ctx context.Context, trace string) (context.Context, bool) { if trace == "" { return ctx, false } // Jaeger specific separator parts := strings.Split(trace, separator) if len(parts) != 4 { return ctx, false } if parts[0] == "" { return ctx, false } // Correctly left pad the trace to a length of 32 if len(parts[0]) < traceID128bitsWidth { left := traceID128bitsWidth - len(parts[0]) parts[0] = strings.Repeat("0", left) + parts[0] trace = strings.Join(parts, separator) } // Override the 'cf-trace-id' as 'uber-trace-id' so the jaeger propagator can extract it. traceHeader := map[string]string{TracerContextNameOverride: trace} remoteCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(traceHeader)) return remoteCtx, true } // extractTrace attempts to check for a cf-trace-id from a request and return the // trace context with the provided http.Request. func extractTrace(req *http.Request) (context.Context, bool) { // Only add tracing for requests with appropriately tagged headers remoteTraces := req.Header.Values(TracerContextName) if len(remoteTraces) <= 0 { // Strip the cf-trace-id header req.Header.Del(TracerContextName) return nil, false } traceHeader := map[string]string{} for _, t := range remoteTraces { // Override the 'cf-trace-id' as 'uber-trace-id' so the jaeger propagator can extract it. // Last entry wins if multiple provided traceHeader[TracerContextNameOverride] = t } // Strip the cf-trace-id header req.Header.Del(TracerContextName) if traceHeader[TracerContextNameOverride] == "" { return nil, false } remoteCtx := otel.GetTextMapPropagator().Extract(req.Context(), propagation.MapCarrier(traceHeader)) return remoteCtx, true } func NewNoopSpan() trace.Span { return trace.SpanFromContext(nil) }