TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span

(cherry picked from commit f48a7cd3dd)
This commit is contained in:
Devin Carr 2022-07-26 14:00:53 -07:00
parent 7f1c890a82
commit b9cba7f2ae
13 changed files with 166 additions and 62 deletions

View File

@ -123,7 +123,7 @@ func (t Type) String() string {
// OriginProxy is how data flows from cloudflared to the origin services running behind it.
type OriginProxy interface {
ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error
ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error
ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
}
@ -133,13 +133,14 @@ type TCPRequest struct {
CFRay string
LBProbe bool
FlowID string
CfTraceID string
}
// ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
// accepted the connection.
type ReadWriteAcker interface {
io.ReadWriter
AckConnection() error
AckConnection(tracePropagation string) error
}
// HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
@ -168,7 +169,7 @@ func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
// AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
// upgrade to streams.
func (h *HTTPResponseReadWriteAcker) AckConnection() error {
func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) error {
resp := &http.Response{
Status: switchingProtocolText,
StatusCode: http.StatusSwitchingProtocols,
@ -179,6 +180,10 @@ func (h *HTTPResponseReadWriteAcker) AckConnection() error {
resp.Header = websocket.NewResponseHeader(h.req)
}
if tracePropagation != "" {
resp.Header.Add(tracing.CanonicalCloudflaredTracingHeader, tracePropagation)
}
return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
}

View File

@ -30,6 +30,8 @@ var (
testLargeResp = make([]byte, largeFileSize)
)
var _ ReadWriteAcker = (*HTTPResponseReadWriteAcker)(nil)
type testRequest struct {
name string
endpoint string
@ -60,7 +62,7 @@ type mockOriginProxy struct{}
func (moc *mockOriginProxy) ProxyHTTP(
w ResponseWriter,
tr *tracing.TracedRequest,
tr *tracing.TracedHTTPRequest,
isWebsocket bool,
) error {
req := tr.Request

View File

@ -69,6 +69,7 @@ func NewH2muxConnection(
connIndex uint8,
observer *Observer,
gracefulShutdownC <-chan struct{},
log *zerolog.Logger,
) (*h2muxConnection, error, bool) {
h := &h2muxConnection{
orchestrator: orchestrator,
@ -79,6 +80,7 @@ func NewH2muxConnection(
observer: observer,
gracefulShutdownC: gracefulShutdownC,
newRPCClientFunc: newRegistrationRPCClient,
log: log,
}
// Establish a muxed connection with the edge
@ -234,7 +236,7 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
return err
}
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), sourceConnectionType == TypeWebsocket)
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
if err != nil {
respWriter.WriteErrorResponse()
}

View File

@ -48,7 +48,7 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
}()
var connIndex = uint8(0)
testObserver := NewObserver(&log, &log)
h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
require.NoError(t, err)
return h2muxConn, <-edgeMuxChan
}

View File

@ -132,7 +132,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case TypeWebsocket, TypeHTTP:
stripWebsocketUpgradeHeader(r)
// Check for tracing on request
tr := tracing.NewTracedRequest(r)
tr := tracing.NewTracedHTTPRequest(r, c.log)
if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
err := fmt.Errorf("Failed to proxy HTTP: %w", err)
c.log.Error().Err(err)

View File

@ -197,7 +197,7 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
switch request.Type {
case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
tracedReq, err := buildHTTPRequest(ctx, request, stream)
tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
if err != nil {
return err
}
@ -210,6 +210,7 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
Dest: request.Dest,
FlowID: metadata[QUICMetadataFlowID],
CfTraceID: metadata[tracing.TracerContextName],
})
}
return nil
@ -296,8 +297,12 @@ type streamReadWriteAcker struct {
}
// AckConnection acks response back to the proxy.
func (s *streamReadWriteAcker) AckConnection() error {
return s.WriteConnectResponseData(nil)
func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
metadata := quicpogs.Metadata{
Key: tracing.CanonicalCloudflaredTracingHeader,
Val: tracePropagation,
}
return s.WriteConnectResponseData(nil, metadata)
}
// httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
@ -325,7 +330,12 @@ func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
}
func buildHTTPRequest(ctx context.Context, connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*tracing.TracedRequest, error) {
func buildHTTPRequest(
ctx context.Context,
connectRequest *quicpogs.ConnectRequest,
body io.ReadCloser,
log *zerolog.Logger,
) (*tracing.TracedHTTPRequest, error) {
metadata := connectRequest.MetadataMap()
dest := connectRequest.Dest
method := metadata[HTTPMethodKey]
@ -367,7 +377,7 @@ func buildHTTPRequest(ctx context.Context, connectRequest *quicpogs.ConnectReque
stripWebsocketUpgradeHeader(req)
// Check for tracing on request
tracedReq := tracing.NewTracedRequest(req)
tracedReq := tracing.NewTracedHTTPRequest(req, log)
return tracedReq, err
}

View File

@ -36,6 +36,8 @@ var (
}
)
var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
// TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
// It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
func TestQUICServer(t *testing.T) {
@ -220,7 +222,7 @@ func quicServer(
type mockOriginProxyWithRequest struct{}
func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error {
func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error {
// These are a series of crude tests to ensure the headers and http related data is transferred from
// metadata.
r := tr.Request
@ -475,9 +477,10 @@ func TestBuildHTTPRequest(t *testing.T) {
},
}
log := zerolog.Nop()
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body)
req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
assert.NoError(t, err)
test.req = test.req.WithContext(req.Context())
assert.Equal(t, test.req, req.Request)
@ -486,7 +489,7 @@ func TestBuildHTTPRequest(t *testing.T) {
}
func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWriteAcker, tcpRequest *TCPRequest) error {
rwa.AckConnection()
rwa.AckConnection("")
io.Copy(rwa, rwa)
return nil
}

View File

@ -355,7 +355,7 @@ func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Respo
return nil, err
}
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), false)
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), false)
if err != nil {
return nil, err
}
@ -604,7 +604,7 @@ func TestPersistentConnection(t *testing.T) {
respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
require.NoError(t, err)
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), true)
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), true)
require.NoError(t, err)
}()

View File

@ -63,7 +63,7 @@ func NewOriginProxy(
// a simple roundtrip or a tcp/websocket dial depending on ingres rule setup.
func (p *Proxy) ProxyHTTP(
w connection.ResponseWriter,
tr *tracing.TracedRequest,
tr *tracing.TracedHTTPRequest,
isWebsocket bool,
) error {
incrementRequests()
@ -108,7 +108,7 @@ func (p *Proxy) ProxyHTTP(
}
rws := connection.NewHTTPResponseReadWriterAcker(w, req)
if err := p.proxyStream(req.Context(), rws, dest, originProxy); err != nil {
if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy); err != nil {
rule, srv := ruleField(p.ingressRules, ruleNum)
p.logRequestError(err, cfRay, "", rule, srv)
return err
@ -137,9 +137,11 @@ func (p *Proxy) ProxyTCP(
serveCtx, cancel := context.WithCancel(ctx)
defer cancel()
tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, p.log)
p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream started")
if err := p.proxyStream(serveCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
p.logRequestError(err, req.CFRay, req.FlowID, "", ingress.ServiceWarpRouting)
return err
}
@ -160,7 +162,7 @@ func ruleField(ing ingress.Ingress, ruleNum int) (ruleID string, srv string) {
// ProxyHTTPRequest proxies requests of underlying type http and websocket to the origin service.
func (p *Proxy) proxyHTTPRequest(
w connection.ResponseWriter,
tr *tracing.TracedRequest,
tr *tracing.TracedHTTPRequest,
httpService ingress.HTTPOriginProxy,
isWebsocket bool,
disableChunkedEncoding bool,
@ -211,7 +213,7 @@ func (p *Proxy) proxyHTTPRequest(
}
// Add spans to response header (if available)
tr.AddSpans(resp.Header, p.log)
tr.AddSpans(resp.Header)
err = w.WriteRespHeaders(resp.StatusCode, resp.Header)
if err != nil {
@ -248,17 +250,23 @@ func (p *Proxy) proxyHTTPRequest(
// proxyStream proxies type TCP and other underlying types if the connection is defined as a stream oriented
// ingress rule.
func (p *Proxy) proxyStream(
ctx context.Context,
tr *tracing.TracedContext,
rwa connection.ReadWriteAcker,
dest string,
connectionProxy ingress.StreamBasedOriginProxy,
) error {
ctx := tr.Context
_, connectSpan := tr.Tracer().Start(ctx, "stream_connect")
originConn, err := connectionProxy.EstablishConnection(ctx, dest)
if err != nil {
tracing.EndWithErrorStatus(connectSpan, err)
return err
}
connectSpan.End()
if err := rwa.AckConnection(); err != nil {
encodedSpans := tr.GetSpans()
if err := rwa.AckConnection(encodedSpans); err != nil {
return err
}

View File

@ -157,7 +157,8 @@ func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
require.NoError(t, err)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
log := zerolog.Nop()
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
require.NoError(t, err)
for _, tag := range testTags {
assert.Equal(t, tag.Value, req.Header.Get(TagHeaderNamePrefix+tag.Name))
@ -184,7 +185,8 @@ func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
errGroup, ctx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), true)
log := zerolog.Nop()
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), true)
require.NoError(t, err)
require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
@ -245,7 +247,8 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
log := zerolog.Nop()
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
require.NoError(t, err)
require.Equal(t, http.StatusOK, responseWriter.Code)
@ -357,7 +360,7 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
req, err := http.NewRequest(http.MethodGet, test.url, nil)
require.NoError(t, err)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
require.NoError(t, err)
assert.Equal(t, test.expectedStatus, responseWriter.Code)
@ -404,7 +407,7 @@ func TestProxyError(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
assert.NoError(t, err)
assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false))
assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false))
}
type replayer struct {
@ -682,7 +685,8 @@ func TestConnections(t *testing.T) {
rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
} else {
err = proxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), test.args.connectionType == connection.TypeWebsocket)
log := zerolog.Nop()
err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), test.args.connectionType == connection.TypeWebsocket)
}
cancel()

View File

@ -557,6 +557,7 @@ func ServeH2mux(
connIndex,
config.Observer,
gracefulShutdownC,
config.Log,
)
if err != nil {
if !recoverable {

View File

@ -8,6 +8,7 @@ import (
"net/http"
"os"
"runtime"
"strings"
"github.com/rs/zerolog"
otelContrib "go.opentelemetry.io/contrib/propagators/jaeger"
@ -33,6 +34,9 @@ const (
MaxErrorDescriptionLen = 100
traceHttpStatusCodeKey = "upstreamStatusCode"
traceID128bitsWidth = 128 / 4
separator = ":"
)
var (
@ -66,22 +70,50 @@ func Init(version string) {
cloudflaredVersionAttribute = semconv.ProcessRuntimeVersionKey.String(version)
}
type TracedRequest struct {
type TracedHTTPRequest struct {
*http.Request
trace.TracerProvider
exporter InMemoryClient
*cfdTracer
}
// NewTracedRequest creates a new tracer for the current request context.
func NewTracedRequest(req *http.Request) *TracedRequest {
// NewTracedHTTPRequest creates a new tracer for the current HTTP request context.
func NewTracedHTTPRequest(req *http.Request, log *zerolog.Logger) *TracedHTTPRequest {
ctx, exists := extractTrace(req)
if !exists {
return &TracedRequest{req, trace.NewNoopTracerProvider(), &NoopOtlpClient{}}
return &TracedHTTPRequest{req, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}}
}
return &TracedHTTPRequest{req.WithContext(ctx), newCfdTracer(ctx, log)}
}
func (tr *TracedHTTPRequest) ToTracedContext() *TracedContext {
return &TracedContext{tr.Context(), tr.cfdTracer}
}
type TracedContext struct {
context.Context
*cfdTracer
}
// NewTracedHTTPRequest creates a new tracer for the current HTTP request context.
func NewTracedContext(ctx context.Context, traceContext string, log *zerolog.Logger) *TracedContext {
ctx, exists := extractTraceFromString(ctx, traceContext)
if !exists {
return &TracedContext{ctx, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}}
}
return &TracedContext{ctx, newCfdTracer(ctx, log)}
}
type cfdTracer struct {
trace.TracerProvider
exporter InMemoryClient
log *zerolog.Logger
}
// NewCfdTracer creates a new tracer for the current request context.
func newCfdTracer(ctx context.Context, log *zerolog.Logger) *cfdTracer {
mc := new(InMemoryOtlpClient)
exp, err := otlptrace.New(req.Context(), mc)
exp, err := otlptrace.New(ctx, mc)
if err != nil {
return &TracedRequest{req, trace.NewNoopTracerProvider(), &NoopOtlpClient{}}
return &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}
}
tp := tracesdk.NewTracerProvider(
// We want to dump to in-memory exporter immediately
@ -98,36 +130,41 @@ func NewTracedRequest(req *http.Request) *TracedRequest {
)),
)
return &TracedRequest{req.WithContext(ctx), tp, mc}
return &cfdTracer{tp, mc, log}
}
func (cft *TracedRequest) Tracer() trace.Tracer {
func (cft *cfdTracer) Tracer() trace.Tracer {
return cft.TracerProvider.Tracer(tracerInstrumentName)
}
// Spans returns the spans as base64 encoded protobuf otlp traces.
func (cft *TracedRequest) AddSpans(headers http.Header, log *zerolog.Logger) {
if headers == nil {
log.Error().Msgf("provided headers map is nil")
return
}
// GetSpans returns the spans as base64 encoded string of protobuf otlp traces.
func (cft *cfdTracer) GetSpans() (enc string) {
enc, err := cft.exporter.Spans()
switch err {
case nil:
break
case errNoTraces:
log.Error().Err(err).Msgf("expected traces to be available")
cft.log.Trace().Err(err).Msgf("expected traces to be available")
return
case errNoopTracer:
return // noop tracer has no traces
default:
log.Error().Err(err)
cft.log.Debug().Err(err)
return
}
return
}
// AddSpans assigns spans as base64 encoded protobuf otlp traces to provided
// HTTP headers.
func (cft *cfdTracer) AddSpans(headers http.Header) {
if headers == nil {
return
}
enc := cft.GetSpans()
// No need to add header if no traces
if enc == "" {
log.Error().Msgf("no traces provided and no error from exporter")
return
}
@ -166,6 +203,33 @@ func endSpan(span trace.Span, upstreamStatusCode int, spanStatusCode codes.Code,
span.End()
}
// extractTraceFromString will extract the trace information from the provided
// propagated trace string context.
func extractTraceFromString(ctx context.Context, trace string) (context.Context, bool) {
if trace == "" {
return ctx, false
}
// Jaeger specific separator
parts := strings.Split(trace, separator)
if len(parts) != 4 {
return ctx, false
}
if parts[0] == "" {
return ctx, false
}
// Correctly left pad the trace to a length of 32
if len(parts[0]) < traceID128bitsWidth {
left := traceID128bitsWidth - len(parts[0])
parts[0] = strings.Repeat("0", left) + parts[0]
trace = strings.Join(parts, separator)
}
// Override the 'cf-trace-id' as 'uber-trace-id' so the jaeger propagator can extract it.
traceHeader := map[string]string{TracerContextNameOverride: trace}
remoteCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(traceHeader))
return remoteCtx, true
}
// extractTrace attempts to check for a cf-trace-id from a request and return the
// trace context with the provided http.Request.
func extractTrace(req *http.Request) (context.Context, bool) {

View File

@ -14,38 +14,42 @@ import (
)
func TestNewCfTracer(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
tr := NewTracedRequest(req)
tr := NewTracedHTTPRequest(req, &log)
assert.NotNil(t, tr)
assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
}
func TestNewCfTracerMultiple(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header.Add(TracerContextName, "1241ce3ecdefc68854e8514e69ba42ca:b38f1bf5eae406f3:0:1")
req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
tr := NewTracedRequest(req)
tr := NewTracedHTTPRequest(req, &log)
assert.NotNil(t, tr)
assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
}
func TestNewCfTracerNilHeader(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header[http.CanonicalHeaderKey(TracerContextName)] = nil
tr := NewTracedRequest(req)
tr := NewTracedHTTPRequest(req, &log)
assert.NotNil(t, tr)
assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
}
func TestNewCfTracerInvalidHeaders(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
for _, test := range [][]string{nil, {""}} {
req.Header[http.CanonicalHeaderKey(TracerContextName)] = test
tr := NewTracedRequest(req)
tr := NewTracedHTTPRequest(req, &log)
assert.NotNil(t, tr)
assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
@ -53,9 +57,10 @@ func TestNewCfTracerInvalidHeaders(t *testing.T) {
}
func TestAddingSpansWithNilMap(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
tr := NewTracedRequest(req)
tr := NewTracedHTTPRequest(req, &log)
exporter := tr.exporter.(*InMemoryOtlpClient)
@ -65,5 +70,5 @@ func TestAddingSpansWithNilMap(t *testing.T) {
assert.NoError(t, err)
// a panic shouldn't occur
tr.AddSpans(nil, &zerolog.Logger{})
tr.AddSpans(nil)
}