TUN-1613: improved cloudflared RegisterTunnel fail metrics

origin/metrics.go

@@ -53,6 +53,7 @@ type TunnelMetrics struct {
 
 	regSuccess prometheus.Counter
 	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
 
 	muxerMetrics *muxerMetrics
 	tunnelsHA    tunnelsForHA
@@ -345,6 +346,15 @@ func NewTunnelMetrics() *TunnelMetrics {
 	)
 	prometheus.MustRegister(serverLocations)
 
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "tunnel_rpc_fail",
+			Help: "Count of RPC connection errors by type",
+		},
+		[]string{"error"},
+	)
+	prometheus.MustRegister(rpcFail)
+
 	registerFail := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "tunnel_register_fail",
@@ -378,6 +388,7 @@ func NewTunnelMetrics() *TunnelMetrics {
 		tunnelsHA:                      NewTunnelsForHA(),
 		regSuccess:                     registerSuccess,
 		regFail:                        registerFail,
+		rpcFail:                        rpcFail,
 	}
 }
 

origin/tunnel.go

@@ -24,6 +24,7 @@ import (
 	raven "github.com/getsentry/raven-go"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	_ "github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
@@ -105,6 +106,11 @@ type clientRegisterTunnelError struct {
 	cause error
 }
 
+func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error()).Inc()
+	return clientRegisterTunnelError{cause: cause}
+}
+
 func (e clientRegisterTunnelError) Error() string {
 	return e.cause.Error()
 }
@@ -323,11 +329,11 @@ func RegisterTunnel(
 	}, nil)
 	if err != nil {
 		// RPC stream open error
-		return clientRegisterTunnelError{cause: err}
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
 	}
 	if !IsRPCStreamResponse(stream.Headers) {
 		// stream response error
-		return clientRegisterTunnelError{cause: err}
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
 	}
 	conn := rpc.NewConn(
 		tunnelrpc.NewTransportLogger(config.TransportLogger.WithField("subsystem", "rpc-register"), rpc.StreamTransport(stream)),
@@ -349,7 +355,7 @@ func RegisterTunnel(
 	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, config.Logger)
 	if err != nil {
 		// RegisterTunnel RPC failure
-		return clientRegisterTunnelError{cause: err}
+		return newClientRegisterTunnelError(err, config.Metrics.regFail)
 	}
 	for _, logLine := range registration.LogLines {
 		config.Logger.Info(logLine)