diff --git a/origin/metrics.go b/origin/metrics.go index a3a01a74..6cc53960 100644 --- a/origin/metrics.go +++ b/origin/metrics.go @@ -53,6 +53,7 @@ type TunnelMetrics struct { regSuccess prometheus.Counter regFail *prometheus.CounterVec + rpcFail *prometheus.CounterVec muxerMetrics *muxerMetrics tunnelsHA tunnelsForHA @@ -345,6 +346,15 @@ func NewTunnelMetrics() *TunnelMetrics { ) prometheus.MustRegister(serverLocations) + rpcFail := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "tunnel_rpc_fail", + Help: "Count of RPC connection errors by type", + }, + []string{"error"}, + ) + prometheus.MustRegister(rpcFail) + registerFail := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "tunnel_register_fail", @@ -378,6 +388,7 @@ func NewTunnelMetrics() *TunnelMetrics { tunnelsHA: NewTunnelsForHA(), regSuccess: registerSuccess, regFail: registerFail, + rpcFail: rpcFail, } } diff --git a/origin/tunnel.go b/origin/tunnel.go index e728ee8b..9da7e5c4 100644 --- a/origin/tunnel.go +++ b/origin/tunnel.go @@ -24,6 +24,7 @@ import ( raven "github.com/getsentry/raven-go" "github.com/google/uuid" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" _ "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" @@ -105,6 +106,11 @@ type clientRegisterTunnelError struct { cause error } +func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec) clientRegisterTunnelError { + counter.WithLabelValues(cause.Error()).Inc() + return clientRegisterTunnelError{cause: cause} +} + func (e clientRegisterTunnelError) Error() string { return e.cause.Error() } @@ -323,11 +329,11 @@ func RegisterTunnel( }, nil) if err != nil { // RPC stream open error - return clientRegisterTunnelError{cause: err} + return newClientRegisterTunnelError(err, config.Metrics.rpcFail) } if !IsRPCStreamResponse(stream.Headers) { // stream response error - return clientRegisterTunnelError{cause: err} + return newClientRegisterTunnelError(err, config.Metrics.rpcFail) } conn := rpc.NewConn( tunnelrpc.NewTransportLogger(config.TransportLogger.WithField("subsystem", "rpc-register"), rpc.StreamTransport(stream)), @@ -349,7 +355,7 @@ func RegisterTunnel( LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, config.Logger) if err != nil { // RegisterTunnel RPC failure - return clientRegisterTunnelError{cause: err} + return newClientRegisterTunnelError(err, config.Metrics.regFail) } for _, logLine := range registration.LogLines { config.Logger.Info(logLine)