TUN-7553: Add flag to enable management diagnostic services

With the new flag --management-diagnostics (an opt-in flag)
cloudflared's will be able to report additional diagnostic information
over the management.argotunnel.com request path.
Additions include the /metrics prometheus endpoint; which is already
bound to a local port via --metrics.
/debug/pprof/(goroutine|heap) are also provided to allow for remotely
retrieving heap information from a running cloudflared connector.
This commit is contained in:
Devin Carr 2023-07-05 13:28:30 -07:00
parent 39847a70f2
commit 8a3eade6d3
6 changed files with 77 additions and 20 deletions

View File

@ -1,3 +1,7 @@
## 2023.6.2
### New Features
- You can now enable additional diagnostics over the management.argotunnel.com service for your active cloudflared connectors via a new runtime flag `--management-diagnostics` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`). This feature is provided as opt-out and requires the flag to enable. Endpoints such as /metrics provides your prometheus metrics endpoint another mechanism to be reached. Additionally /debug/pprof/(goroutine|heap) are also introduced to allow for remotely retrieving active pprof information from a running cloudflared connector.
## 2023.4.1 ## 2023.4.1
### New Features ### New Features
- You can now stream your logs from your remote cloudflared to your local terminal with `cloudflared tail <TUNNEL-ID>`. This new feature requires the remote cloudflared to be version 2023.4.1 or higher. - You can now stream your logs from your remote cloudflared to your local terminal with `cloudflared tail <TUNNEL-ID>`. This new feature requires the remote cloudflared to be version 2023.4.1 or higher.

View File

@ -413,6 +413,7 @@ func StartServer(
mgmt := management.New( mgmt := management.New(
c.String("management-hostname"), c.String("management-hostname"),
c.Bool("management-diagnostics"),
serviceIP, serviceIP,
clientID, clientID,
c.String(connectorLabelFlag), c.String(connectorLabelFlag),
@ -764,6 +765,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
EnvVars: []string{"TUNNEL_POST_QUANTUM"}, EnvVars: []string{"TUNNEL_POST_QUANTUM"},
Hidden: FipsEnabled, Hidden: FipsEnabled,
}), }),
altsrc.NewBoolFlag(&cli.BoolFlag{
Name: "management-diagnostics",
Usage: "Enables the in-depth diagnostic routes to be made available over the management service (/debug/pprof, /metrics, etc.)",
EnvVars: []string{"TUNNEL_MANAGEMENT_DIAGNOSTICS"},
Value: false,
}),
selectProtocolFlag, selectProtocolFlag,
overwriteDNSFlag, overwriteDNSFlag,
}...) }...)

View File

@ -55,7 +55,7 @@ class TestManagement:
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False) config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
LOGGER.debug(config) LOGGER.debug(config)
config_path = write_config(tmp_path, config.full_config) config_path = write_config(tmp_path, config.full_config)
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True): with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
wait_tunnel_ready(require_min_connections=1) wait_tunnel_ready(require_min_connections=1)
cfd_cli = CloudflaredCli(config, config_path, LOGGER) cfd_cli = CloudflaredCli(config, config_path, LOGGER)
url = cfd_cli.get_management_url("metrics", config, config_path) url = cfd_cli.get_management_url("metrics", config, config_path)
@ -76,7 +76,7 @@ class TestManagement:
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False) config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
LOGGER.debug(config) LOGGER.debug(config)
config_path = write_config(tmp_path, config.full_config) config_path = write_config(tmp_path, config.full_config)
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True): with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
wait_tunnel_ready(require_min_connections=1) wait_tunnel_ready(require_min_connections=1)
cfd_cli = CloudflaredCli(config, config_path, LOGGER) cfd_cli = CloudflaredCli(config, config_path, LOGGER)
url = cfd_cli.get_management_url("debug/pprof/heap", config, config_path) url = cfd_cli.get_management_url("debug/pprof/heap", config, config_path)
@ -86,6 +86,26 @@ class TestManagement:
assert resp.status_code == 200, "Expected cloudflared to return 200 for /debug/pprof/heap" assert resp.status_code == 200, "Expected cloudflared to return 200 for /debug/pprof/heap"
assert resp.headers["Content-Type"] == "application/octet-stream", "Expected /debug/pprof/heap to have return a binary response" assert resp.headers["Content-Type"] == "application/octet-stream", "Expected /debug/pprof/heap to have return a binary response"
"""
test_get_metrics_when_disabled will verify that diagnostic endpoints (such as /metrics) return 404 and are unmounted.
"""
def test_get_metrics_when_disabled(self, tmp_path, component_tests_config):
# TUN-7377 : wait_tunnel_ready does not work properly in windows.
# Skipping this test for windows for now and will address it as part of tun-7377
if platform.system() == "Windows":
return
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
LOGGER.debug(config)
config_path = write_config(tmp_path, config.full_config)
with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
wait_tunnel_ready(require_min_connections=1)
cfd_cli = CloudflaredCli(config, config_path, LOGGER)
url = cfd_cli.get_management_url("metrics", config, config_path)
resp = send_request(url)
# Assert response.
assert resp.status_code == 404, "Expected cloudflared to return 404 for /metrics"
@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000) @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
def send_request(url, headers={}): def send_request(url, headers={}):

View File

@ -31,6 +31,17 @@ const (
reasonIdleLimitExceeded = "session was idle for too long" reasonIdleLimitExceeded = "session was idle for too long"
) )
var (
// CORS middleware required to allow dash to access management.argotunnel.com requests
corsHandler = cors.Handler(cors.Options{
// Allows for any subdomain of cloudflare.com
AllowedOrigins: []string{"https://*.cloudflare.com"},
// Required to present cookies or other authentication across origin boundries
AllowCredentials: true,
MaxAge: 300, // Maximum value not ignored by any of major browsers
})
)
type ManagementService struct { type ManagementService struct {
// The management tunnel hostname // The management tunnel hostname
Hostname string Hostname string
@ -54,6 +65,7 @@ type ManagementService struct {
} }
func New(managementHostname string, func New(managementHostname string,
enableDiagServices bool,
serviceIP string, serviceIP string,
clientID uuid.UUID, clientID uuid.UUID,
label string, label string,
@ -71,25 +83,21 @@ func New(managementHostname string,
} }
r := chi.NewRouter() r := chi.NewRouter()
r.Use(ValidateAccessTokenQueryMiddleware) r.Use(ValidateAccessTokenQueryMiddleware)
r.Get("/ping", ping)
r.Head("/ping", ping) // Default management services
r.With(corsHandler).Get("/ping", ping)
r.With(corsHandler).Head("/ping", ping)
r.Get("/logs", s.logs) r.Get("/logs", s.logs)
r.Get("/metrics", s.metricsHandler.ServeHTTP) r.With(corsHandler).Get("/host_details", s.getHostDetails)
// Diagnostic management services
if enableDiagServices {
// Prometheus endpoint
r.With(corsHandler).Get("/metrics", s.metricsHandler.ServeHTTP)
// Supports only heap and goroutine // Supports only heap and goroutine
r.Get("/debug/pprof/{profile:heap|goroutine}", pprof.Index) r.With(corsHandler).Get("/debug/pprof/{profile:heap|goroutine}", pprof.Index)
}
r.Route("/host_details", func(r chi.Router) {
// CORS middleware required to allow dash to access management.argotunnel.com requests
r.Use(cors.Handler(cors.Options{
// Allows for any subdomain of cloudflare.com
AllowedOrigins: []string{"https://*.cloudflare.com"},
// Required to present cookies or other authentication across origin boundries
AllowCredentials: true,
MaxAge: 300, // Maximum value not ignored by any of major browsers
}))
r.Get("/", s.getHostDetails)
})
s.router = r s.router = r
return s return s
} }

View File

@ -3,9 +3,13 @@ package management
import ( import (
"context" "context"
"io" "io"
"net/http"
"net/http/httptest"
"strings"
"testing" "testing"
"time" "time"
"github.com/google/uuid"
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -16,8 +20,22 @@ import (
var ( var (
noopLogger = zerolog.New(io.Discard) noopLogger = zerolog.New(io.Discard)
managementHostname = "https://management.argotunnel.com"
) )
func TestDisableDiagnosticRoutes(t *testing.T) {
mgmt := New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &noopLogger, nil)
for _, path := range []string{"/metrics", "/debug/pprof/goroutine", "/debug/pprof/heap"} {
t.Run(strings.Replace(path, "/", "_", -1), func(t *testing.T) {
req := httptest.NewRequest("GET", managementHostname+path+"?access_token="+validToken, nil)
recorder := httptest.NewRecorder()
mgmt.ServeHTTP(recorder, req)
resp := recorder.Result()
require.Equal(t, http.StatusNotFound, resp.StatusCode)
})
}
}
func TestReadEventsLoop(t *testing.T) { func TestReadEventsLoop(t *testing.T) {
sentEvent := EventStartStreaming{ sentEvent := EventStartStreaming{
ClientEvent: ClientEvent{Type: StartStreaming}, ClientEvent: ClientEvent{Type: StartStreaming},

View File

@ -51,7 +51,7 @@ func TestUpdateConfiguration(t *testing.T) {
initConfig := &Config{ initConfig := &Config{
Ingress: &ingress.Ingress{}, Ingress: &ingress.Ingress{},
} }
orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger) orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger)
require.NoError(t, err) require.NoError(t, err)
initOriginProxy, err := orchestrator.GetOriginProxy() initOriginProxy, err := orchestrator.GetOriginProxy()
require.NoError(t, err) require.NoError(t, err)