2018-05-01 23:45:06 +00:00
|
|
|
package metrics
|
|
|
|
|
|
|
|
import (
|
2019-01-10 20:55:44 +00:00
|
|
|
"context"
|
2019-11-12 22:55:40 +00:00
|
|
|
"fmt"
|
2018-05-01 23:45:06 +00:00
|
|
|
"net"
|
|
|
|
"net/http"
|
|
|
|
_ "net/http/pprof"
|
|
|
|
"runtime"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
2020-11-25 06:55:13 +00:00
|
|
|
"github.com/rs/zerolog"
|
|
|
|
"golang.org/x/net/trace"
|
2018-05-01 23:45:06 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2022-11-22 11:30:28 +00:00
|
|
|
startupTime = time.Millisecond * 500
|
|
|
|
defaultShutdownTimeout = time.Second * 15
|
2018-05-01 23:45:06 +00:00
|
|
|
)
|
|
|
|
|
2022-11-22 11:30:28 +00:00
|
|
|
type Config struct {
|
|
|
|
ReadyServer *ReadyServer
|
|
|
|
QuickTunnelHostname string
|
|
|
|
Orchestrator orchestrator
|
|
|
|
|
|
|
|
ShutdownTimeout time.Duration
|
|
|
|
}
|
|
|
|
|
2022-03-14 17:51:10 +00:00
|
|
|
type orchestrator interface {
|
2022-04-27 10:51:06 +00:00
|
|
|
GetVersionedConfigJSON() ([]byte, error)
|
2022-03-14 17:51:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func newMetricsHandler(
|
2022-11-22 11:30:28 +00:00
|
|
|
config Config,
|
2022-03-14 17:51:10 +00:00
|
|
|
log *zerolog.Logger,
|
2022-12-25 04:05:51 +00:00
|
|
|
) *http.ServeMux {
|
|
|
|
router := http.NewServeMux()
|
2023-01-31 11:17:49 +00:00
|
|
|
router.Handle("/debug/", http.DefaultServeMux)
|
2021-01-19 11:53:54 +00:00
|
|
|
router.Handle("/metrics", promhttp.Handler())
|
|
|
|
router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
|
2020-11-25 06:55:13 +00:00
|
|
|
_, _ = fmt.Fprintf(w, "OK\n")
|
2020-11-30 20:05:37 +00:00
|
|
|
})
|
2022-11-22 11:30:28 +00:00
|
|
|
if config.ReadyServer != nil {
|
|
|
|
router.Handle("/ready", config.ReadyServer)
|
2021-01-14 22:33:36 +00:00
|
|
|
}
|
2021-07-09 17:10:43 +00:00
|
|
|
router.HandleFunc("/quicktunnel", func(w http.ResponseWriter, r *http.Request) {
|
2022-11-22 11:30:28 +00:00
|
|
|
_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, config.QuickTunnelHostname)
|
2021-07-09 17:10:43 +00:00
|
|
|
})
|
2022-11-22 11:30:28 +00:00
|
|
|
if config.Orchestrator != nil {
|
2022-03-14 17:51:10 +00:00
|
|
|
router.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
|
2022-11-22 11:30:28 +00:00
|
|
|
json, err := config.Orchestrator.GetVersionedConfigJSON()
|
2022-03-14 17:51:10 +00:00
|
|
|
if err != nil {
|
|
|
|
w.WriteHeader(500)
|
|
|
|
_, _ = fmt.Fprintf(w, "ERR: %v", err)
|
|
|
|
log.Err(err).Msg("Failed to serve config")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
_, _ = w.Write(json)
|
|
|
|
})
|
|
|
|
}
|
2021-01-19 11:53:54 +00:00
|
|
|
|
|
|
|
return router
|
2020-11-30 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func ServeMetrics(
|
|
|
|
l net.Listener,
|
2022-11-22 11:30:28 +00:00
|
|
|
ctx context.Context,
|
|
|
|
config Config,
|
2020-11-25 06:55:13 +00:00
|
|
|
log *zerolog.Logger,
|
2020-11-30 20:05:37 +00:00
|
|
|
) (err error) {
|
2018-05-01 23:45:06 +00:00
|
|
|
var wg sync.WaitGroup
|
|
|
|
// Metrics port is privileged, so no need for further access control
|
|
|
|
trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
|
|
|
|
// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
|
|
|
|
// profile CPU usage depends on WriteTimeout
|
2022-11-22 11:30:28 +00:00
|
|
|
h := newMetricsHandler(config, log)
|
2018-05-01 23:45:06 +00:00
|
|
|
server := &http.Server{
|
|
|
|
ReadTimeout: 10 * time.Second,
|
|
|
|
WriteTimeout: 10 * time.Second,
|
2020-11-30 20:05:37 +00:00
|
|
|
Handler: h,
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
err = server.Serve(l)
|
|
|
|
}()
|
2020-11-25 06:55:13 +00:00
|
|
|
log.Info().Msgf("Starting metrics server on %s", fmt.Sprintf("%v/metrics", l.Addr()))
|
2018-05-01 23:45:06 +00:00
|
|
|
// server.Serve will hang if server.Shutdown is called before the server is
|
|
|
|
// fully started up. So add artificial delay.
|
|
|
|
time.Sleep(startupTime)
|
|
|
|
|
2022-11-22 11:30:28 +00:00
|
|
|
<-ctx.Done()
|
|
|
|
shutdownTimeout := config.ShutdownTimeout
|
|
|
|
if shutdownTimeout == 0 {
|
|
|
|
shutdownTimeout = defaultShutdownTimeout
|
|
|
|
}
|
|
|
|
|
2018-05-01 23:45:06 +00:00
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
2020-11-25 06:55:13 +00:00
|
|
|
_ = server.Shutdown(ctx)
|
2018-05-01 23:45:06 +00:00
|
|
|
cancel()
|
|
|
|
|
|
|
|
wg.Wait()
|
|
|
|
if err == http.ErrServerClosed {
|
2020-11-25 06:55:13 +00:00
|
|
|
log.Info().Msg("Metrics server stopped")
|
2018-05-01 23:45:06 +00:00
|
|
|
return nil
|
|
|
|
}
|
2020-12-28 18:10:01 +00:00
|
|
|
log.Err(err).Msg("Metrics server failed")
|
2018-05-01 23:45:06 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-12-27 19:05:14 +00:00
|
|
|
func RegisterBuildInfo(buildType, buildTime, version string) {
|
2018-05-01 23:45:06 +00:00
|
|
|
buildInfo := prometheus.NewGaugeVec(
|
|
|
|
prometheus.GaugeOpts{
|
|
|
|
// Don't namespace build_info, since we want it to be consistent across all Cloudflare services
|
|
|
|
Name: "build_info",
|
|
|
|
Help: "Build and version information",
|
|
|
|
},
|
2021-12-27 19:05:14 +00:00
|
|
|
[]string{"goversion", "type", "revision", "version"},
|
2018-05-01 23:45:06 +00:00
|
|
|
)
|
|
|
|
prometheus.MustRegister(buildInfo)
|
2021-12-27 19:05:14 +00:00
|
|
|
buildInfo.WithLabelValues(runtime.Version(), buildType, buildTime, version).Set(1)
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|