2022-02-07 09:42:07 +00:00
|
|
|
package supervisor
|
2018-05-01 23:45:06 +00:00
|
|
|
|
|
|
|
import (
|
2019-01-10 20:55:44 +00:00
|
|
|
"context"
|
2019-12-04 17:22:08 +00:00
|
|
|
"errors"
|
2022-09-19 11:36:25 +00:00
|
|
|
"net"
|
2022-06-18 00:24:37 +00:00
|
|
|
"strings"
|
2018-05-01 23:45:06 +00:00
|
|
|
"time"
|
2019-02-01 20:11:12 +00:00
|
|
|
|
2023-05-06 00:42:41 +00:00
|
|
|
"github.com/quic-go/quic-go"
|
2021-03-23 14:30:43 +00:00
|
|
|
"github.com/rs/zerolog"
|
|
|
|
|
2019-03-18 23:14:47 +00:00
|
|
|
"github.com/cloudflare/cloudflared/connection"
|
2020-02-06 00:55:26 +00:00
|
|
|
"github.com/cloudflare/cloudflared/edgediscovery"
|
2022-02-11 10:49:06 +00:00
|
|
|
"github.com/cloudflare/cloudflared/orchestration"
|
2021-03-26 04:04:56 +00:00
|
|
|
"github.com/cloudflare/cloudflared/retry"
|
2019-03-04 19:48:56 +00:00
|
|
|
"github.com/cloudflare/cloudflared/signal"
|
2022-08-11 20:31:36 +00:00
|
|
|
"github.com/cloudflare/cloudflared/tunnelstate"
|
2018-05-01 23:45:06 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// Waiting time before retrying a failed tunnel connection
|
|
|
|
tunnelRetryDuration = time.Second * 10
|
|
|
|
// Interval between registering new tunnels
|
|
|
|
registrationInterval = time.Second
|
2019-12-04 17:22:08 +00:00
|
|
|
|
|
|
|
subsystemRefreshAuth = "refresh_auth"
|
|
|
|
// Maximum exponent for 'Authenticate' exponential backoff
|
|
|
|
refreshAuthMaxBackoff = 10
|
|
|
|
// Waiting time before retrying a failed 'Authenticate' connection
|
|
|
|
refreshAuthRetryDuration = time.Second * 10
|
2018-05-01 23:45:06 +00:00
|
|
|
)
|
|
|
|
|
2020-02-06 00:55:26 +00:00
|
|
|
// Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
|
|
|
|
// reconnects them if they disconnect.
|
2018-05-01 23:45:06 +00:00
|
|
|
type Supervisor struct {
|
2022-06-18 00:24:37 +00:00
|
|
|
config *TunnelConfig
|
|
|
|
orchestrator *orchestration.Orchestrator
|
|
|
|
edgeIPs *edgediscovery.Edge
|
2022-11-16 09:08:45 +00:00
|
|
|
edgeTunnelServer TunnelServer
|
2022-06-18 00:24:37 +00:00
|
|
|
tunnelErrors chan tunnelError
|
|
|
|
tunnelsConnecting map[int]chan struct{}
|
|
|
|
tunnelsProtocolFallback map[int]*protocolFallback
|
2018-05-01 23:45:06 +00:00
|
|
|
// nextConnectedIndex and nextConnectedSignal are used to wait for all
|
|
|
|
// currently-connecting tunnels to finish connecting so we can reset backoff timer
|
|
|
|
nextConnectedIndex int
|
|
|
|
nextConnectedSignal chan struct{}
|
2019-06-17 21:18:47 +00:00
|
|
|
|
2021-11-08 15:43:36 +00:00
|
|
|
log *ConnAwareLogger
|
2021-02-03 18:32:54 +00:00
|
|
|
logTransport *zerolog.Logger
|
2019-12-04 17:22:08 +00:00
|
|
|
|
2021-01-20 19:41:09 +00:00
|
|
|
reconnectCh chan ReconnectSignal
|
2021-02-05 00:07:49 +00:00
|
|
|
gracefulShutdownC <-chan struct{}
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
|
2021-02-05 00:07:49 +00:00
|
|
|
var errEarlyShutdown = errors.New("shutdown started")
|
|
|
|
|
2018-05-01 23:45:06 +00:00
|
|
|
type tunnelError struct {
|
|
|
|
index int
|
|
|
|
err error
|
|
|
|
}
|
|
|
|
|
2022-02-11 10:49:06 +00:00
|
|
|
func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrator, reconnectCh chan ReconnectSignal, gracefulShutdownC <-chan struct{}) (*Supervisor, error) {
|
2022-06-18 00:24:37 +00:00
|
|
|
isStaticEdge := len(config.EdgeAddrs) > 0
|
|
|
|
|
2023-02-06 17:13:05 +00:00
|
|
|
var err error
|
2021-01-20 19:41:09 +00:00
|
|
|
var edgeIPs *edgediscovery.Edge
|
2022-06-18 00:24:37 +00:00
|
|
|
if isStaticEdge { // static edge addresses
|
2020-11-25 06:55:13 +00:00
|
|
|
edgeIPs, err = edgediscovery.StaticEdge(config.Log, config.EdgeAddrs)
|
2019-12-24 05:11:00 +00:00
|
|
|
} else {
|
2022-05-20 21:51:36 +00:00
|
|
|
edgeIPs, err = edgediscovery.ResolveEdge(config.Log, config.Region, config.EdgeIPVersion)
|
2019-12-24 05:11:00 +00:00
|
|
|
}
|
2019-12-13 23:05:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-06-25 18:25:39 +00:00
|
|
|
|
2022-08-11 20:31:36 +00:00
|
|
|
tracker := tunnelstate.NewConnTracker(config.Log)
|
|
|
|
log := NewConnAwareLogger(config.Log, tracker, config.Observer)
|
2022-05-20 21:51:36 +00:00
|
|
|
|
2022-12-14 11:43:52 +00:00
|
|
|
edgeAddrHandler := NewIPAddrFallback(config.MaxEdgeAddrRetries)
|
2023-02-28 16:11:42 +00:00
|
|
|
edgeBindAddr := config.EdgeBindAddr
|
2022-05-20 21:51:36 +00:00
|
|
|
|
|
|
|
edgeTunnelServer := EdgeTunnelServer{
|
|
|
|
config: config,
|
|
|
|
orchestrator: orchestrator,
|
|
|
|
edgeAddrs: edgeIPs,
|
|
|
|
edgeAddrHandler: edgeAddrHandler,
|
2023-02-28 16:11:42 +00:00
|
|
|
edgeBindAddr: edgeBindAddr,
|
2022-08-11 20:31:36 +00:00
|
|
|
tracker: tracker,
|
2022-05-20 21:51:36 +00:00
|
|
|
reconnectCh: reconnectCh,
|
|
|
|
gracefulShutdownC: gracefulShutdownC,
|
|
|
|
connAwareLogger: log,
|
|
|
|
}
|
2022-09-20 08:01:35 +00:00
|
|
|
|
2018-05-01 23:45:06 +00:00
|
|
|
return &Supervisor{
|
2024-05-23 16:48:34 +00:00
|
|
|
config: config,
|
|
|
|
orchestrator: orchestrator,
|
|
|
|
edgeIPs: edgeIPs,
|
|
|
|
edgeTunnelServer: &edgeTunnelServer,
|
|
|
|
tunnelErrors: make(chan tunnelError),
|
|
|
|
tunnelsConnecting: map[int]chan struct{}{},
|
|
|
|
tunnelsProtocolFallback: map[int]*protocolFallback{},
|
|
|
|
log: log,
|
|
|
|
logTransport: config.LogTransport,
|
|
|
|
reconnectCh: reconnectCh,
|
|
|
|
gracefulShutdownC: gracefulShutdownC,
|
2019-12-13 23:05:21 +00:00
|
|
|
}, nil
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
|
2021-01-20 19:41:09 +00:00
|
|
|
func (s *Supervisor) Run(
|
|
|
|
ctx context.Context,
|
|
|
|
connectedSignal *signal.Signal,
|
|
|
|
) error {
|
2022-09-20 10:39:51 +00:00
|
|
|
if s.config.PacketConfig != nil {
|
2022-08-18 15:03:47 +00:00
|
|
|
go func() {
|
2022-09-20 10:39:51 +00:00
|
|
|
if err := s.config.PacketConfig.ICMPRouter.Serve(ctx); err != nil {
|
2022-09-19 11:36:25 +00:00
|
|
|
if errors.Is(err, net.ErrClosed) {
|
|
|
|
s.log.Logger().Info().Err(err).Msg("icmp router terminated")
|
|
|
|
} else {
|
|
|
|
s.log.Logger().Err(err).Msg("icmp router terminated")
|
|
|
|
}
|
2022-08-18 15:03:47 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2021-01-20 19:41:09 +00:00
|
|
|
if err := s.initialize(ctx, connectedSignal); err != nil {
|
2021-02-05 00:07:49 +00:00
|
|
|
if err == errEarlyShutdown {
|
|
|
|
return nil
|
|
|
|
}
|
2018-05-01 23:45:06 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
var tunnelsWaiting []int
|
2019-12-06 21:32:15 +00:00
|
|
|
tunnelsActive := s.config.HAConnections
|
|
|
|
|
2024-05-23 16:48:34 +00:00
|
|
|
backoff := retry.NewBackoff(s.config.Retries, tunnelRetryDuration, true)
|
2018-05-01 23:45:06 +00:00
|
|
|
var backoffTimer <-chan time.Time
|
|
|
|
|
2021-02-05 00:07:49 +00:00
|
|
|
shuttingDown := false
|
2018-05-01 23:45:06 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
// Context cancelled
|
|
|
|
case <-ctx.Done():
|
|
|
|
for tunnelsActive > 0 {
|
|
|
|
<-s.tunnelErrors
|
|
|
|
tunnelsActive--
|
|
|
|
}
|
|
|
|
return nil
|
2022-05-20 21:51:36 +00:00
|
|
|
// startTunnel completed with a response
|
2018-05-01 23:45:06 +00:00
|
|
|
// (note that this may also be caused by context cancellation)
|
|
|
|
case tunnelError := <-s.tunnelErrors:
|
|
|
|
tunnelsActive--
|
2021-02-05 00:07:49 +00:00
|
|
|
if tunnelError.err != nil && !shuttingDown {
|
2022-05-20 21:51:36 +00:00
|
|
|
switch tunnelError.err.(type) {
|
|
|
|
case ReconnectSignal:
|
|
|
|
// For tunnels that closed with reconnect signal, we reconnect immediately
|
|
|
|
go s.startTunnel(ctx, tunnelError.index, s.newConnectedTunnelSignal(tunnelError.index))
|
|
|
|
tunnelsActive++
|
|
|
|
continue
|
|
|
|
}
|
2022-06-18 00:24:37 +00:00
|
|
|
// Make sure we don't continue if there is no more fallback allowed
|
|
|
|
if _, retry := s.tunnelsProtocolFallback[tunnelError.index].GetMaxBackoffDuration(ctx); !retry {
|
|
|
|
continue
|
|
|
|
}
|
2021-11-08 15:43:36 +00:00
|
|
|
s.log.ConnAwareLogger().Err(tunnelError.err).Int(connection.LogFieldConnIndex, tunnelError.index).Msg("Connection terminated")
|
2018-05-01 23:45:06 +00:00
|
|
|
tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
|
|
|
|
s.waitForNextTunnel(tunnelError.index)
|
|
|
|
|
|
|
|
if backoffTimer == nil {
|
|
|
|
backoffTimer = backoff.BackoffTimer()
|
|
|
|
}
|
2021-01-20 19:41:09 +00:00
|
|
|
} else if tunnelsActive == 0 {
|
2022-05-20 21:51:36 +00:00
|
|
|
s.log.ConnAwareLogger().Msg("no more connections active and exiting")
|
|
|
|
// All connected tunnels exited gracefully, no more work to do
|
2021-01-20 19:41:09 +00:00
|
|
|
return nil
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
// Backoff was set and its timer expired
|
|
|
|
case <-backoffTimer:
|
|
|
|
backoffTimer = nil
|
|
|
|
for _, index := range tunnelsWaiting {
|
2021-01-20 19:41:09 +00:00
|
|
|
go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index))
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
tunnelsActive += len(tunnelsWaiting)
|
|
|
|
tunnelsWaiting = nil
|
|
|
|
// Tunnel successfully connected
|
|
|
|
case <-s.nextConnectedSignal:
|
|
|
|
if !s.waitForNextTunnel(s.nextConnectedIndex) && len(tunnelsWaiting) == 0 {
|
|
|
|
// No more tunnels outstanding, clear backoff timer
|
|
|
|
backoff.SetGracePeriod()
|
|
|
|
}
|
2021-02-05 00:07:49 +00:00
|
|
|
case <-s.gracefulShutdownC:
|
|
|
|
shuttingDown = true
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-06 21:32:15 +00:00
|
|
|
// Returns nil if initialization succeeded, else the initialization error.
|
2022-05-20 21:51:36 +00:00
|
|
|
// Attempts here will be made to connect one tunnel, if successful, it will
|
|
|
|
// connect the available tunnels up to config.HAConnections.
|
2021-01-20 19:41:09 +00:00
|
|
|
func (s *Supervisor) initialize(
|
|
|
|
ctx context.Context,
|
|
|
|
connectedSignal *signal.Signal,
|
|
|
|
) error {
|
|
|
|
availableAddrs := s.edgeIPs.AvailableAddrs()
|
2019-12-13 23:05:21 +00:00
|
|
|
if s.config.HAConnections > availableAddrs {
|
2021-11-08 15:43:36 +00:00
|
|
|
s.log.Logger().Info().Msgf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
|
2019-12-13 23:05:21 +00:00
|
|
|
s.config.HAConnections = availableAddrs
|
|
|
|
}
|
2022-06-18 00:24:37 +00:00
|
|
|
s.tunnelsProtocolFallback[0] = &protocolFallback{
|
2024-05-23 16:48:34 +00:00
|
|
|
retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
|
2022-06-18 00:24:37 +00:00
|
|
|
s.config.ProtocolSelector.Current(),
|
|
|
|
false,
|
|
|
|
}
|
2019-12-13 23:05:21 +00:00
|
|
|
|
2021-01-20 19:41:09 +00:00
|
|
|
go s.startFirstTunnel(ctx, connectedSignal)
|
2022-05-20 21:51:36 +00:00
|
|
|
|
|
|
|
// Wait for response from first tunnel before proceeding to attempt other HA edge tunnels
|
2018-05-01 23:45:06 +00:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
<-s.tunnelErrors
|
2019-12-04 17:22:08 +00:00
|
|
|
return ctx.Err()
|
2018-05-01 23:45:06 +00:00
|
|
|
case tunnelError := <-s.tunnelErrors:
|
|
|
|
return tunnelError.err
|
2021-02-05 00:07:49 +00:00
|
|
|
case <-s.gracefulShutdownC:
|
|
|
|
return errEarlyShutdown
|
2019-03-04 19:48:56 +00:00
|
|
|
case <-connectedSignal.Wait():
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
2022-05-20 21:51:36 +00:00
|
|
|
|
2018-05-01 23:45:06 +00:00
|
|
|
// At least one successful connection, so start the rest
|
|
|
|
for i := 1; i < s.config.HAConnections; i++ {
|
2022-06-18 00:24:37 +00:00
|
|
|
s.tunnelsProtocolFallback[i] = &protocolFallback{
|
2024-05-23 16:48:34 +00:00
|
|
|
retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
|
2022-11-16 09:08:45 +00:00
|
|
|
// Set the protocol we know the first tunnel connected with.
|
|
|
|
s.tunnelsProtocolFallback[0].protocol,
|
2022-06-18 00:24:37 +00:00
|
|
|
false,
|
|
|
|
}
|
2022-08-31 19:52:44 +00:00
|
|
|
go s.startTunnel(ctx, i, s.newConnectedTunnelSignal(i))
|
2018-05-01 23:45:06 +00:00
|
|
|
time.Sleep(registrationInterval)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// startTunnel starts the first tunnel connection. The resulting error will be sent on
|
|
|
|
// s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
|
2021-01-20 19:41:09 +00:00
|
|
|
func (s *Supervisor) startFirstTunnel(
|
|
|
|
ctx context.Context,
|
|
|
|
connectedSignal *signal.Signal,
|
|
|
|
) {
|
2019-12-13 23:05:21 +00:00
|
|
|
var (
|
2022-05-20 21:51:36 +00:00
|
|
|
err error
|
2019-12-13 23:05:21 +00:00
|
|
|
)
|
2020-06-25 18:25:39 +00:00
|
|
|
const firstConnIndex = 0
|
2022-06-18 00:24:37 +00:00
|
|
|
isStaticEdge := len(s.config.EdgeAddrs) > 0
|
2018-05-01 23:45:06 +00:00
|
|
|
defer func() {
|
2022-06-02 17:57:37 +00:00
|
|
|
s.tunnelErrors <- tunnelError{index: firstConnIndex, err: err}
|
2018-05-01 23:45:06 +00:00
|
|
|
}()
|
|
|
|
|
2020-02-06 00:55:26 +00:00
|
|
|
// If the first tunnel disconnects, keep restarting it.
|
2022-06-18 00:24:37 +00:00
|
|
|
for {
|
|
|
|
err = s.edgeTunnelServer.Serve(ctx, firstConnIndex, s.tunnelsProtocolFallback[firstConnIndex], connectedSignal)
|
2019-12-13 23:05:21 +00:00
|
|
|
if ctx.Err() != nil {
|
2018-05-01 23:45:06 +00:00
|
|
|
return
|
|
|
|
}
|
2022-05-20 21:51:36 +00:00
|
|
|
if err == nil {
|
2018-05-01 23:45:06 +00:00
|
|
|
return
|
|
|
|
}
|
2022-06-18 00:24:37 +00:00
|
|
|
// Make sure we don't continue if there is no more fallback allowed
|
|
|
|
if _, retry := s.tunnelsProtocolFallback[firstConnIndex].GetMaxBackoffDuration(ctx); !retry {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Try again for Unauthorized errors because we hope them to be
|
|
|
|
// transient due to edge propagation lag on new Tunnels.
|
|
|
|
if strings.Contains(err.Error(), "Unauthorized") {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
switch err.(type) {
|
|
|
|
case edgediscovery.ErrNoAddressesLeft:
|
|
|
|
// If your provided addresses are not available, we will keep trying regardless.
|
|
|
|
if !isStaticEdge {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
case connection.DupConnRegisterTunnelError,
|
|
|
|
*quic.IdleTimeoutError,
|
2023-04-26 11:58:01 +00:00
|
|
|
*quic.ApplicationError,
|
2022-06-18 00:24:37 +00:00
|
|
|
edgediscovery.DialError,
|
|
|
|
*connection.EdgeQuicDialError:
|
|
|
|
// Try again for these types of errors
|
|
|
|
default:
|
|
|
|
// Uncaught errors should bail startup
|
|
|
|
return
|
|
|
|
}
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// startTunnel starts a new tunnel connection. The resulting error will be sent on
|
2022-05-20 21:51:36 +00:00
|
|
|
// s.tunnelError as this is expected to run in a goroutine.
|
2021-01-20 19:41:09 +00:00
|
|
|
func (s *Supervisor) startTunnel(
|
|
|
|
ctx context.Context,
|
|
|
|
index int,
|
|
|
|
connectedSignal *signal.Signal,
|
|
|
|
) {
|
2019-12-13 23:05:21 +00:00
|
|
|
var (
|
2022-05-20 21:51:36 +00:00
|
|
|
err error
|
2019-12-13 23:05:21 +00:00
|
|
|
)
|
|
|
|
defer func() {
|
2022-06-02 17:57:37 +00:00
|
|
|
s.tunnelErrors <- tunnelError{index: index, err: err}
|
2019-12-13 23:05:21 +00:00
|
|
|
}()
|
|
|
|
|
2022-06-18 00:24:37 +00:00
|
|
|
err = s.edgeTunnelServer.Serve(ctx, uint8(index), s.tunnelsProtocolFallback[index], connectedSignal)
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
|
2019-03-04 19:48:56 +00:00
|
|
|
func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
|
|
|
|
sig := make(chan struct{})
|
|
|
|
s.tunnelsConnecting[index] = sig
|
|
|
|
s.nextConnectedSignal = sig
|
2018-05-01 23:45:06 +00:00
|
|
|
s.nextConnectedIndex = index
|
2019-03-04 19:48:56 +00:00
|
|
|
return signal.New(sig)
|
2018-05-01 23:45:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (s *Supervisor) waitForNextTunnel(index int) bool {
|
|
|
|
delete(s.tunnelsConnecting, index)
|
|
|
|
s.nextConnectedSignal = nil
|
|
|
|
for k, v := range s.tunnelsConnecting {
|
|
|
|
s.nextConnectedIndex = k
|
|
|
|
s.nextConnectedSignal = v
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-12-13 23:05:21 +00:00
|
|
|
func (s *Supervisor) unusedIPs() bool {
|
|
|
|
return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
|
2019-06-17 21:18:47 +00:00
|
|
|
}
|