Add new files missed from release 2017.11.3
This commit is contained in:
Chris Branch
parent
ee499dfa88
commit
d40eb85da6
|
|
@ -0,0 +1,82 @@
|
|||
package origin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
)
|
||||
|
||||
const (
|
||||
// Used to discover HA Warp servers
|
||||
srvService = "warp"
|
||||
srvProto = "tcp"
|
||||
srvName = "cloudflarewarp.com"
|
||||
)
|
||||
|
||||
func ResolveEdgeIPs(addresses []string) ([]*net.TCPAddr, error) {
|
||||
if len(addresses) > 0 {
|
||||
var tcpAddrs []*net.TCPAddr
|
||||
for _, address := range addresses {
|
||||
// Addresses specified (for testing, usually)
|
||||
tcpAddr, err := net.ResolveTCPAddr("tcp", address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tcpAddrs = append(tcpAddrs, tcpAddr)
|
||||
}
|
||||
return tcpAddrs, nil
|
||||
}
|
||||
// HA service discovery lookup
|
||||
_, addrs, err := net.LookupSRV(srvService, srvProto, srvName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resolvedIPsPerCNAME [][]*net.TCPAddr
|
||||
var lookupErr error
|
||||
for _, addr := range addrs {
|
||||
ips, err := ResolveSRVToTCP(addr)
|
||||
if err != nil || len(ips) == 0 {
|
||||
// don't return early, we might be able to resolve other addresses
|
||||
lookupErr = err
|
||||
continue
|
||||
}
|
||||
resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
|
||||
}
|
||||
ips := FlattenServiceIPs(resolvedIPsPerCNAME)
|
||||
if lookupErr == nil && len(ips) == 0 {
|
||||
return nil, fmt.Errorf("Unknown service discovery error")
|
||||
}
|
||||
return ips, lookupErr
|
||||
}
|
||||
|
||||
func ResolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
|
||||
ips, err := net.LookupIP(srv.Target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addrs := make([]*net.TCPAddr, len(ips))
|
||||
for i, ip := range ips {
|
||||
addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
|
||||
}
|
||||
return addrs, nil
|
||||
}
|
||||
|
||||
// FlattenServiceIPs transposes and flattens the input slices such that the
|
||||
// first element of the n inner slices are the first n elements of the result.
|
||||
func FlattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
|
||||
var result []*net.TCPAddr
|
||||
for len(ipsByService) > 0 {
|
||||
filtered := ipsByService[:0]
|
||||
for _, ips := range ipsByService {
|
||||
if len(ips) == 0 {
|
||||
// sanity check
|
||||
continue
|
||||
}
|
||||
result = append(result, ips[0])
|
||||
if len(ips) > 1 {
|
||||
filtered = append(filtered, ips[1:])
|
||||
}
|
||||
}
|
||||
ipsByService = filtered
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package origin
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFlattenServiceIPs(t *testing.T) {
|
||||
result := FlattenServiceIPs([][]*net.TCPAddr{
|
||||
[]*net.TCPAddr{
|
||||
&net.TCPAddr{Port: 1},
|
||||
&net.TCPAddr{Port: 2},
|
||||
&net.TCPAddr{Port: 3},
|
||||
&net.TCPAddr{Port: 4},
|
||||
},
|
||||
[]*net.TCPAddr{
|
||||
&net.TCPAddr{Port: 10},
|
||||
&net.TCPAddr{Port: 12},
|
||||
&net.TCPAddr{Port: 13},
|
||||
},
|
||||
[]*net.TCPAddr{
|
||||
&net.TCPAddr{Port: 21},
|
||||
&net.TCPAddr{Port: 22},
|
||||
&net.TCPAddr{Port: 23},
|
||||
&net.TCPAddr{Port: 24},
|
||||
&net.TCPAddr{Port: 25},
|
||||
},
|
||||
})
|
||||
assert.EqualValues(t, []*net.TCPAddr{
|
||||
&net.TCPAddr{Port: 1},
|
||||
&net.TCPAddr{Port: 10},
|
||||
&net.TCPAddr{Port: 21},
|
||||
&net.TCPAddr{Port: 2},
|
||||
&net.TCPAddr{Port: 12},
|
||||
&net.TCPAddr{Port: 22},
|
||||
&net.TCPAddr{Port: 3},
|
||||
&net.TCPAddr{Port: 13},
|
||||
&net.TCPAddr{Port: 23},
|
||||
&net.TCPAddr{Port: 4},
|
||||
&net.TCPAddr{Port: 24},
|
||||
&net.TCPAddr{Port: 25},
|
||||
}, result)
|
||||
}
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
package origin
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/cloudflare/cloudflare-warp/h2mux"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type TunnelMetrics struct {
|
||||
totalRequests prometheus.Counter
|
||||
requestsPerTunnel *prometheus.CounterVec
|
||||
// concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
|
||||
concurrentRequestsLock sync.Mutex
|
||||
concurrentRequestsPerTunnel *prometheus.GaugeVec
|
||||
// concurrentRequests records count of concurrent requests for each tunnel
|
||||
concurrentRequests map[string]uint64
|
||||
maxConcurrentRequestsPerTunnel *prometheus.GaugeVec
|
||||
// concurrentRequests records max count of concurrent requests for each tunnel
|
||||
maxConcurrentRequests map[string]uint64
|
||||
rtt prometheus.Gauge
|
||||
rttMin prometheus.Gauge
|
||||
rttMax prometheus.Gauge
|
||||
timerRetries prometheus.Gauge
|
||||
receiveWindowSizeAve prometheus.Gauge
|
||||
sendWindowSizeAve prometheus.Gauge
|
||||
receiveWindowSizeMin prometheus.Gauge
|
||||
receiveWindowSizeMax prometheus.Gauge
|
||||
sendWindowSizeMin prometheus.Gauge
|
||||
sendWindowSizeMax prometheus.Gauge
|
||||
responseByCode *prometheus.CounterVec
|
||||
responseCodePerTunnel *prometheus.CounterVec
|
||||
serverLocations *prometheus.GaugeVec
|
||||
// locationLock is a mutex for oldServerLocations
|
||||
locationLock sync.Mutex
|
||||
// oldServerLocations stores the last server the tunnel was connected to
|
||||
oldServerLocations map[string]string
|
||||
}
|
||||
|
||||
// Metrics that can be collected without asking the edge
|
||||
func NewTunnelMetrics() *TunnelMetrics {
|
||||
totalRequests := prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "total_requests",
|
||||
Help: "Amount of requests proxied through all the tunnels",
|
||||
})
|
||||
prometheus.MustRegister(totalRequests)
|
||||
|
||||
requestsPerTunnel := prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "requests_per_tunnel",
|
||||
Help: "Amount of requests proxied through each tunnel",
|
||||
},
|
||||
[]string{"connection_id"},
|
||||
)
|
||||
prometheus.MustRegister(requestsPerTunnel)
|
||||
|
||||
concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "concurrent_requests_per_tunnel",
|
||||
Help: "Concurrent requests proxied through each tunnel",
|
||||
},
|
||||
[]string{"connection_id"},
|
||||
)
|
||||
prometheus.MustRegister(concurrentRequestsPerTunnel)
|
||||
|
||||
maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "max_concurrent_requests_per_tunnel",
|
||||
Help: "Largest number of concurrent requests proxied through each tunnel so far",
|
||||
},
|
||||
[]string{"connection_id"},
|
||||
)
|
||||
prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
|
||||
|
||||
rtt := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "rtt",
|
||||
Help: "Round-trip time",
|
||||
})
|
||||
prometheus.MustRegister(rtt)
|
||||
|
||||
rttMin := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "rtt_min",
|
||||
Help: "Shortest round-trip time",
|
||||
})
|
||||
prometheus.MustRegister(rttMin)
|
||||
|
||||
rttMax := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "rtt_max",
|
||||
Help: "Longest round-trip time",
|
||||
})
|
||||
prometheus.MustRegister(rttMax)
|
||||
|
||||
timerRetries := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "timer_retries",
|
||||
Help: "Unacknowledged heart beats count",
|
||||
})
|
||||
prometheus.MustRegister(timerRetries)
|
||||
|
||||
receiveWindowSizeAve := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "receive_window_ave",
|
||||
Help: "Average receive window size",
|
||||
})
|
||||
prometheus.MustRegister(receiveWindowSizeAve)
|
||||
|
||||
sendWindowSizeAve := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "send_window_ave",
|
||||
Help: "Average send window size",
|
||||
})
|
||||
prometheus.MustRegister(sendWindowSizeAve)
|
||||
|
||||
receiveWindowSizeMin := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "receive_window_min",
|
||||
Help: "Smallest receive window size",
|
||||
})
|
||||
prometheus.MustRegister(receiveWindowSizeMin)
|
||||
|
||||
receiveWindowSizeMax := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "receive_window_max",
|
||||
Help: "Largest receive window size",
|
||||
})
|
||||
prometheus.MustRegister(receiveWindowSizeMax)
|
||||
|
||||
sendWindowSizeMin := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "send_window_min",
|
||||
Help: "Smallest send window size",
|
||||
})
|
||||
prometheus.MustRegister(sendWindowSizeMin)
|
||||
|
||||
sendWindowSizeMax := prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "send_window_max",
|
||||
Help: "Largest send window size",
|
||||
})
|
||||
prometheus.MustRegister(sendWindowSizeMax)
|
||||
|
||||
responseByCode := prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "response_by_code",
|
||||
Help: "Count of responses by HTTP status code",
|
||||
},
|
||||
[]string{"status_code"},
|
||||
)
|
||||
prometheus.MustRegister(responseByCode)
|
||||
|
||||
responseCodePerTunnel := prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "response_code_per_tunnel",
|
||||
Help: "Count of responses by HTTP status code fore each tunnel",
|
||||
},
|
||||
[]string{"connection_id", "status_code"},
|
||||
)
|
||||
prometheus.MustRegister(responseCodePerTunnel)
|
||||
|
||||
serverLocations := prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "server_locations",
|
||||
Help: "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
|
||||
},
|
||||
[]string{"connection_id", "location"},
|
||||
)
|
||||
prometheus.MustRegister(serverLocations)
|
||||
|
||||
return &TunnelMetrics{
|
||||
totalRequests: totalRequests,
|
||||
requestsPerTunnel: requestsPerTunnel,
|
||||
concurrentRequestsPerTunnel: concurrentRequestsPerTunnel,
|
||||
concurrentRequests: make(map[string]uint64),
|
||||
maxConcurrentRequestsPerTunnel: maxConcurrentRequestsPerTunnel,
|
||||
maxConcurrentRequests: make(map[string]uint64),
|
||||
rtt: rtt,
|
||||
rttMin: rttMin,
|
||||
rttMax: rttMax,
|
||||
timerRetries: timerRetries,
|
||||
receiveWindowSizeAve: receiveWindowSizeAve,
|
||||
sendWindowSizeAve: sendWindowSizeAve,
|
||||
receiveWindowSizeMin: receiveWindowSizeMin,
|
||||
receiveWindowSizeMax: receiveWindowSizeMax,
|
||||
sendWindowSizeMin: sendWindowSizeMin,
|
||||
sendWindowSizeMax: sendWindowSizeMax,
|
||||
responseByCode: responseByCode,
|
||||
responseCodePerTunnel: responseCodePerTunnel,
|
||||
serverLocations: serverLocations,
|
||||
oldServerLocations: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TunnelMetrics) updateTunnelFlowControlMetrics(metrics *h2mux.FlowControlMetrics) {
|
||||
t.receiveWindowSizeAve.Set(float64(metrics.AverageReceiveWindowSize))
|
||||
t.sendWindowSizeAve.Set(float64(metrics.AverageSendWindowSize))
|
||||
t.receiveWindowSizeMin.Set(float64(metrics.MinReceiveWindowSize))
|
||||
t.receiveWindowSizeMax.Set(float64(metrics.MaxReceiveWindowSize))
|
||||
t.sendWindowSizeMin.Set(float64(metrics.MinSendWindowSize))
|
||||
t.sendWindowSizeMax.Set(float64(metrics.MaxSendWindowSize))
|
||||
}
|
||||
|
||||
func (t *TunnelMetrics) incrementRequests(connectionID string) {
|
||||
t.concurrentRequestsLock.Lock()
|
||||
var concurrentRequests uint64
|
||||
var ok bool
|
||||
if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
|
||||
t.concurrentRequests[connectionID] += 1
|
||||
concurrentRequests++
|
||||
} else {
|
||||
t.concurrentRequests[connectionID] = 1
|
||||
concurrentRequests = 1
|
||||
}
|
||||
if maxConcurrentRequests, ok := t.maxConcurrentRequests[connectionID]; (ok && maxConcurrentRequests < concurrentRequests) || !ok {
|
||||
t.maxConcurrentRequests[connectionID] = concurrentRequests
|
||||
t.maxConcurrentRequestsPerTunnel.WithLabelValues(connectionID).Set(float64(concurrentRequests))
|
||||
}
|
||||
t.concurrentRequestsLock.Unlock()
|
||||
|
||||
t.totalRequests.Inc()
|
||||
t.requestsPerTunnel.WithLabelValues(connectionID).Inc()
|
||||
t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Inc()
|
||||
}
|
||||
|
||||
func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
|
||||
t.concurrentRequestsLock.Lock()
|
||||
if _, ok := t.concurrentRequests[connectionID]; ok {
|
||||
t.concurrentRequests[connectionID] -= 1
|
||||
} else {
|
||||
log.Error("Concurrent requests per tunnel metrics went wrong; you can't decrement concurrent requests count without increment it first.")
|
||||
}
|
||||
t.concurrentRequestsLock.Unlock()
|
||||
|
||||
t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Dec()
|
||||
}
|
||||
|
||||
func (t *TunnelMetrics) incrementResponses(connectionID, code string) {
|
||||
t.responseByCode.WithLabelValues(code).Inc()
|
||||
t.responseCodePerTunnel.WithLabelValues(connectionID, code).Inc()
|
||||
|
||||
}
|
||||
|
||||
func (t *TunnelMetrics) registerServerLocation(connectionID, loc string) {
|
||||
t.locationLock.Lock()
|
||||
defer t.locationLock.Unlock()
|
||||
if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
|
||||
return
|
||||
} else if ok {
|
||||
t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
|
||||
}
|
||||
t.serverLocations.WithLabelValues(connectionID, loc).Inc()
|
||||
t.oldServerLocations[connectionID] = loc
|
||||
}
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
package origin
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// can only be called once
|
||||
var m = NewTunnelMetrics()
|
||||
|
||||
func TestConcurrentRequestsSingleTunnel(t *testing.T) {
|
||||
routines := 20
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(routines)
|
||||
for i := 0; i < routines; i++ {
|
||||
go func() {
|
||||
m.incrementRequests("0")
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
assert.Len(t, m.concurrentRequests, 1)
|
||||
assert.Equal(t, uint64(routines), m.concurrentRequests["0"])
|
||||
assert.Len(t, m.maxConcurrentRequests, 1)
|
||||
assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
|
||||
|
||||
wg.Add(routines / 2)
|
||||
for i := 0; i < routines/2; i++ {
|
||||
go func() {
|
||||
m.decrementConcurrentRequests("0")
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
assert.Equal(t, uint64(routines-routines/2), m.concurrentRequests["0"])
|
||||
assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
|
||||
}
|
||||
|
||||
func TestConcurrentRequestsMultiTunnel(t *testing.T) {
|
||||
m.concurrentRequests = make(map[string]uint64)
|
||||
m.maxConcurrentRequests = make(map[string]uint64)
|
||||
tunnels := 20
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(tunnels)
|
||||
for i := 0; i < tunnels; i++ {
|
||||
go func(i int) {
|
||||
// if we have j < i, then tunnel 0 won't have a chance to call incrementRequests
|
||||
for j := 0; j < i+1; j++ {
|
||||
id := strconv.Itoa(i)
|
||||
m.incrementRequests(id)
|
||||
}
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
assert.Len(t, m.concurrentRequests, tunnels)
|
||||
assert.Len(t, m.maxConcurrentRequests, tunnels)
|
||||
for i := 0; i < tunnels; i++ {
|
||||
id := strconv.Itoa(i)
|
||||
assert.Equal(t, uint64(i+1), m.concurrentRequests[id])
|
||||
assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
|
||||
}
|
||||
|
||||
wg.Add(tunnels)
|
||||
for i := 0; i < tunnels; i++ {
|
||||
go func(i int) {
|
||||
for j := 0; j < i+1; j++ {
|
||||
id := strconv.Itoa(i)
|
||||
m.decrementConcurrentRequests(id)
|
||||
}
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
assert.Len(t, m.concurrentRequests, tunnels)
|
||||
assert.Len(t, m.maxConcurrentRequests, tunnels)
|
||||
for i := 0; i < tunnels; i++ {
|
||||
id := strconv.Itoa(i)
|
||||
assert.Equal(t, uint64(0), m.concurrentRequests[id])
|
||||
assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestRegisterServerLocation(t *testing.T) {
|
||||
tunnels := 20
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(tunnels)
|
||||
for i := 0; i < tunnels; i++ {
|
||||
go func(i int) {
|
||||
id := strconv.Itoa(i)
|
||||
m.registerServerLocation(id, "LHR")
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
for i := 0; i < tunnels; i++ {
|
||||
id := strconv.Itoa(i)
|
||||
assert.Equal(t, "LHR", m.oldServerLocations[id])
|
||||
}
|
||||
|
||||
wg.Add(tunnels)
|
||||
for i := 0; i < tunnels; i++ {
|
||||
go func(i int) {
|
||||
id := strconv.Itoa(i)
|
||||
m.registerServerLocation(id, "AUS")
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
for i := 0; i < tunnels; i++ {
|
||||
id := strconv.Itoa(i)
|
||||
assert.Equal(t, "AUS", m.oldServerLocations[id])
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
package origin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const (
|
||||
// Waiting time before retrying a failed tunnel connection
|
||||
tunnelRetryDuration = time.Minute
|
||||
// Limit on the exponential backoff time period. (2^5 = 32 minutes)
|
||||
tunnelRetryLimit = 5
|
||||
// SRV record resolution TTL
|
||||
resolveTTL = time.Hour
|
||||
)
|
||||
|
||||
type Supervisor struct {
|
||||
config *TunnelConfig
|
||||
edgeIPs []*net.TCPAddr
|
||||
lastResolve time.Time
|
||||
resolverC chan resolveResult
|
||||
tunnelErrors chan tunnelError
|
||||
tunnelsConnecting map[int]chan struct{}
|
||||
nextConnectedIndex int
|
||||
nextConnectedSignal chan struct{}
|
||||
}
|
||||
|
||||
type resolveResult struct {
|
||||
edgeIPs []*net.TCPAddr
|
||||
err error
|
||||
}
|
||||
|
||||
type tunnelError struct {
|
||||
index int
|
||||
err error
|
||||
}
|
||||
|
||||
func NewSupervisor(config *TunnelConfig) *Supervisor {
|
||||
return &Supervisor{
|
||||
config: config,
|
||||
tunnelErrors: make(chan tunnelError),
|
||||
tunnelsConnecting: map[int]chan struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Supervisor) Run(ctx context.Context, connectedSignal chan struct{}) error {
|
||||
if err := s.initialize(ctx, connectedSignal); err != nil {
|
||||
return err
|
||||
}
|
||||
tunnelsActive := s.config.HAConnections
|
||||
tunnelsWaiting := []int{}
|
||||
backoff := BackoffHandler{MaxRetries: tunnelRetryLimit, BaseTime: tunnelRetryDuration, RetryForever: true}
|
||||
var backoffTimer <-chan time.Time
|
||||
for tunnelsActive > 0 {
|
||||
select {
|
||||
// Context cancelled
|
||||
case <-ctx.Done():
|
||||
for tunnelsActive > 0 {
|
||||
<-s.tunnelErrors
|
||||
tunnelsActive--
|
||||
}
|
||||
return nil
|
||||
// startTunnel returned with error
|
||||
// (note that this may also be caused by context cancellation)
|
||||
case tunnelError := <-s.tunnelErrors:
|
||||
tunnelsActive--
|
||||
if tunnelError.err != nil {
|
||||
log.WithError(tunnelError.err).Warn("Tunnel disconnected due to error")
|
||||
tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
|
||||
s.waitForNextTunnel(tunnelError.index)
|
||||
if backoffTimer != nil {
|
||||
backoffTimer = backoff.BackoffTimer()
|
||||
}
|
||||
s.refreshEdgeIPs()
|
||||
}
|
||||
// Backoff was set and its timer expired
|
||||
case <-backoffTimer:
|
||||
backoffTimer = nil
|
||||
for _, index := range tunnelsWaiting {
|
||||
go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index))
|
||||
}
|
||||
tunnelsActive += len(tunnelsWaiting)
|
||||
tunnelsWaiting = nil
|
||||
// Tunnel successfully connected
|
||||
case <-s.nextConnectedSignal:
|
||||
if !s.waitForNextTunnel(s.nextConnectedIndex) && len(tunnelsWaiting) == 0 {
|
||||
// No more tunnels outstanding, clear backoff timer
|
||||
backoff.SetGracePeriod()
|
||||
}
|
||||
// DNS resolution returned
|
||||
case result := <-s.resolverC:
|
||||
s.lastResolve = time.Now()
|
||||
s.resolverC = nil
|
||||
if result.err == nil {
|
||||
log.Debug("Service discovery refresh complete")
|
||||
s.edgeIPs = result.edgeIPs
|
||||
} else {
|
||||
log.WithError(result.err).Error("Service discovery error")
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("All tunnels terminated")
|
||||
}
|
||||
|
||||
func (s *Supervisor) initialize(ctx context.Context, connectedSignal chan struct{}) error {
|
||||
edgeIPs, err := ResolveEdgeIPs(s.config.EdgeAddrs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.edgeIPs = edgeIPs
|
||||
s.lastResolve = time.Now()
|
||||
go s.startTunnel(ctx, 0, connectedSignal)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
<-s.tunnelErrors
|
||||
return nil
|
||||
case tunnelError := <-s.tunnelErrors:
|
||||
return tunnelError.err
|
||||
case <-connectedSignal:
|
||||
}
|
||||
// At least one successful connection, so start the rest
|
||||
for i := 1; i < s.config.HAConnections; i++ {
|
||||
go s.startTunnel(ctx, i, make(chan struct{}))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// startTunnel starts a new tunnel connection. The resulting error will be sent on
|
||||
// s.tunnelErrors.
|
||||
func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal chan struct{}) {
|
||||
err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(index), uint8(index), connectedSignal)
|
||||
s.tunnelErrors <- tunnelError{index: index, err: err}
|
||||
}
|
||||
|
||||
func (s *Supervisor) newConnectedTunnelSignal(index int) chan struct{} {
|
||||
signal := make(chan struct{})
|
||||
s.tunnelsConnecting[index] = signal
|
||||
s.nextConnectedSignal = signal
|
||||
s.nextConnectedIndex = index
|
||||
return signal
|
||||
}
|
||||
|
||||
func (s *Supervisor) waitForNextTunnel(index int) bool {
|
||||
delete(s.tunnelsConnecting, index)
|
||||
s.nextConnectedSignal = nil
|
||||
for k, v := range s.tunnelsConnecting {
|
||||
s.nextConnectedIndex = k
|
||||
s.nextConnectedSignal = v
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr {
|
||||
return s.edgeIPs[index%len(s.edgeIPs)]
|
||||
}
|
||||
|
||||
func (s *Supervisor) refreshEdgeIPs() {
|
||||
if s.resolverC != nil {
|
||||
return
|
||||
}
|
||||
if time.Since(s.lastResolve) < resolveTTL {
|
||||
return
|
||||
}
|
||||
s.resolverC = make(chan resolveResult)
|
||||
go func() {
|
||||
edgeIPs, err := ResolveEdgeIPs(s.config.EdgeAddrs)
|
||||
s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err}
|
||||
}()
|
||||
}
|
||||
Loading…
Reference in New Issue