From 7e31b77646c29937d4c36c8374da4faad0bfb2ae Mon Sep 17 00:00:00 2001 From: Nick Vollmar Date: Fri, 13 Dec 2019 17:05:21 -0600 Subject: [PATCH] TUN-2637: Manage edge IPs in a region-aware manner --- connection/connection.go | 5 +- connection/discovery.go | 356 +++++++++++++++++++++++++---------- connection/discovery_test.go | 342 ++++++++++++++++++++++++++++----- connection/manager.go | 15 +- connection/mocks_for_test.go | 118 ++++++++++++ go.mod | 5 +- go.sum | 235 +++++++++++++++++++++++ origin/supervisor.go | 115 +++++------ origin/supervisor_test.go | 20 +- origin/tunnel.go | 6 +- 10 files changed, 1011 insertions(+), 206 deletions(-) create mode 100644 connection/mocks_for_test.go diff --git a/connection/connection.go b/connection/connection.go index 7e032745..ab318d25 100644 --- a/connection/connection.go +++ b/connection/connection.go @@ -2,6 +2,7 @@ package connection import ( "context" + "net" "time" "github.com/google/uuid" @@ -19,9 +20,10 @@ const ( type Connection struct { id uuid.UUID muxer *h2mux.Muxer + addr *net.TCPAddr } -func newConnection(muxer *h2mux.Muxer) (*Connection, error) { +func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) { id, err := uuid.NewRandom() if err != nil { return nil, err @@ -29,6 +31,7 @@ func newConnection(muxer *h2mux.Muxer) (*Connection, error) { return &Connection{ id: id, muxer: muxer, + addr: addr, }, nil } diff --git a/connection/discovery.go b/connection/discovery.go index 8a9ac3fa..18c6cad5 100644 --- a/connection/discovery.go +++ b/connection/discovery.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "fmt" + "math/rand" "net" "sync" "time" @@ -26,8 +27,28 @@ const ( // SRV record resolution TTL resolveEdgeAddrTTL = 1 * time.Hour + + subsystemEdgeAddrResolver = "edgeAddrResolver" ) +// Redeclare network functions so they can be overridden in tests. +var ( + netLookupSRV = net.LookupSRV + netLookupIP = net.LookupIP +) + +// If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly. +// +// Note: Instead of DoT, we could also have used DoH. Either of these: +// - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv) +// - indirectly via `tunneldns.NewUpstreamHTTPS()` +// But both of these cases miss out on a key feature from the stdlib: +// "The returned records are sorted by priority and randomized by weight within a priority." +// (https://golang.org/pkg/net/#Resolver.LookupSRV) +// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it. +// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552 +var fallbackLookupSRV = lookupSRVWithDOT + var friendlyDNSErrorLines = []string{ `Please try the following things to diagnose this issue:`, ` 1. ensure that argotunnel.com is returning "origintunneld" service records.`, @@ -40,11 +61,26 @@ var friendlyDNSErrorLines = []string{ // EdgeServiceDiscoverer is an interface for looking up Cloudflare's edge network addresses type EdgeServiceDiscoverer interface { - // Addr returns an address to connect to cloudflare's edge network - Addr() *net.TCPAddr - // AvailableAddrs returns the number of unique addresses - AvailableAddrs() uint8 - // Refresh rediscover Cloudflare's edge network addresses + // Addr returns an unused address to connect to cloudflare's edge network. + // Before this method returns, the address will be removed from the pool of available addresses, + // so the caller can assume they have exclusive access to the address for tunneling purposes. + // The caller should remember to put it back via ReplaceAddr or MarkAddrBad. + Addr() (*net.TCPAddr, error) + // AnyAddr returns an address to connect to cloudflare's edge network. + // It may or may not be in active use for a tunnel. + // The caller should NOT return it via ReplaceAddr or MarkAddrBad! + AnyAddr() (*net.TCPAddr, error) + // ReplaceAddr is called when the address is no longer needed, e.g. due to a scaling-down of numHAConnections. + // It returns the address to the pool of available addresses. + ReplaceAddr(addr *net.TCPAddr) + // MarkAddrBad is called when there was a connectivity error for the address. + // It marks the address as unused but doesn't return it to the pool of available addresses. + MarkAddrBad(addr *net.TCPAddr) + // AvailableAddrs returns the number of addresses available for use + // (less those that have been marked bad). + AvailableAddrs() int + // Refresh rediscovers Cloudflare's edge network addresses. + // It resets the state of "bad" addresses but not those in active use. Refresh() error } @@ -52,16 +88,31 @@ type EdgeServiceDiscoverer interface { // It implements EdgeServiceDiscoverer interface type EdgeAddrResolver struct { sync.Mutex - // Addrs to connect to cloudflare's edge network + // HA regions + regions []*region + // Logger for noteworthy events + logger *logrus.Entry +} + +type region struct { + // Addresses that we expect will be in active use addrs []*net.TCPAddr - // index of the next element to use in addrs - nextAddrIndex int - logger *logrus.Entry + // Addresses that are in active use. + // This is actually a set of net.TCPAddr's, but we can't make a map like + // map[net.TCPAddr]bool + // since net.TCPAddr contains a field of type net.IP and therefore it cannot be used as a map key. + // So instead we use map[string]*net.TCPAddr, where the keys are obtained by net.TCPAddr.String(). + // (We keep the "raw" *net.TCPAddr values for the convenience of AnyAddr(). If that method didn't + // exist, we wouldn't strictly need the values, and this could be a map[string]bool.) + inUse map[string]*net.TCPAddr + // Addresses that were discarded due to a network error. + // Not sure what we'll do with these, but it feels good to keep them around for now. + bad []*net.TCPAddr } func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) { r := &EdgeAddrResolver{ - logger: logger.WithField("subsystem", " edgeAddrResolver"), + logger: logger.WithField("subsystem", subsystemEdgeAddrResolver), } if err := r.Refresh(); err != nil { return nil, err @@ -69,83 +120,153 @@ func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) { return r, nil } -func (r *EdgeAddrResolver) Addr() *net.TCPAddr { +func (r *EdgeAddrResolver) Addr() (*net.TCPAddr, error) { r.Lock() defer r.Unlock() - addr := r.addrs[r.nextAddrIndex] - r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs) - return addr + + // compute the largest region based on len(addrs) + var largestRegion *region + { + if len(r.regions) == 0 { + return nil, errors.New("No HA regions") + } + largestRegion = r.regions[0] + for _, region := range r.regions[1:] { + if len(region.addrs) > len(largestRegion.addrs) { + largestRegion = region + } + } + if len(largestRegion.addrs) == 0 { + return nil, errors.New("No IP address to claim") + } + } + + var addr *net.TCPAddr + addr, largestRegion.addrs = popAddr(largestRegion.addrs) + largestRegion.inUse[addr.String()] = addr + return addr, nil } -func (r *EdgeAddrResolver) AvailableAddrs() uint8 { +func (r *EdgeAddrResolver) AnyAddr() (*net.TCPAddr, error) { r.Lock() defer r.Unlock() - return uint8(len(r.addrs)) + for _, region := range r.regions { + // return an unused addr + if len(region.addrs) > 0 { + return region.addrs[rand.Intn(len(region.addrs))], nil + } + // return an addr that's in use + for _, addr := range region.inUse { + return addr, nil + } + } + return nil, fmt.Errorf("No IP addresses") +} + +func (r *EdgeAddrResolver) ReplaceAddr(addr *net.TCPAddr) { + r.Lock() + defer r.Unlock() + addrString := addr.String() + for _, region := range r.regions { + if _, ok := region.inUse[addrString]; ok { + delete(region.inUse, addrString) + region.addrs = append(region.addrs, addr) + break + } + } +} + +func (r *EdgeAddrResolver) MarkAddrBad(addr *net.TCPAddr) { + r.Lock() + defer r.Unlock() + addrString := addr.String() + for _, region := range r.regions { + if _, ok := region.inUse[addrString]; ok { + delete(region.inUse, addrString) + region.bad = append(region.bad, addr) + break + } + } +} + +func (r *EdgeAddrResolver) AvailableAddrs() int { + r.Lock() + defer r.Unlock() + result := 0 + for _, region := range r.regions { + result += len(region.addrs) + } + return result } func (r *EdgeAddrResolver) Refresh() error { - newAddrs, err := EdgeDiscovery(r.logger) + addrLists, err := EdgeDiscovery(r.logger) if err != nil { return err } + r.Lock() defer r.Unlock() - r.addrs = newAddrs - r.nextAddrIndex = 0 + inUse := allInUse(r.regions) + r.regions = makeHARegions(addrLists, inUse) return nil } -// HA service discovery lookup -func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) { - _, addrs, err := net.LookupSRV(srvService, srvProto, srvName) +// EdgeDiscovery implements HA service discovery lookup. +func EdgeDiscovery(logger *logrus.Entry) ([][]*net.TCPAddr, error) { + _, addrs, err := netLookupSRV(srvService, srvProto, srvName) if err != nil { - // Try to fall back to DoT from Cloudflare directly. - // - // Note: Instead of DoT, we could also have used DoH. Either of these: - // - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv) - // - indirectly via `tunneldns.NewUpstreamHTTPS()` - // But both of these cases miss out on a key feature from the stdlib: - // "The returned records are sorted by priority and randomized by weight within a priority." - // (https://golang.org/pkg/net/#Resolver.LookupSRV) - // Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it. - // See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552 - r := fallbackResolver(dotServerName, dotServerAddr) - ctx, cancel := context.WithTimeout(context.Background(), dotTimeout) - defer cancel() - _, fallbackAddrs, fallbackErr := r.LookupSRV(ctx, srvService, srvProto, srvName) + _, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName) if fallbackErr != nil || len(fallbackAddrs) == 0 { // use the original DNS error `err` in messages, not `fallbackErr` logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err) for _, s := range friendlyDNSErrorLines { logger.Errorln(s) } - return nil, errors.Wrap(err, "Could not lookup srv records on _origintunneld._tcp.argotunnel.com") + return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName) } // Accept the fallback results and keep going addrs = fallbackAddrs } + var resolvedIPsPerCNAME [][]*net.TCPAddr - var lookupErr error for _, addr := range addrs { ips, err := resolveSRVToTCP(addr) - if err != nil || len(ips) == 0 { - // don't return early, we might be able to resolve other addresses - lookupErr = err - continue + if err != nil { + return nil, err } resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips) } - ips := flattenServiceIPs(resolvedIPsPerCNAME) - if lookupErr == nil && len(ips) == 0 { - return nil, fmt.Errorf("Unknown service discovery error") + + return resolvedIPsPerCNAME, nil +} + +func lookupSRVWithDOT(service, proto, name string) (cname string, addrs []*net.SRV, err error) { + // Inspiration: https://github.com/artyom/dot/blob/master/dot.go + r := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) { + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", dotServerAddr) + if err != nil { + return nil, err + } + tlsConfig := &tls.Config{ServerName: dotServerName} + return tls.Client(conn, tlsConfig), nil + }, } - return ips, lookupErr + ctx, cancel := context.WithTimeout(context.Background(), dotTimeout) + defer cancel() + return r.LookupSRV(ctx, srvService, srvProto, srvName) } func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) { - ips, err := net.LookupIP(srv.Target) + ips, err := netLookupIP(srv.Target) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "Couldn't resolve SRV record %v", srv) + } + if len(ips) == 0 { + return nil, fmt.Errorf("SRV record %v had no IPs", srv) } addrs := make([]*net.TCPAddr, len(ips)) for i, ip := range ips { @@ -154,43 +275,6 @@ func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) { return addrs, nil } -// FlattenServiceIPs transposes and flattens the input slices such that the -// first element of the n inner slices are the first n elements of the result. -func flattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr { - var result []*net.TCPAddr - for len(ipsByService) > 0 { - filtered := ipsByService[:0] - for _, ips := range ipsByService { - if len(ips) == 0 { - // sanity check - continue - } - result = append(result, ips[0]) - if len(ips) > 1 { - filtered = append(filtered, ips[1:]) - } - } - ipsByService = filtered - } - return result -} - -// Inspiration: https://github.com/artyom/dot/blob/master/dot.go -func fallbackResolver(serverName, serverAddress string) *net.Resolver { - return &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) { - var dialer net.Dialer - conn, err := dialer.DialContext(ctx, "tcp", serverAddress) - if err != nil { - return nil, err - } - tlsConfig := &tls.Config{ServerName: serverName} - return tls.Client(conn, tlsConfig), nil - }, - } -} - // EdgeHostnameResolver discovers the addresses of Cloudflare's edge network via a list of server hostnames. // It implements EdgeServiceDiscoverer interface, and is used mainly for testing connectivity. type EdgeHostnameResolver struct { @@ -199,13 +283,20 @@ type EdgeHostnameResolver struct { hostnames []string // Addrs to connect to cloudflare's edge network addrs []*net.TCPAddr - // index of the next element to use in addrs - nextAddrIndex int + // Addresses that are in active use. + // This is actually a set of net.TCPAddr's. We have to encode the keys + // with .String(), since net.TCPAddr contains a field of type net.IP and + // therefore it cannot be used as a map key + inUse map[string]*net.TCPAddr + // Addresses that were discarded due to a network error. + // Not sure what we'll do with these, but it feels good to keep them around for now. + bad []*net.TCPAddr } func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, error) { r := &EdgeHostnameResolver{ hostnames: edgeHostnames, + inUse: map[string]*net.TCPAddr{}, } if err := r.Refresh(); err != nil { return nil, err @@ -213,18 +304,49 @@ func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, err return r, nil } -func (r *EdgeHostnameResolver) Addr() *net.TCPAddr { +func (r *EdgeHostnameResolver) Addr() (*net.TCPAddr, error) { r.Lock() defer r.Unlock() - addr := r.addrs[r.nextAddrIndex] - r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs) - return addr + if len(r.addrs) == 0 { + return nil, errors.New("No IP address to claim") + } + var addr *net.TCPAddr + addr, r.addrs = popAddr(r.addrs) + r.inUse[addr.String()] = addr + return addr, nil } -func (r *EdgeHostnameResolver) AvailableAddrs() uint8 { +func (r *EdgeHostnameResolver) AnyAddr() (*net.TCPAddr, error) { r.Lock() defer r.Unlock() - return uint8(len(r.addrs)) + // return an unused addr + if len(r.addrs) > 0 { + return r.addrs[rand.Intn(len(r.addrs))], nil + } + // return an addr that's in use + for _, addr := range r.inUse { + return addr, nil + } + return nil, errors.New("No IP addresses") +} + +func (r *EdgeHostnameResolver) ReplaceAddr(addr *net.TCPAddr) { + r.Lock() + defer r.Unlock() + delete(r.inUse, addr.String()) + r.addrs = append(r.addrs, addr) +} +func (r *EdgeHostnameResolver) MarkAddrBad(addr *net.TCPAddr) { + r.Lock() + defer r.Unlock() + delete(r.inUse, addr.String()) + r.bad = append(r.bad, addr) +} + +func (r *EdgeHostnameResolver) AvailableAddrs() int { + r.Lock() + defer r.Unlock() + return len(r.addrs) } func (r *EdgeHostnameResolver) Refresh() error { @@ -234,8 +356,14 @@ func (r *EdgeHostnameResolver) Refresh() error { } r.Lock() defer r.Unlock() - r.addrs = newAddrs - r.nextAddrIndex = 0 + var notInUse []*net.TCPAddr + for _, newAddr := range newAddrs { + if _, ok := r.inUse[newAddr.String()]; !ok { + notInUse = append(notInUse, newAddr) + } + } + r.addrs = notInUse + r.bad = nil return nil } @@ -252,3 +380,41 @@ func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) { } return tcpAddrs, nil } + +// Compute total set of IP addresses in use. This is useful if the regions +// are returned in a different order, or if an IP address is assigned to +// a different region for some reasion. +func allInUse(regions []*region) map[string]*net.TCPAddr { + result := make(map[string]*net.TCPAddr) + for _, region := range regions { + for k, v := range region.inUse { + result[k] = v + } + } + return result +} + +func makeHARegions(addrLists [][]*net.TCPAddr, inUse map[string]*net.TCPAddr) (regions []*region) { + for _, addrList := range addrLists { + region := ®ion{inUse: map[string]*net.TCPAddr{}} + for _, addr := range addrList { + addrString := addr.String() + // No matter what region `addr` used to belong to, it's now a part + // of this region, so add it to this region's `inUse` map. + if _, ok := inUse[addrString]; ok { + region.inUse[addrString] = addr + } else { + region.addrs = append(region.addrs, addr) + } + } + regions = append(regions, region) + } + return +} + +func popAddr(addrs []*net.TCPAddr) (*net.TCPAddr, []*net.TCPAddr) { + first := addrs[0] + addrs[0] = nil // prevent memory leak + addrs = addrs[1:] + return first, addrs +} diff --git a/connection/discovery_test.go b/connection/discovery_test.go index 806df8bb..c74ebe81 100644 --- a/connection/discovery_test.go +++ b/connection/discovery_test.go @@ -2,62 +2,316 @@ package connection import ( "net" + "sync" "testing" + "testing/quick" + "time" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) -type mockEdgeServiceDiscoverer struct { +func TestEdgeDiscovery(t *testing.T) { + mockAddrs := newMockAddrs(19, 2, 5) + netLookupSRV = mockNetLookupSRV(mockAddrs) + netLookupIP = mockNetLookupIP(mockAddrs) + + expectedAddrSet := map[string]bool{} + for _, addrs := range mockAddrs.addrMap { + for _, addr := range addrs { + expectedAddrSet[addr.String()] = true + } + } + + addrLists, err := EdgeDiscovery(logrus.New().WithFields(logrus.Fields{})) + assert.NoError(t, err) + actualAddrSet := map[string]bool{} + for _, addrs := range addrLists { + for _, addr := range addrs { + actualAddrSet[addr.String()] = true + } + } + + assert.Equal(t, expectedAddrSet, actualAddrSet) } -func (mr *mockEdgeServiceDiscoverer) Addr() *net.TCPAddr { - return &net.TCPAddr{ - IP: net.ParseIP("127.0.0.1"), - Port: 63102, +func TestAllInUse(t *testing.T) { + for _, testCase := range []struct { + regions []*region + expected map[string]*net.TCPAddr + }{ + { + regions: nil, + expected: map[string]*net.TCPAddr{}, + }, + { + regions: []*region{ + ®ion{inUse: map[string]*net.TCPAddr{}}, + ®ion{inUse: map[string]*net.TCPAddr{}}, + }, + expected: map[string]*net.TCPAddr{}, + }, + { + regions: []*region{ + ®ion{inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}}, + ®ion{inUse: map[string]*net.TCPAddr{":4": &net.TCPAddr{Port: 4}}}, + }, + expected: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}, ":4": &net.TCPAddr{Port: 4}}, + }, + } { + actual := allInUse(testCase.regions) + assert.Equal(t, testCase.expected, actual) } } -func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() uint8 { - return 1 +func TestMakeRegions(t *testing.T) { + for _, testCase := range []struct { + addrList [][]*net.TCPAddr + inUse map[string]*net.TCPAddr + expected []*region + }{ + { + addrList: [][]*net.TCPAddr{}, + expected: nil, + }, + { + addrList: [][]*net.TCPAddr{ + []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, + }, + expected: []*region{ + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{}}, + }, + }, + { + addrList: [][]*net.TCPAddr{ + []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, + []*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}}, + }, + expected: []*region{ + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}}, inUse: map[string]*net.TCPAddr{}}, + }, + }, + { + addrList: [][]*net.TCPAddr{ + []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, + []*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}}, + }, + inUse: map[string]*net.TCPAddr{ + ":1": &net.TCPAddr{Port: 1}, + ":4": &net.TCPAddr{Port: 4}, + }, + expected: []*region{ + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 3}}, inUse: map[string]*net.TCPAddr{":4": &net.TCPAddr{Port: 4}}}, + }, + }, + } { + actual := makeHARegions(testCase.addrList, testCase.inUse) + assert.Equal(t, testCase.expected, actual) + } } -func (mr *mockEdgeServiceDiscoverer) Refresh() error { - return nil +func assertIsBalanced(t *testing.T, regions []*region) bool { + // Compute max(len(region.addrs) for region in regions) + // No region should have significantly fewer addresses than this + var longestAddrs int + { + longestAddrs = 0 + for _, region := range regions { + if l := len(region.addrs); l > longestAddrs { + longestAddrs = l + } + } + } + for _, region := range regions { + if len(region.addrs) == longestAddrs || len(region.addrs) == longestAddrs-1 { + continue + } + return assert.Fail(t, + "found a region with %v free addrs, while the longest addrs list is %v", + len(region.addrs), longestAddrs) + } + return true } -func TestFlattenServiceIPs(t *testing.T) { - result := flattenServiceIPs([][]*net.TCPAddr{ - []*net.TCPAddr{ - &net.TCPAddr{Port: 1}, - &net.TCPAddr{Port: 2}, - &net.TCPAddr{Port: 3}, - &net.TCPAddr{Port: 4}, - }, - []*net.TCPAddr{ - &net.TCPAddr{Port: 10}, - &net.TCPAddr{Port: 12}, - &net.TCPAddr{Port: 13}, - }, - []*net.TCPAddr{ - &net.TCPAddr{Port: 21}, - &net.TCPAddr{Port: 22}, - &net.TCPAddr{Port: 23}, - &net.TCPAddr{Port: 24}, - &net.TCPAddr{Port: 25}, - }, - }) - assert.EqualValues(t, []*net.TCPAddr{ - &net.TCPAddr{Port: 1}, - &net.TCPAddr{Port: 10}, - &net.TCPAddr{Port: 21}, - &net.TCPAddr{Port: 2}, - &net.TCPAddr{Port: 12}, - &net.TCPAddr{Port: 22}, - &net.TCPAddr{Port: 3}, - &net.TCPAddr{Port: 13}, - &net.TCPAddr{Port: 23}, - &net.TCPAddr{Port: 4}, - &net.TCPAddr{Port: 24}, - &net.TCPAddr{Port: 25}, - }, result) +// Various end-to-end tests, run with quickcheck (i.e. the testing/quick package) +func TestEdgeAddrResolver(t *testing.T) { + concurrentReplacement := func(mockAddrs mockAddrs) bool { + netLookupSRV = mockNetLookupSRV(mockAddrs) + netLookupIP = mockNetLookupIP(mockAddrs) + + resolver, err := NewEdgeAddrResolver(logrus.New()) + if !assert.NoError(t, err) { + return false + } + assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(), + "every address should be initially available") + + // Create several goroutines to simulate HA connections that acquire + // and replace IP addresses. + var wg sync.WaitGroup + wg.Add(mockAddrs.numAddrs) + for i := 0; i < mockAddrs.numAddrs; i++ { + go func() { + defer wg.Done() + const reconnectionCount = 50 + for i := 0; i < reconnectionCount; i++ { + if resolver.AvailableAddrs() == 0 { + err = resolver.Refresh() + assert.NoError(t, err) + } + addr, err := resolver.Addr() + if !assert.NoError(t, err) { + return + } + time.Sleep(0) // allow some other goroutine to run + resolver.ReplaceAddr(addr) + time.Sleep(0) // allow some other goroutine to run + } + }() + } + wg.Wait() + assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(), + "every address should be available after replacement") + return !t.Failed() + } + + badAddrWithRefresh := func(mockAddrs mockAddrs) bool { + netLookupSRV = mockNetLookupSRV(mockAddrs) + netLookupIP = mockNetLookupIP(mockAddrs) + + resolver, err := NewEdgeAddrResolver(logrus.New()) + if !assert.NoError(t, err) { + return false + } + assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(), + "every address should be initially available") + + var addrs []*net.TCPAddr + for i := 0; i < mockAddrs.numAddrs; i++ { + assert.Equal(t, mockAddrs.numAddrs-i, resolver.AvailableAddrs()) + addr, err := resolver.Addr() + assert.NoError(t, err) + addrs = append(addrs, addr) + } + assert.Equal(t, 0, resolver.AvailableAddrs(), "all addresses should have been taken") + _, err = resolver.Addr() + assert.Error(t, err) + + anyAddr, err := resolver.AnyAddr() + assert.NoError(t, err, "should still be okay to call AnyAddr") + + resolver.MarkAddrBad(anyAddr) + + assert.Equal(t, 0, resolver.AvailableAddrs(), "all addresses should still be used") + _, err = resolver.Addr() + assert.Error(t, err, "all addresses should still be used") + + err = resolver.Refresh() + assert.NoError(t, err, "Refresh() should have worked") + + assert.Equal(t, 1, resolver.AvailableAddrs(), + "Refresh() should have reset the state of the 'bad' address") + addr, err := resolver.Addr() + assert.NoError(t, err) + assert.Equal(t, anyAddr, addr) + + _, err = resolver.Addr() + assert.Error(t, err, "all addresses should be used again") + + return !t.Failed() + } + + assert.NoError(t, quick.Check(concurrentReplacement, nil)) + assert.NoError(t, quick.Check(badAddrWithRefresh, nil)) +} + +// "White-box" test: runs Addr() and checks internal state +func TestEdgeAddrResolver_Addr(t *testing.T) { + e := &EdgeAddrResolver{regions: nil} + addr, err := e.Addr() + assert.Error(t, err) + + testRegions := func() []*region { + return []*region{ + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}}, + } + } + e = &EdgeAddrResolver{regions: testRegions()} + addr, err = e.Addr() + assert.NoError(t, err) + assert.Equal(t, &net.TCPAddr{Port: 4}, addr) + var expected []*region + { + expected = testRegions() + expected[1].addrs = expected[1].addrs[1:] + expected[1].inUse[":4"] = &net.TCPAddr{Port: 4} + } + assert.Equal(t, expected, e.regions) +} + +// "White-box" test: runs AnyAddr() and checks internal state +func TestEdgeAddrResolver_AnyAddr(t *testing.T) { + e := &EdgeAddrResolver{regions: nil} + addr, err := e.AnyAddr() + assert.Error(t, err) + + e = &EdgeAddrResolver{regions: []*region{®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}}}}} + addr, err = e.AnyAddr() + assert.NoError(t, err) + assert.Equal(t, &net.TCPAddr{Port: 1}, addr, "should have chosen the inactive address") + + e = &EdgeAddrResolver{regions: []*region{®ion{inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}}}} + addr, err = e.AnyAddr() + assert.NoError(t, err) + assert.Equal(t, &net.TCPAddr{Port: 1}, addr, "should have chosen an active address rather than nothing") +} + +// "White-box" test: runs ReplaceAddr() and checks internal state +func TestEdgeAddrResolver_ReplaceAddr(t *testing.T) { + e := &EdgeAddrResolver{regions: nil} + e.ReplaceAddr(&net.TCPAddr{Port: 1}) // this shouldn't panic, I guess + + testRegions := func() []*region { + return []*region{ + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}}, + } + } + e = &EdgeAddrResolver{regions: testRegions()} + e.ReplaceAddr(&net.TCPAddr{Port: 6}) + var expected []*region + { + expected = testRegions() + delete(expected[1].inUse, ":6") + expected[1].addrs = append(expected[1].addrs, &net.TCPAddr{Port: 6}) + } + assert.Equal(t, expected, e.regions) +} + +// "White-box" test: runs MarkAddrBad() and checks internal state +func TestEdgeAddrResolver_MarkAddrBad(t *testing.T) { + e := &EdgeAddrResolver{regions: nil} + e.ReplaceAddr(&net.TCPAddr{Port: 1}) // this shouldn't panic, I guess + + testRegions := func() []*region { + return []*region{ + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}}, + ®ion{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}}, + } + } + e = &EdgeAddrResolver{regions: testRegions()} + e.MarkAddrBad(&net.TCPAddr{Port: 6}) + var expected []*region + { + expected = testRegions() + delete(expected[1].inUse, ":6") + expected[1].bad = append(expected[1].bad, &net.TCPAddr{Port: 6}) + } + assert.Equal(t, expected, e.regions) } diff --git a/connection/manager.go b/connection/manager.go index 9ffc50ec..b5ceacfe 100644 --- a/connection/manager.go +++ b/connection/manager.go @@ -127,7 +127,10 @@ func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurab } func (em *EdgeManager) newConnection(ctx context.Context) *tunnelpogs.ConnectError { - edgeTCPAddr := em.serviceDiscoverer.Addr() + edgeTCPAddr, err := em.serviceDiscoverer.Addr() + if err != nil { + return retryConnection(fmt.Sprintf("edge address discovery error: %v", err)) + } configurable := em.state.getConfigurable() edgeConn, err := DialEdge(ctx, configurable.Timeout, em.tlsConfig, edgeTCPAddr) if err != nil { @@ -147,7 +150,7 @@ func (em *EdgeManager) newConnection(ctx context.Context) *tunnelpogs.ConnectErr retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err)) } - h2muxConn, err := newConnection(muxer) + h2muxConn, err := newConnection(muxer, edgeTCPAddr) if err != nil { return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err)) } @@ -186,6 +189,7 @@ func (em *EdgeManager) closeConnection(ctx context.Context) error { return fmt.Errorf("no connection to close") } conn.Shutdown() + // teardown will be handled by EdgeManager.serveConn in another goroutine return nil } @@ -193,6 +197,7 @@ func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) { err := conn.Serve(ctx) em.logger.WithError(err).Warn("Connection closed") em.state.closeConnection(conn) + em.serviceDiscoverer.ReplaceAddr(conn.addr) } func (em *EdgeManager) noRetryMessage() string { @@ -221,14 +226,14 @@ func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCr } } -func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs uint8) bool { +func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs int) bool { ems.RLock() defer ems.RUnlock() - expectedHAConns := ems.configurable.NumHAConnections + expectedHAConns := int(ems.configurable.NumHAConnections) if availableEdgeAddrs < expectedHAConns { expectedHAConns = availableEdgeAddrs } - return uint8(len(ems.conns)) < expectedHAConns + return len(ems.conns) < expectedHAConns } func (ems *edgeManagerState) shouldReduceConnection() bool { diff --git a/connection/mocks_for_test.go b/connection/mocks_for_test.go new file mode 100644 index 00000000..03c08d8b --- /dev/null +++ b/connection/mocks_for_test.go @@ -0,0 +1,118 @@ +package connection + +import ( + "fmt" + "math" + "math/rand" + "net" + "reflect" + "testing/quick" +) + +type mockAddrs struct { + // a set of synthetic SRV records + addrMap map[net.SRV][]*net.TCPAddr + // the total number of addresses, aggregated across addrMap. + // For the convenience of test code that would otherwise have to compute + // this by hand every time. + numAddrs int +} + +func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs { + addrMap := make(map[net.SRV][]*net.TCPAddr) + numAddrs := 0 + + for r := uint8(0); r < numRegions; r++ { + var ( + srv = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port} + addrs []*net.TCPAddr + ) + for a := uint8(0); a < numAddrsPerRegion; a++ { + addrs = append(addrs, &net.TCPAddr{ + IP: net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)), + Port: int(port), + }) + } + addrMap[srv] = addrs + numAddrs += len(addrs) + } + return mockAddrs{addrMap: addrMap, numAddrs: numAddrs} +} + +var _ quick.Generator = mockAddrs{} + +func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value { + port := uint16(rand.Intn(math.MaxUint16)) + numRegions := uint8(1 + rand.Intn(10)) + numAddrsPerRegion := uint8(1 + rand.Intn(32)) + result := newMockAddrs(port, numRegions, numAddrsPerRegion) + return reflect.ValueOf(result) +} + +// Returns a function compatible with net.LookupSRV that will return the SRV +// records from mockAddrs. +func mockNetLookupSRV( + m mockAddrs, +) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) { + var addrs []*net.SRV + for k := range m.addrMap { + addr := k + addrs = append(addrs, &addr) + // We can't just do + // addrs = append(addrs, &k) + // `k` will be reused by subsequent loop iterations, + // so all the copies of `&k` would point to the same location. + } + return func(_, _, _ string) (string, []*net.SRV, error) { + return "", addrs, nil + } +} + +// Returns a function compatible with net.LookupIP that translates the SRV records +// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs. +func mockNetLookupIP( + m mockAddrs, +) func(host string) ([]net.IP, error) { + return func(host string) ([]net.IP, error) { + for srv, tcpAddrs := range m.addrMap { + if srv.Target != host { + continue + } + result := make([]net.IP, len(tcpAddrs)) + for i, tcpAddr := range tcpAddrs { + result[i] = tcpAddr.IP + } + return result, nil + } + return nil, fmt.Errorf("No IPs for %v", host) + } +} + +type mockEdgeServiceDiscoverer struct { +} + +func (mr *mockEdgeServiceDiscoverer) Addr() (*net.TCPAddr, error) { + return &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 63102, + }, nil +} + +func (mr *mockEdgeServiceDiscoverer) AnyAddr() (*net.TCPAddr, error) { + return &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 63102, + }, nil +} + +func (mr *mockEdgeServiceDiscoverer) ReplaceAddr(addr *net.TCPAddr) {} + +func (mr *mockEdgeServiceDiscoverer) MarkAddrBad(addr *net.TCPAddr) {} + +func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() int { + return 1 +} + +func (mr *mockEdgeServiceDiscoverer) Refresh() error { + return nil +} diff --git a/go.mod b/go.mod index bce9d655..bc4f4046 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,10 @@ require ( github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 github.com/go-sql-driver/mysql v1.4.1 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 + github.com/google/certificate-transparency-go v1.1.0 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.3 - github.com/gorilla/websocket v1.2.0 + github.com/gorilla/websocket v1.4.0 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/jmoiron/sqlx v1.2.0 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect @@ -56,8 +57,6 @@ require ( golang.org/x/net v0.0.0-20191007182048-72f939374954 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20191008105621-543471e840be - golang.org/x/text v0.3.2 // indirect - google.golang.org/appengine v1.4.0 // indirect google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect google.golang.org/grpc v1.24.0 // indirect gopkg.in/coreos/go-oidc.v2 v2.1.0 diff --git a/go.sum b/go.sum index cb81dbe9..ae7fcad3 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,28 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= +contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs= github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -24,15 +34,26 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/coredns/coredns v1.2.0 h1:YEI38K2BJYzL/SxO2tZFD727T/C68DqVWkBQjT0sWPU= github.com/coredns/coredns v1.2.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmqg700pbXhzvmoaXKyfxWRkjRym7Q= github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA= github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= @@ -45,73 +66,176 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 h1:gF8ngtda767ddth2SH0YSAhswhz6qUkvyI9EZFYCWJA= github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4= github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= +github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/certificate-transparency-go v1.1.0 h1:10MlrYzh5wfkToxWI4yJzffsxLfxcEDlOATMx/V9Kzw= +github.com/google/certificate-transparency-go v1.1.0/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/monologue v0.0.0-20190606152607-4b11a32b5934/go.mod h1:6NTfaQoUpg5QmPsCUWLR3ig33FHrKXhTtWzF0DVdmuk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/trillian v1.2.2-0.20190612132142-05461f4df60a/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA= +github.com/google/trillian-examples v0.0.0-20190603134952-4e75ba15216c/go.mod h1:WgL3XZ3pA8/9cm7yxqWrZE6iZkESB2ItGxy5Fo6k2lk= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw= github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/letsencrypt/pkcs11key v2.0.1-0.20170608213348-396559074696+incompatible/go.mod h1:iGYXKqDXt0cpBthCHdr9ZdsQwyGlYFh/+8xa4WzIQ34= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b h1:/BbY4n99iMazlr2igipph+hj0MwlZIWpcsP8Iy+na+s= github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY= github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI= github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= @@ -131,80 +255,191 @@ github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLy github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= +github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= +github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0 h1:6DtWz8hNS4qbq0OCRPhdBMG9E2qKTSDKlwnP3dmZvuA= github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd h1:84VQPzup3IpKLxuIAZjHMhVjJ8fZ4/i3yUnj3k6fUdw= google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I= gopkg.in/coreos/go-oidc.v2 v2.1.0/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.4.0 h1:0kXPskUMGAXXWJlP05ktEMOV0vmzFQUWw6d+aZJQU8A= gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8 h1:/pLAskKF+d5SawboKd8GB8ew4ClHDbt2c3K9EBFeRGU= gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7 h1:CZoOFlTPbKfAShKYrMuUfYbnXexFT1rYRUX1SPnrdE4= zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7/go.mod h1:TMGa8HWGJkXiq4nHe9Zu/JgRF5oUtg4XizFC+Vexbec= diff --git a/origin/supervisor.go b/origin/supervisor.go index f5d08ed4..00eb9d7a 100644 --- a/origin/supervisor.go +++ b/origin/supervisor.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "net" "sync" "time" @@ -41,11 +40,9 @@ var ( ) type Supervisor struct { - cloudflaredUUID uuid.UUID - config *TunnelConfig - edgeIPs []*net.TCPAddr - // nextUnusedEdgeIP is the index of the next addr k edgeIPs to try - nextUnusedEdgeIP int + cloudflaredUUID uuid.UUID + config *TunnelConfig + edgeIPs connection.EdgeServiceDiscoverer lastResolve time.Time resolverC chan resolveResult tunnelErrors chan tunnelError @@ -65,25 +62,30 @@ type Supervisor struct { } type resolveResult struct { - edgeIPs []*net.TCPAddr - err error + err error } type tunnelError struct { index int + addr *net.TCPAddr err error } -func NewSupervisor(config *TunnelConfig, u uuid.UUID) *Supervisor { +func NewSupervisor(config *TunnelConfig, u uuid.UUID) (*Supervisor, error) { + edgeIPs, err := connection.NewEdgeAddrResolver(config.Logger) + if err != nil { + return nil, err + } return &Supervisor{ cloudflaredUUID: u, config: config, + edgeIPs: edgeIPs, tunnelErrors: make(chan tunnelError), tunnelsConnecting: map[int]chan struct{}{}, logger: config.Logger.WithField("subsystem", "supervisor"), jwtLock: &sync.RWMutex{}, eventDigestLock: &sync.RWMutex{}, - } + }, nil } func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) error { @@ -134,8 +136,8 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) er // If the error is a dial error, the problem is likely to be network related // try another addr before refreshing since we are likely to get back the // same IPs in the same order. Same problem with duplicate connection error. - if s.unusedIPs() { - s.replaceEdgeIP(tunnelError.index) + if s.unusedIPs() && tunnelError.addr != nil { + s.edgeIPs.MarkAddrBad(tunnelError.addr) } else { s.refreshEdgeIPs() } @@ -170,7 +172,6 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) er s.resolverC = nil if result.err == nil { logger.Debug("Service discovery refresh complete") - s.edgeIPs = result.edgeIPs } else { logger.WithError(result.err).Error("Service discovery error") } @@ -182,19 +183,18 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) er func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal) error { logger := s.logger - edgeIPs, err := s.resolveEdgeIPs() + err := s.edgeIPs.Refresh() if err != nil { logger.Infof("ResolveEdgeIPs err") return err } - s.edgeIPs = edgeIPs - if s.config.HAConnections > len(edgeIPs) { - logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, len(edgeIPs)) - s.config.HAConnections = len(edgeIPs) - } s.lastResolve = time.Now() - // check entitlement and version too old error before attempting to register more tunnels - s.nextUnusedEdgeIP = s.config.HAConnections + availableAddrs := int(s.edgeIPs.AvailableAddrs()) + if s.config.HAConnections > availableAddrs { + logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs) + s.config.HAConnections = availableAddrs + } + go s.startFirstTunnel(ctx, connectedSignal) select { case <-ctx.Done(): @@ -216,16 +216,24 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig // startTunnel starts the first tunnel connection. The resulting error will be sent on // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal) { - err := ServeTunnelLoop(ctx, s, s.config, s.getEdgeIP(0), 0, connectedSignal, s.cloudflaredUUID) + var ( + addr *net.TCPAddr + err error + ) defer func() { - s.tunnelErrors <- tunnelError{index: 0, err: err} + s.tunnelErrors <- tunnelError{index: 0, addr: addr, err: err} }() + addr, err = s.edgeIPs.Addr() + if err != nil { + return + } + + err = ServeTunnelLoop(ctx, s, s.config, addr, 0, connectedSignal, s.cloudflaredUUID) + for s.unusedIPs() { - select { - case <-ctx.Done(): + if ctx.Err() != nil { return - default: } switch err.(type) { case nil: @@ -233,19 +241,34 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign // try the next address if it was a dialError(network problem) or // dupConnRegisterTunnelError case connection.DialError, dupConnRegisterTunnelError: - s.replaceEdgeIP(0) + s.edgeIPs.MarkAddrBad(addr) default: return } - err = ServeTunnelLoop(ctx, s, s.config, s.getEdgeIP(0), 0, connectedSignal, s.cloudflaredUUID) + addr, err = s.edgeIPs.Addr() + if err != nil { + return + } + err = ServeTunnelLoop(ctx, s, s.config, addr, 0, connectedSignal, s.cloudflaredUUID) } } // startTunnel starts a new tunnel connection. The resulting error will be sent on // s.tunnelErrors. func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal) { - err := ServeTunnelLoop(ctx, s, s.config, s.getEdgeIP(index), uint8(index), connectedSignal, s.cloudflaredUUID) - s.tunnelErrors <- tunnelError{index: index, err: err} + var ( + addr *net.TCPAddr + err error + ) + defer func() { + s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err} + }() + + addr, err = s.edgeIPs.Addr() + if err != nil { + return + } + err = ServeTunnelLoop(ctx, s, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID) } func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal { @@ -267,17 +290,8 @@ func (s *Supervisor) waitForNextTunnel(index int) bool { return false } -func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr { - return s.edgeIPs[index%len(s.edgeIPs)] -} - -func (s *Supervisor) resolveEdgeIPs() ([]*net.TCPAddr, error) { - // If --edge is specfied, resolve edge server addresses - if len(s.config.EdgeAddrs) > 0 { - return connection.ResolveAddrs(s.config.EdgeAddrs) - } - // Otherwise lookup edge server addresses through service discovery - return connection.EdgeDiscovery(s.logger) +func (s *Supervisor) unusedIPs() bool { + return s.edgeIPs.AvailableAddrs() > s.config.HAConnections } func (s *Supervisor) refreshEdgeIPs() { @@ -289,20 +303,11 @@ func (s *Supervisor) refreshEdgeIPs() { } s.resolverC = make(chan resolveResult) go func() { - edgeIPs, err := s.resolveEdgeIPs() - s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err} + err := s.edgeIPs.Refresh() + s.resolverC <- resolveResult{err: err} }() } -func (s *Supervisor) unusedIPs() bool { - return s.nextUnusedEdgeIP < len(s.edgeIPs) -} - -func (s *Supervisor) replaceEdgeIP(badIPIndex int) { - s.edgeIPs[badIPIndex] = s.edgeIPs[s.nextUnusedEdgeIP] - s.nextUnusedEdgeIP++ -} - func (s *Supervisor) ReconnectToken() ([]byte, error) { s.jwtLock.RLock() defer s.jwtLock.RUnlock() @@ -366,7 +371,11 @@ func (s *Supervisor) refreshAuth( } func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error) { - arbitraryEdgeIP := s.getEdgeIP(rand.Int()) + arbitraryEdgeIP, err := s.edgeIPs.AnyAddr() + if err != nil { + return nil, err + } + edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP) if err != nil { return nil, err diff --git a/origin/supervisor_test.go b/origin/supervisor_test.go index 559c0400..986b326f 100644 --- a/origin/supervisor_test.go +++ b/origin/supervisor_test.go @@ -24,7 +24,10 @@ func TestRefreshAuthBackoff(t *testing.T) { return time.After(d) } - s := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + if !assert.NoError(t, err) { + t.FailNow() + } backoff := &BackoffHandler{MaxRetries: 3} auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) { return nil, fmt.Errorf("authentication failure") @@ -66,7 +69,10 @@ func TestRefreshAuthSuccess(t *testing.T) { return time.After(d) } - s := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + if !assert.NoError(t, err) { + t.FailNow() + } backoff := &BackoffHandler{MaxRetries: 3} auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) { return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil @@ -92,7 +98,10 @@ func TestRefreshAuthUnknown(t *testing.T) { return time.After(d) } - s := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + if !assert.NoError(t, err) { + t.FailNow() + } backoff := &BackoffHandler{MaxRetries: 3} auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) { return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil @@ -112,7 +121,10 @@ func TestRefreshAuthFail(t *testing.T) { logger := logrus.New() logger.Level = logrus.ErrorLevel - s := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New()) + if !assert.NoError(t, err) { + t.FailNow() + } backoff := &BackoffHandler{MaxRetries: 3} auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) { return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil diff --git a/origin/tunnel.go b/origin/tunnel.go index 3a05e347..eefd2960 100644 --- a/origin/tunnel.go +++ b/origin/tunnel.go @@ -156,7 +156,11 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str } func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID) error { - return NewSupervisor(config, cloudflaredID).Run(ctx, connectedSignal) + s, err := NewSupervisor(config, cloudflaredID) + if err != nil { + return err + } + return s.Run(ctx, connectedSignal) } func ServeTunnelLoop(ctx context.Context,