curben
/
cloudflared-mirror
mirror of https://github.com/cloudflare/cloudflared.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'cloudflare:master' into tunnel-health

This commit is contained in:
Mads Jon Nielsen 2024-10-28 10:33:08 +01:00 committed by GitHub
parent f2016e7f63 0eddb8a615
commit 2cbe125e0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
190 changed files with 6799 additions and 10595 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
RELEASE_NOTES
View File

@ -1,3 +1,12 @@
2024.10.1
- 2024-10-23 TUN-8694: Fix github release script
- 2024-10-21 Revert "TUN-8592: Use metadata from the edge to determine if request body is empty for QUIC transport"
- 2024-10-18 TUN-8688: Correct UDP bind for IPv6 edge connectivity on macOS
- 2024-10-17 TUN-8685: Bump coredns dependency
- 2024-10-16 TUN-8638: Add datagram v3 serializers and deserializers
- 2024-10-15 chore: Remove h2mux code
- 2024-10-11 TUN-8631: Abort release on version mismatch
2024.10.0
- 2024-10-01 TUN-8646: Add datagram v3 support feature flag
- 2024-09-30 TUN-8621: Fix cloudflared version in change notes to account for release date

7
connection/connection.go
View File

@ -36,6 +36,13 @@ var (
flushableContentTypes = []string{sseContentType, grpcContentType}
)
// TunnelConnection represents the connection to the edge.
// The Serve method is provided to allow clients to handle any errors from the connection encountered during
// processing of the connection. Cancelling of the context provided to Serve will close the connection.
type TunnelConnection interface {
Serve(ctx context.Context) error
}
type Orchestrator interface {
UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
GetConfigJSON() ([]byte, error)

3
connection/errors.go
View File

@ -2,7 +2,6 @@ package connection
import (
"github.com/cloudflare/cloudflared/edgediscovery"
"github.com/cloudflare/cloudflared/h2mux"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
)
@ -71,8 +70,6 @@ func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) b
switch err.(type) {
case edgediscovery.DialError:
log.Error().Msg("Connection unable to dial edge")
case h2mux.MuxerHandshakeError:
log.Error().Msg("Connection handshake with edge server failed")
default:
log.Error().Msg("Connection failed")
return false

32
connection/h2mux.go
View File

@ -1,32 +0,0 @@
package connection
import (
"time"
"github.com/rs/zerolog"
"github.com/cloudflare/cloudflared/h2mux"
)
const (
muxerTimeout = 5 * time.Second
)
type MuxerConfig struct {
HeartbeatInterval time.Duration
MaxHeartbeats uint64
CompressionSetting h2mux.CompressionSetting
MetricsUpdateFreq time.Duration
}
func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Logger) *h2mux.MuxerConfig {
return &h2mux.MuxerConfig{
Timeout: muxerTimeout,
Handler: h,
IsClient: true,
HeartbeatInterval: mc.HeartbeatInterval,
MaxHeartbeats: mc.MaxHeartbeats,
Log: log,
CompressionQuality: mc.CompressionSetting,
}
}

128
connection/h2mux_header.go
View File

@ -1,128 +0,0 @@
package connection
import (
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/cloudflare/cloudflared/h2mux"
)
// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
// to an HTTP/1 Request object destined for the local origin web service.
// This operation includes conversion of the pseudo-headers into their closest
// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
for _, header := range h2 {
name := strings.ToLower(header.Name)
if !IsH2muxControlRequestHeader(name) {
continue
}
switch name {
case ":method":
h1.Method = header.Value
case ":scheme":
// noop - use the preexisting scheme from h1.URL
case ":authority":
// Otherwise the host header will be based on the origin URL
h1.Host = header.Value
case ":path":
// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
// However, this HTTP/1 Request object belongs to the Go standard library,
// whose URL package makes some opinionated decisions about the encoding of
// URL characters: see the docs of https://godoc.org/net/url#URL,
// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
// which is always used when computing url.URL.String(), whether we'd like it or not.
//
// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
// is treated differently when HTTP_PROXY is set!
// See https://github.com/golang/go/issues/5684#issuecomment-66080888
//
// This means we are subject to the behavior of net/url's function `shouldEscape`
// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
if header.Value == "*" {
h1.URL.Path = "*"
continue
}
// Due to the behavior of validation.ValidateUrl, h1.URL may
// already have a partial value, with or without a trailing slash.
base := h1.URL.String()
base = strings.TrimRight(base, "/")
// But we know :path begins with '/', because we handled '*' above - see RFC7540
requestURL, err := url.Parse(base + header.Value)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
}
h1.URL = requestURL
case "content-length":
contentLength, err := strconv.ParseInt(header.Value, 10, 64)
if err != nil {
return fmt.Errorf("unparseable content length")
}
h1.ContentLength = contentLength
case RequestUserHeaders:
// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
// Find and parse user headers serialized into a single one
userHeaders, err := DeserializeHeaders(header.Value)
if err != nil {
return errors.Wrap(err, "Unable to parse user headers")
}
for _, userHeader := range userHeaders {
h1.Header.Add(userHeader.Name, userHeader.Value)
}
default:
// All other control headers shall just be proxied transparently
h1.Header.Add(header.Name, header.Value)
}
}
return nil
}
func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) {
h2 = []h2mux.Header{
{Name: ":status", Value: strconv.Itoa(status)},
}
userHeaders := make(http.Header, len(h1))
for header, values := range h1 {
h2name := strings.ToLower(header)
if h2name == "content-length" {
// This header has meaning in HTTP/2 and will be used by the edge,
// so it should be sent as an HTTP/2 response header.
// Since these are http2 headers, they're required to be lowercase
h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]})
} else if !IsH2muxControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
// User headers, on the other hand, must all be serialized so that
// HTTP/2 header validation won't be applied to HTTP/1 header values
userHeaders[header] = values
}
}
// Perform user header serialization and set them in the single header
h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)})
return h2
}
// IsH2muxControlRequestHeader is called in the direction of eyeball -> origin.
func IsH2muxControlRequestHeader(headerName string) bool {
return headerName == "content-length" ||
headerName == "connection" || headerName == "upgrade" || // Websocket request headers
strings.HasPrefix(headerName, ":") ||
strings.HasPrefix(headerName, "cf-")
}
// IsH2muxControlResponseHeader is called in the direction of eyeball <- origin.
func IsH2muxControlResponseHeader(headerName string) bool {
return headerName == "content-length" ||
strings.HasPrefix(headerName, ":") ||
strings.HasPrefix(headerName, "cf-int-") ||
strings.HasPrefix(headerName, "cf-cloudflared-")
}

642
connection/h2mux_header_test.go
View File

@ -1,642 +0,0 @@
package connection
import (
"fmt"
"math/rand"
"net/http"
"net/url"
"reflect"
"regexp"
"strings"
"testing"
"testing/quick"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/cloudflare/cloudflared/h2mux"
)
type ByName []h2mux.Header
func (a ByName) Len() int { return len(a) }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool {
if a[i].Name == a[j].Name {
return a[i].Value < a[j].Value
}
return a[i].Name < a[j].Name
}
func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
assert.NoError(t, err)
mockHeaders := http.Header{
"Mock header 1": {"Mock value 1"},
"Mock header 2": {"Mock value 2"},
}
headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request)
assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
assert.NoError(t, headersConversionErr)
}
func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header {
return []h2mux.Header{{
Name: headersField,
Value: SerializeHeaders(headers),
}}
}
func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
assert.NoError(t, err)
emptyHeaders := make(http.Header)
headersConversionErr := H2RequestHeadersToH1Request(
[]h2mux.Header{{
Name: RequestUserHeaders,
Value: SerializeHeaders(emptyHeaders),
}},
request,
)
assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header))
assert.NoError(t, headersConversionErr)
}
func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
assert.NoError(t, err)
mockRequestHeaders := []h2mux.Header{
{Name: ":path", Value: "//bad_path/"},
{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
}
headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
assert.Equal(t, http.Header{
"Mock header": []string{"Mock value"},
}, request.Header)
assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
assert.NoError(t, headersConversionErr)
}
func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
assert.NoError(t, err)
mockRequestHeaders := []h2mux.Header{
{Name: ":path", Value: "/?query=mock%20value"},
{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
}
headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
assert.Equal(t, http.Header{
"Mock header": []string{"Mock value"},
}, request.Header)
assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String())
assert.NoError(t, headersConversionErr)
}
func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
assert.NoError(t, err)
mockRequestHeaders := []h2mux.Header{
{Name: ":path", Value: "/mock%20path"},
{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
}
headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
assert.Equal(t, http.Header{
"Mock header": []string{"Mock value"},
}, request.Header)
assert.Equal(t, "http://example.com/mock%20path", request.URL.String())
assert.NoError(t, headersConversionErr)
}
func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
type testCase struct {
path string
want string
}
testCases := []testCase{
{
path: "",
want: "",
},
{
path: "/",
want: "/",
},
{
path: "//",
want: "//",
},
{
path: "/test",
want: "/test",
},
{
path: "//test",
want: "//test",
},
{
// https://github.com/cloudflare/cloudflared/issues/81
path: "//test/",
want: "//test/",
},
{
path: "/%2Ftest",
want: "/%2Ftest",
},
{
path: "//%20test",
want: "//%20test",
},
{
// https://github.com/cloudflare/cloudflared/issues/124
path: "/test?get=somthing%20a",
want: "/test?get=somthing%20a",
},
{
path: "/%20",
want: "/%20",
},
{
// stdlib's EscapedPath() will always percent-encode ' '
path: "/ ",
want: "/%20",
},
{
path: "/ a ",
want: "/%20a%20",
},
{
path: "/a%20b",
want: "/a%20b",
},
{
path: "/foo/bar;param?query#frag",
want: "/foo/bar;param?query#frag",
},
{
// stdlib's EscapedPath() will always percent-encode non-ASCII chars
path: "/a␠b",
want: "/a%E2%90%A0b",
},
{
path: "/a-umlaut-ä",
want: "/a-umlaut-%C3%A4",
},
{
path: "/a-umlaut-%C3%A4",
want: "/a-umlaut-%C3%A4",
},
{
path: "/a-umlaut-%c3%a4",
want: "/a-umlaut-%c3%a4",
},
{
// here the second '#' is treated as part of the fragment
path: "/a#b#c",
want: "/a#b%23c",
},
{
path: "/a#b␠c",
want: "/a#b%E2%90%A0c",
},
{
path: "/a#b%20c",
want: "/a#b%20c",
},
{
path: "/a#b c",
want: "/a#b%20c",
},
{
// stdlib's EscapedPath() will always percent-encode '\'
path: "/\\",
want: "/%5C",
},
{
path: "/a\\",
want: "/a%5C",
},
{
path: "/a,b.c.",
want: "/a,b.c.",
},
{
path: "/.",
want: "/.",
},
{
// stdlib's EscapedPath() will always percent-encode '`'
path: "/a`",
want: "/a%60",
},
{
path: "/a[0]",
want: "/a[0]",
},
{
path: "/?a[0]=5 &b[]=",
want: "/?a[0]=5 &b[]=",
},
{
path: "/?a=%22b%20%22",
want: "/?a=%22b%20%22",
},
}
for index, testCase := range testCases {
requestURL := "https://example.com"
request, err := http.NewRequest(http.MethodGet, requestURL, nil)
assert.NoError(t, err)
mockRequestHeaders := []h2mux.Header{
{Name: ":path", Value: testCase.path},
{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
}
headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
assert.NoError(t, headersConversionErr)
assert.Equal(t,
http.Header{
"Mock header": []string{"Mock value"},
},
request.Header)
assert.Equal(t,
"https://example.com"+testCase.want,
request.URL.String(),
"Failed URL index: %v %#v", index, testCase)
}
}
func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
config := &quick.Config{
Values: func(args []reflect.Value, rand *rand.Rand) {
args[0] = reflect.ValueOf(randomHTTP2Path(t, rand))
},
}
type testOrigin struct {
url string
expectedScheme string
expectedBasePath string
}
testOrigins := []testOrigin{
{
url: "http://origin.hostname.example.com:8080",
expectedScheme: "http",
expectedBasePath: "http://origin.hostname.example.com:8080",
},
{
url: "http://origin.hostname.example.com:8080/",
expectedScheme: "http",
expectedBasePath: "http://origin.hostname.example.com:8080",
},
{
url: "http://origin.hostname.example.com:8080/api",
expectedScheme: "http",
expectedBasePath: "http://origin.hostname.example.com:8080/api",
},
{
url: "http://origin.hostname.example.com:8080/api/",
expectedScheme: "http",
expectedBasePath: "http://origin.hostname.example.com:8080/api",
},
{
url: "https://origin.hostname.example.com:8080/api",
expectedScheme: "https",
expectedBasePath: "https://origin.hostname.example.com:8080/api",
},
}
// use multiple schemes to demonstrate that the URL is based on the
// origin's scheme, not the :scheme header
for _, testScheme := range []string{"http", "https"} {
for _, testOrigin := range testOrigins {
assertion := func(testPath string) bool {
const expectedMethod = "POST"
const expectedHostname = "request.hostname.example.com"
h2 := []h2mux.Header{
{Name: ":method", Value: expectedMethod},
{Name: ":scheme", Value: testScheme},
{Name: ":authority", Value: expectedHostname},
{Name: ":path", Value: testPath},
{Name: RequestUserHeaders, Value: ""},
}
h1, err := http.NewRequest("GET", testOrigin.url, nil)
require.NoError(t, err)
err = H2RequestHeadersToH1Request(h2, h1)
return assert.NoError(t, err) &&
assert.Equal(t, expectedMethod, h1.Method) &&
assert.Equal(t, expectedHostname, h1.Host) &&
assert.Equal(t, testOrigin.expectedScheme, h1.URL.Scheme) &&
assert.Equal(t, testOrigin.expectedBasePath+testPath, h1.URL.String())
}
err := quick.Check(assertion, config)
assert.NoError(t, err)
}
}
}
func randomASCIIPrintableChar(rand *rand.Rand) int {
// smallest printable ASCII char is 32, largest is 126
const startPrintable = 32
const endPrintable = 127
return startPrintable + rand.Intn(endPrintable-startPrintable)
}
// randomASCIIText generates an ASCII string, some of whose characters may be
// percent-encoded. Its "logical length" (ignoring percent-encoding) is
// between 1 and `maxLength`.
func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
length := minLength + rand.Intn(maxLength)
var result strings.Builder
for i := 0; i < length; i++ {
c := randomASCIIPrintableChar(rand)
// 1/4 chance of using percent encoding when not necessary
if c == '%' || rand.Intn(4) == 0 {
result.WriteString(fmt.Sprintf("%%%02X", c))
} else {
result.WriteByte(byte(c))
}
}
return result.String()
}
// Calls `randomASCIIText` and ensures the result is a valid URL path,
// i.e. one that can pass unchanged through url.URL.String()
func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
text := randomASCIIText(rand, minLength, maxLength)
re, err := regexp.Compile("[^/;,]*")
require.NoError(t, err)
return "/" + re.ReplaceAllStringFunc(text, url.PathEscape)
}
// Calls `randomASCIIText` and ensures the result is a valid URL query,
// i.e. one that can pass unchanged through url.URL.String()
func randomHTTP1Query(rand *rand.Rand, minLength int, maxLength int) string {
text := randomASCIIText(rand, minLength, maxLength)
return "?" + strings.ReplaceAll(text, "#", "%23")
}
// Calls `randomASCIIText` and ensures the result is a valid URL fragment,
// i.e. one that can pass unchanged through url.URL.String()
func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
text := randomASCIIText(rand, minLength, maxLength)
u, err := url.Parse("#" + text)
require.NoError(t, err)
return u.String()
}
// Assemble a random :path pseudoheader that is legal by Go stdlib standards
// (i.e. all characters will satisfy "net/url".shouldEscape for their respective locations)
func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
result := randomHTTP1Path(t, rand, 1, 64)
if rand.Intn(2) == 1 {
result += randomHTTP1Query(rand, 1, 32)
}
if rand.Intn(2) == 1 {
result += randomHTTP1Fragment(t, rand, 1, 16)
}
return result
}
func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) {
for name, values := range headers {
for _, value := range values {
h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value})
}
}
return h2muxHeaders
}
func TestParseRequestHeaders(t *testing.T) {
mockUserHeadersToSerialize := http.Header{
"Mock-Header-One": {"1", "1.5"},
"Mock-Header-Two": {"2"},
"Mock-Header-Three": {"3"},
}
mockHeaders := []h2mux.Header{
{Name: "One", Value: "1"}, // will be dropped
{Name: "Cf-Two", Value: "cf-value-1"},
{Name: "Cf-Two", Value: "cf-value-2"},
{Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)},
}
expectedHeaders := []h2mux.Header{
{Name: "Cf-Two", Value: "cf-value-1"},
{Name: "Cf-Two", Value: "cf-value-2"},
{Name: "Mock-Header-One", Value: "1"},
{Name: "Mock-Header-One", Value: "1.5"},
{Name: "Mock-Header-Two", Value: "2"},
{Name: "Mock-Header-Three", Value: "3"},
}
h1 := &http.Request{
Header: make(http.Header),
}
err := H2RequestHeadersToH1Request(mockHeaders, h1)
assert.NoError(t, err)
assert.ElementsMatch(t, expectedHeaders, stdlibHeaderToH2muxHeader(h1.Header))
}
func TestIsH2muxControlRequestHeader(t *testing.T) {
controlRequestHeaders := []string{
// Anything that begins with cf-
"cf-sample-header",
// Any http2 pseudoheader
":sample-pseudo-header",
// content-length is a special case, it has to be there
// for some requests to work (per the HTTP2 spec)
"content-length",
// Websocket request headers
"connection",
"upgrade",
}
for _, header := range controlRequestHeaders {
assert.True(t, IsH2muxControlRequestHeader(header))
}
}
func TestIsH2muxControlResponseHeader(t *testing.T) {
controlResponseHeaders := []string{
// Anything that begins with cf-int- or cf-cloudflared-
"cf-int-sample-header",
"cf-cloudflared-sample-header",
// Any http2 pseudoheader
":sample-pseudo-header",
// content-length is a special case, it has to be there
// for some requests to work (per the HTTP2 spec)
"content-length",
}
for _, header := range controlResponseHeaders {
assert.True(t, IsH2muxControlResponseHeader(header))
}
}
func TestIsNotH2muxControlRequestHeader(t *testing.T) {
notControlRequestHeaders := []string{
"mock-header",
"another-sample-header",
}
for _, header := range notControlRequestHeaders {
assert.False(t, IsH2muxControlRequestHeader(header))
}
}
func TestIsNotH2muxControlResponseHeader(t *testing.T) {
notControlResponseHeaders := []string{
"mock-header",
"another-sample-header",
"upgrade",
"connection",
"cf-whatever", // On the response path, we only want to filter cf-int- and cf-cloudflared-
}
for _, header := range notControlResponseHeaders {
assert.False(t, IsH2muxControlResponseHeader(header))
}
}
func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
mockHeaders := http.Header{
"User-header-one": {""},
"User-header-two": {"1", "2"},
"cf-header": {"cf-value"},
"cf-int-header": {"cf-int-value"},
"cf-cloudflared-header": {"cf-cloudflared-value"},
"Content-Length": {"123"},
}
mockResponse := http.Response{
StatusCode: 200,
Header: mockHeaders,
}
headers := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
serializedHeadersIndex := -1
for i, header := range headers {
if header.Name == ResponseUserHeaders {
serializedHeadersIndex = i
break
}
}
assert.NotEqual(t, -1, serializedHeadersIndex)
actualControlHeaders := append(
headers[:serializedHeadersIndex],
headers[serializedHeadersIndex+1:]...,
)
expectedControlHeaders := []h2mux.Header{
{Name: ":status", Value: "200"},
{Name: "content-length", Value: "123"},
}
assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
expectedUserHeaders := []h2mux.Header{
{Name: "User-header-one", Value: ""},
{Name: "User-header-two", Value: "1"},
{Name: "User-header-two", Value: "2"},
{Name: "cf-header", Value: "cf-value"},
}
assert.NoError(t, err)
assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders)
}
// The purpose of this test is to check that our code and the http.Header
// implementation don't throw validation errors about header size
func TestHeaderSize(t *testing.T) {
largeValue := randSeq(5 * 1024 * 1024) // 5Mb
largeHeaders := http.Header{
"User-header": {largeValue},
}
mockResponse := http.Response{
StatusCode: 200,
Header: largeHeaders,
}
serializedHeaders := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
assert.NoError(t, err)
for _, header := range serializedHeaders {
request.Header.Set(header.Name, header.Value)
}
for _, header := range serializedHeaders {
if header.Name != ResponseUserHeaders {
continue
}
deserializedHeaders, err := DeserializeHeaders(header.Value)
assert.NoError(t, err)
assert.Equal(t, largeValue, deserializedHeaders[0].Value)
}
}
func randSeq(n int) string {
randomizer := rand.New(rand.NewSource(17))
var letters = []rune(":;,+/=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
b := make([]rune, n)
for i := range b {
b[i] = letters[randomizer.Intn(len(letters))]
}
return string(b)
}
func BenchmarkH1ResponseToH2ResponseHeaders(b *testing.B) {
ser := "eC1mb3J3YXJkZWQtcHJvdG8:aHR0cHM;dXBncmFkZS1pbnNlY3VyZS1yZXF1ZXN0cw:MQ;YWNjZXB0LWxhbmd1YWdl:ZW4tVVMsZW47cT0wLjkscnU7cT0wLjg;YWNjZXB0LWVuY29kaW5n:Z3ppcA;eC1mb3J3YXJkZWQtZm9y:MTczLjI0NS42MC42;dXNlci1hZ2VudA:TW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTBfMTRfNikgQXBwbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzg0LjAuNDE0Ny44OSBTYWZhcmkvNTM3LjM2;c2VjLWZldGNoLW1vZGU:bmF2aWdhdGU;Y2RuLWxvb3A:Y2xvdWRmbGFyZQ;c2VjLWZldGNoLWRlc3Q:ZG9jdW1lbnQ;c2VjLWZldGNoLXVzZXI:PzE;c2VjLWZldGNoLXNpdGU:bm9uZQ;Y29va2ll:X19jZmR1aWQ9ZGNkOWZjOGNjNWMxMzE0NTMyYTFkMjhlZDEyOWRhOTYwMTU2OTk1MTYzNDsgX19jZl9ibT1mYzY2MzMzYzAzZmM0MWFiZTZmOWEyYzI2ZDUwOTA0YzIxYzZhMTQ2LTE1OTU2MjIzNDEtMTgwMC1BZTVzS2pIU2NiWGVFM05mMUhrTlNQMG1tMHBLc2pQWkloVnM1Z2g1SkNHQkFhS1UxVDB2b003alBGN3FjMHVSR2NjZGcrWHdhL1EzbTJhQzdDVU4xZ2M9;YWNjZXB0:dGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksaW1hZ2Uvd2VicCxpbWFnZS9hcG5nLCovKjtxPTAuOCxhcHBsaWNhdGlvbi9zaWduZWQtZXhjaGFuZ2U7dj1iMztxPTAuOQ"
h2, _ := DeserializeHeaders(ser)
h1 := make(http.Header)
for _, header := range h2 {
h1.Add(header.Name, header.Value)
}
h1.Add("Content-Length", "200")
h1.Add("Cf-Something", "Else")
h1.Add("Upgrade", "websocket")
h1resp := &http.Response{
StatusCode: 200,
Header: h1,
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = H1ResponseToH2ResponseHeaders(h1resp.StatusCode, h1resp.Header)
}
}

19
connection/header.go
View File

@ -7,17 +7,15 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/cloudflare/cloudflared/h2mux"
)
var (
// h2mux-style special headers
// internal special headers
RequestUserHeaders = "cf-cloudflared-request-headers"
ResponseUserHeaders = "cf-cloudflared-response-headers"
ResponseMetaHeader = "cf-cloudflared-response-meta"
// h2mux-style special headers
// internal special headers
CanonicalResponseUserHeaders = http.CanonicalHeaderKey(ResponseUserHeaders)
CanonicalResponseMetaHeader = http.CanonicalHeaderKey(ResponseMetaHeader)
)
@ -28,6 +26,13 @@ var (
responseMetaHeaderOrigin = mustInitRespMetaHeader("origin")
)
// HTTPHeader is a custom header struct that expects only ever one value for the header.
// This structure is used to serialize the headers and attach them to the HTTP2 request when proxying.
type HTTPHeader struct {
Name string
Value string
}
type responseMetaHeader struct {
Source string `json:"src"`
}
@ -104,10 +109,10 @@ func SerializeHeaders(h1Headers http.Header) string {
}
// Deserialize headers serialized by `SerializeHeader`
func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) {
func DeserializeHeaders(serializedHeaders string) ([]HTTPHeader, error) {
const unableToDeserializeErr = "Unable to deserialize headers"
var deserialized []h2mux.Header
var deserialized []HTTPHeader
for _, serializedPair := range strings.Split(serializedHeaders, ";") {
if len(serializedPair) == 0 {
continue
@ -130,7 +135,7 @@ func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) {
return nil, errors.Wrap(err, unableToDeserializeErr)
}
deserialized = append(deserialized, h2mux.Header{
deserialized = append(deserialized, HTTPHeader{
Name: string(deserializedName),
Value: string(deserializedValue),
})

30
connection/header_test.go
View File

@ -46,18 +46,40 @@ func TestSerializeHeaders(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 13, len(deserializedHeaders))
h2muxExpectedHeaders := stdlibHeaderToH2muxHeader(mockHeaders)
expectedHeaders := headerToReqHeader(mockHeaders)
sort.Sort(ByName(deserializedHeaders))
sort.Sort(ByName(h2muxExpectedHeaders))
sort.Sort(ByName(expectedHeaders))
assert.True(
t,
reflect.DeepEqual(h2muxExpectedHeaders, deserializedHeaders),
fmt.Sprintf("got = %#v, want = %#v\n", deserializedHeaders, h2muxExpectedHeaders),
reflect.DeepEqual(expectedHeaders, deserializedHeaders),
fmt.Sprintf("got = %#v, want = %#v\n", deserializedHeaders, expectedHeaders),
)
}
type ByName []HTTPHeader
func (a ByName) Len() int { return len(a) }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool {
if a[i].Name == a[j].Name {
return a[i].Value < a[j].Value
}
return a[i].Name < a[j].Name
}
func headerToReqHeader(headers http.Header) (reqHeaders []HTTPHeader) {
for name, values := range headers {
for _, value := range values {
reqHeaders = append(reqHeaders, HTTPHeader{Name: name, Value: value})
}
}
return reqHeaders
}
func TestSerializeNoHeaders(t *testing.T) {
request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
assert.NoError(t, err)

3
connection/http2.go
View File

@ -385,8 +385,7 @@ func determineHTTP2Type(r *http.Request) Type {
func handleMissingRequestParts(connType Type, r *http.Request) {
if connType == TypeHTTP {
// http library has no guarantees that we receive a filled URL. If not, then we fill it, as we reuse the request
// for proxying. We use the same values as we used to in h2mux. For proxying they should not matter since we
// control the dialer on every egress proxied.
// for proxying. For proxying they should not matter since we control the dialer on every egress proxied.
if len(r.URL.Scheme) == 0 {
r.URL.Scheme = "http"
}

276
connection/metrics.go
View File

@ -2,11 +2,8 @@ package connection
import (
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/cloudflare/cloudflared/h2mux"
)
const (
@ -16,27 +13,6 @@ const (
configSubsystem = "config"
)
type muxerMetrics struct {
rtt *prometheus.GaugeVec
rttMin *prometheus.GaugeVec
rttMax *prometheus.GaugeVec
receiveWindowAve *prometheus.GaugeVec
sendWindowAve *prometheus.GaugeVec
receiveWindowMin *prometheus.GaugeVec
receiveWindowMax *prometheus.GaugeVec
sendWindowMin *prometheus.GaugeVec
sendWindowMax *prometheus.GaugeVec
inBoundRateCurr *prometheus.GaugeVec
inBoundRateMin *prometheus.GaugeVec
inBoundRateMax *prometheus.GaugeVec
outBoundRateCurr *prometheus.GaugeVec
outBoundRateMin *prometheus.GaugeVec
outBoundRateMax *prometheus.GaugeVec
compBytesBefore *prometheus.GaugeVec
compBytesAfter *prometheus.GaugeVec
compRateAve *prometheus.GaugeVec
}
type localConfigMetrics struct {
pushes prometheus.Counter
pushesErrors prometheus.Counter
@ -53,7 +29,6 @@ type tunnelMetrics struct {
regFail *prometheus.CounterVec
rpcFail *prometheus.CounterVec
muxerMetrics *muxerMetrics
tunnelsHA tunnelsForHA
userHostnamesCounts *prometheus.CounterVec
@ -91,252 +66,6 @@ func newLocalConfigMetrics() *localConfigMetrics {
}
}
func newMuxerMetrics() *muxerMetrics {
rtt := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "rtt",
Help: "Round-trip time in millisecond",
},
[]string{"connection_id"},
)
prometheus.MustRegister(rtt)
rttMin := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "rtt_min",
Help: "Shortest round-trip time in millisecond",
},
[]string{"connection_id"},
)
prometheus.MustRegister(rttMin)
rttMax := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "rtt_max",
Help: "Longest round-trip time in millisecond",
},
[]string{"connection_id"},
)
prometheus.MustRegister(rttMax)
receiveWindowAve := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "receive_window_ave",
Help: "Average receive window size in bytes",
},
[]string{"connection_id"},
)
prometheus.MustRegister(receiveWindowAve)
sendWindowAve := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "send_window_ave",
Help: "Average send window size in bytes",
},
[]string{"connection_id"},
)
prometheus.MustRegister(sendWindowAve)
receiveWindowMin := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "receive_window_min",
Help: "Smallest receive window size in bytes",
},
[]string{"connection_id"},
)
prometheus.MustRegister(receiveWindowMin)
receiveWindowMax := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "receive_window_max",
Help: "Largest receive window size in bytes",
},
[]string{"connection_id"},
)
prometheus.MustRegister(receiveWindowMax)
sendWindowMin := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "send_window_min",
Help: "Smallest send window size in bytes",
},
[]string{"connection_id"},
)
prometheus.MustRegister(sendWindowMin)
sendWindowMax := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "send_window_max",
Help: "Largest send window size in bytes",
},
[]string{"connection_id"},
)
prometheus.MustRegister(sendWindowMax)
inBoundRateCurr := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "inbound_bytes_per_sec_curr",
Help: "Current inbounding bytes per second, 0 if there is no incoming connection",
},
[]string{"connection_id"},
)
prometheus.MustRegister(inBoundRateCurr)
inBoundRateMin := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "inbound_bytes_per_sec_min",
Help: "Minimum non-zero inbounding bytes per second",
},
[]string{"connection_id"},
)
prometheus.MustRegister(inBoundRateMin)
inBoundRateMax := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "inbound_bytes_per_sec_max",
Help: "Maximum inbounding bytes per second",
},
[]string{"connection_id"},
)
prometheus.MustRegister(inBoundRateMax)
outBoundRateCurr := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "outbound_bytes_per_sec_curr",
Help: "Current outbounding bytes per second, 0 if there is no outgoing traffic",
},
[]string{"connection_id"},
)
prometheus.MustRegister(outBoundRateCurr)
outBoundRateMin := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "outbound_bytes_per_sec_min",
Help: "Minimum non-zero outbounding bytes per second",
},
[]string{"connection_id"},
)
prometheus.MustRegister(outBoundRateMin)
outBoundRateMax := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "outbound_bytes_per_sec_max",
Help: "Maximum outbounding bytes per second",
},
[]string{"connection_id"},
)
prometheus.MustRegister(outBoundRateMax)
compBytesBefore := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "comp_bytes_before",
Help: "Bytes sent via cross-stream compression, pre compression",
},
[]string{"connection_id"},
)
prometheus.MustRegister(compBytesBefore)
compBytesAfter := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "comp_bytes_after",
Help: "Bytes sent via cross-stream compression, post compression",
},
[]string{"connection_id"},
)
prometheus.MustRegister(compBytesAfter)
compRateAve := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Subsystem: muxerSubsystem,
Name: "comp_rate_ave",
Help: "Average outbound cross-stream compression ratio",
},
[]string{"connection_id"},
)
prometheus.MustRegister(compRateAve)
return &muxerMetrics{
rtt: rtt,
rttMin: rttMin,
rttMax: rttMax,
receiveWindowAve: receiveWindowAve,
sendWindowAve: sendWindowAve,
receiveWindowMin: receiveWindowMin,
receiveWindowMax: receiveWindowMax,
sendWindowMin: sendWindowMin,
sendWindowMax: sendWindowMax,
inBoundRateCurr: inBoundRateCurr,
inBoundRateMin: inBoundRateMin,
inBoundRateMax: inBoundRateMax,
outBoundRateCurr: outBoundRateCurr,
outBoundRateMin: outBoundRateMin,
outBoundRateMax: outBoundRateMax,
compBytesBefore: compBytesBefore,
compBytesAfter: compBytesAfter,
compRateAve: compRateAve,
}
}
func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
}
func convertRTTMilliSec(t time.Duration) float64 {
return float64(t / time.Millisecond)
}
// Metrics that can be collected without asking the edge
func initTunnelMetrics() *tunnelMetrics {
maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
@ -408,7 +137,6 @@ func initTunnelMetrics() *tunnelMetrics {
return &tunnelMetrics{
serverLocations: serverLocations,
oldServerLocations: make(map[string]string),
muxerMetrics: newMuxerMetrics(),
tunnelsHA: newTunnelsForHA(),
regSuccess: registerSuccess,
regFail: registerFail,
@ -418,10 +146,6 @@ func initTunnelMetrics() *tunnelMetrics {
}
}
func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
t.muxerMetrics.update(connectionID, metrics)
}
func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
t.locationLock.Lock()
defer t.locationLock.Unlock()

2
connection/protocol.go
View File

@ -13,7 +13,7 @@ import (
const (
AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge"
// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge (unused, but kept for legacy reference).
edgeH2muxTLSServerName = "cftunnel.com"
// edgeH2TLSServerName is the server name to establish http2 connection with edge
edgeH2TLSServerName = "h2.cftunnel.com"

623
connection/quic.go
View File

@ -1,114 +1,38 @@
package connection
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/quic-go/quic-go"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
"github.com/cloudflare/cloudflared/datagramsession"
"github.com/cloudflare/cloudflared/ingress"
"github.com/cloudflare/cloudflared/management"
"github.com/cloudflare/cloudflared/packet"
cfdquic "github.com/cloudflare/cloudflared/quic"
"github.com/cloudflare/cloudflared/tracing"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
)
const (
// HTTPHeaderKey is used to get or set http headers in QUIC ALPN if the underlying proxy connection type is HTTP.
HTTPHeaderKey = "HttpHeader"
// HTTPMethodKey is used to get or set http method in QUIC ALPN if the underlying proxy connection type is HTTP.
HTTPMethodKey = "HttpMethod"
// HTTPHostKey is used to get or set http Method in QUIC ALPN if the underlying proxy connection type is HTTP.
HTTPHostKey = "HttpHost"
// HTTPRequestBodyHintKey is used in ConnectRequest metadata to indicate if the request has body
HTTPRequestBodyHintKey = "HttpReqBodyHint"
QUICMetadataFlowID = "FlowID"
// emperically this capacity has been working well
demuxChanCapacity = 16
)
type RequestBodyHint uint64
const (
RequestBodyHintMissing RequestBodyHint = iota
RequestBodyHintEmpty
RequestBodyHintHasData
)
func (rbh RequestBodyHint) String() string {
return [...]string{"missing", "empty", "data"}[rbh]
}
var (
portForConnIndex = make(map[uint8]int, 0)
portMapMutex sync.Mutex
)
// QUICConnection represents the type that facilitates Proxying via QUIC streams.
type QUICConnection struct {
session quic.Connection
logger *zerolog.Logger
orchestrator Orchestrator
// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
sessionManager datagramsession.Manager
// datagramMuxer mux/demux datagrams from quic connection
datagramMuxer *cfdquic.DatagramMuxerV2
packetRouter *ingress.PacketRouter
controlStreamHandler ControlStreamHandler
connOptions *tunnelpogs.ConnectionOptions
connIndex uint8
rpcTimeout time.Duration
streamWriteTimeout time.Duration
gracePeriod time.Duration
}
// NewQUICConnection returns a new instance of QUICConnection.
func NewQUICConnection(
func DialQuic(
ctx context.Context,
quicConfig *quic.Config,
edgeAddr net.Addr,
tlsConfig *tls.Config,
edgeAddr netip.AddrPort,
localAddr net.IP,
connIndex uint8,
tlsConfig *tls.Config,
orchestrator Orchestrator,
connOptions *tunnelpogs.ConnectionOptions,
controlStreamHandler ControlStreamHandler,
logger *zerolog.Logger,
packetRouterConfig *ingress.GlobalRouterConfig,
rpcTimeout time.Duration,
streamWriteTimeout time.Duration,
gracePeriod time.Duration,
) (*QUICConnection, error) {
udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
) (quic.Connection, error) {
udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, edgeAddr, logger)
if err != nil {
return nil, err
}
session, err := quic.Dial(ctx, udpConn, edgeAddr, tlsConfig, quicConfig)
conn, err := quic.Dial(ctx, udpConn, net.UDPAddrFromAddrPort(edgeAddr), tlsConfig, quicConfig)
if err != nil {
// close the udp server socket in case of error connecting to the edge
udpConn.Close()
@ -116,543 +40,22 @@ func NewQUICConnection(
}
// wrap the session, so that the UDPConn is closed after session is closed.
session = &wrapCloseableConnQuicConnection{
session,
conn = &wrapCloseableConnQuicConnection{
conn,
udpConn,
}
sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
datagramMuxer := cfdquic.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger)
return &QUICConnection{
session: session,
orchestrator: orchestrator,
logger: logger,
sessionManager: sessionManager,
datagramMuxer: datagramMuxer,
packetRouter: packetRouter,
controlStreamHandler: controlStreamHandler,
connOptions: connOptions,
connIndex: connIndex,
rpcTimeout: rpcTimeout,
streamWriteTimeout: streamWriteTimeout,
gracePeriod: gracePeriod,
}, nil
}
// Serve starts a QUIC session that begins accepting streams.
func (q *QUICConnection) Serve(ctx context.Context) error {
// origintunneld assumes the first stream is used for the control plane
controlStream, err := q.session.OpenStream()
if err != nil {
return fmt.Errorf("failed to open a registration control stream: %w", err)
}
// If either goroutine returns nil error, we rely on this cancellation to make sure the other goroutine exits
// as fast as possible as well. Nil error means we want to exit for good (caller code won't retry serving this
// connection).
// If either goroutine returns a non nil error, then the error group cancels the context, thus also canceling the
// other goroutine as fast as possible.
ctx, cancel := context.WithCancel(ctx)
errGroup, ctx := errgroup.WithContext(ctx)
// In the future, if cloudflared can autonomously push traffic to the edge, we have to make sure the control
// stream is already fully registered before the other goroutines can proceed.
errGroup.Go(func() error {
// err is equal to nil if we exit due to unregistration. If that happens we want to wait the full
// amount of the grace period, allowing requests to finish before we cancel the context, which will
// make cloudflared exit.
if err := q.serveControlStream(ctx, controlStream); err == nil {
select {
case <-ctx.Done():
case <-time.Tick(q.gracePeriod):
}
}
cancel()
return err
})
errGroup.Go(func() error {
defer cancel()
return q.acceptStream(ctx)
})
errGroup.Go(func() error {
defer cancel()
return q.sessionManager.Serve(ctx)
})
errGroup.Go(func() error {
defer cancel()
return q.datagramMuxer.ServeReceive(ctx)
})
errGroup.Go(func() error {
defer cancel()
return q.packetRouter.Serve(ctx)
})
return errGroup.Wait()
}
func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream quic.Stream) error {
// This blocks until the control plane is done.
err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions, q.orchestrator)
if err != nil {
// Not wrapping error here to be consistent with the http2 message.
return err
}
return nil
}
// Close closes the session with no errors specified.
func (q *QUICConnection) Close() {
q.session.CloseWithError(0, "")
}
func (q *QUICConnection) acceptStream(ctx context.Context) error {
defer q.Close()
for {
quicStream, err := q.session.AcceptStream(ctx)
if err != nil {
// context.Canceled is usually a user ctrl+c. We don't want to log an error here as it's intentional.
if errors.Is(err, context.Canceled) || q.controlStreamHandler.IsStopped() {
return nil
}
return fmt.Errorf("failed to accept QUIC stream: %w", err)
}
go q.runStream(quicStream)
}
}
func (q *QUICConnection) runStream(quicStream quic.Stream) {
ctx := quicStream.Context()
stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
defer stream.Close()
// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
// code executed in the code path of handleStream don't trigger an earlier close to the downstream write stream.
// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
// A call to close will simulate a close to the read-side, which will fail subsequent reads.
noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
ss := rpcquic.NewCloudflaredServer(q.handleDataStream, q, q, q.rpcTimeout)
if err := ss.Serve(ctx, noCloseStream); err != nil {
q.logger.Debug().Err(err).Msg("Failed to handle QUIC stream")
// if we received an error at this level, then close write side of stream with an error, which will result in
// RST_STREAM frame.
quicStream.CancelWrite(0)
}
}
func (q *QUICConnection) handleDataStream(ctx context.Context, stream *rpcquic.RequestServerStream) error {
request, err := stream.ReadConnectRequestData()
if err != nil {
return err
}
if err, connectResponseSent := q.dispatchRequest(ctx, stream, err, request); err != nil {
q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
// if the connectResponse was already sent and we had an error, we need to propagate it up, so that the stream is
// closed with an RST_STREAM frame
if connectResponseSent {
return err
}
if writeRespErr := stream.WriteConnectResponseData(err); writeRespErr != nil {
return writeRespErr
}
}
return nil
}
// dispatchRequest will dispatch the request depending on the type and returns an error if it occurs.
// More importantly, it also tells if the during processing of the request the ConnectResponse metadata was sent downstream.
// This is important since it informs
func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *rpcquic.RequestServerStream, err error, request *pogs.ConnectRequest) (error, bool) {
originProxy, err := q.orchestrator.GetOriginProxy()
if err != nil {
return err, false
}
switch request.Type {
case pogs.ConnectionTypeHTTP, pogs.ConnectionTypeWebsocket:
tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
if err != nil {
return err, false
}
w := newHTTPResponseAdapter(stream)
return originProxy.ProxyHTTP(&w, tracedReq, request.Type == pogs.ConnectionTypeWebsocket), w.connectResponseSent
case pogs.ConnectionTypeTCP:
rwa := &streamReadWriteAcker{RequestServerStream: stream}
metadata := request.MetadataMap()
return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
Dest: request.Dest,
FlowID: metadata[QUICMetadataFlowID],
CfTraceID: metadata[tracing.TracerContextName],
ConnIndex: q.connIndex,
}), rwa.connectResponseSent
default:
return errors.Errorf("unsupported error type: %s", request.Type), false
}
}
// RegisterUdpSession is the RPC method invoked by edge to register and run a session
func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
traceCtx := tracing.NewTracedContext(ctx, traceContext, q.logger)
ctx, registerSpan := traceCtx.Tracer().Start(traceCtx, "register-session", trace.WithAttributes(
attribute.String("session-id", sessionID.String()),
attribute.String("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)),
))
log := q.logger.With().Int(management.EventTypeKey, int(management.UDP)).Logger()
// Each session is a series of datagram from an eyeball to a dstIP:dstPort.
// (src port, dst IP, dst port) uniquely identifies a session, so it needs a dedicated connected socket.
originProxy, err := ingress.DialUDP(dstIP, dstPort)
if err != nil {
log.Err(err).Msgf("Failed to create udp proxy to %s:%d", dstIP, dstPort)
tracing.EndWithErrorStatus(registerSpan, err)
return nil, err
}
registerSpan.SetAttributes(
attribute.Bool("socket-bind-success", true),
attribute.String("src", originProxy.LocalAddr().String()),
)
session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
if err != nil {
originProxy.Close()
log.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
tracing.EndWithErrorStatus(registerSpan, err)
return nil, err
}
go q.serveUDPSession(session, closeAfterIdleHint)
log.Debug().
Str("sessionID", sessionID.String()).
Str("src", originProxy.LocalAddr().String()).
Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).
Msgf("Registered session")
tracing.End(registerSpan)
resp := tunnelpogs.RegisterUdpSessionResponse{
Spans: traceCtx.GetProtoSpans(),
}
return &resp, nil
}
func (q *QUICConnection) serveUDPSession(session *datagramsession.Session, closeAfterIdleHint time.Duration) {
ctx := q.session.Context()
closedByRemote, err := session.Serve(ctx, closeAfterIdleHint)
// If session is terminated by remote, then we know it has been unregistered from session manager and edge
if !closedByRemote {
if err != nil {
q.closeUDPSession(ctx, session.ID, err.Error())
} else {
q.closeUDPSession(ctx, session.ID, "terminated without error")
}
}
q.logger.Debug().Err(err).
Int(management.EventTypeKey, int(management.UDP)).
Str("sessionID", session.ID.String()).
Msg("Session terminated")
}
// closeUDPSession first unregisters the session from session manager, then it tries to unregister from edge
func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUID, message string) {
q.sessionManager.UnregisterSession(ctx, sessionID, message, false)
quicStream, err := q.session.OpenStream()
if err != nil {
// Log this at debug because this is not an error if session was closed due to lost connection
// with edge
q.logger.Debug().Err(err).
Int(management.EventTypeKey, int(management.UDP)).
Str("sessionID", sessionID.String()).
Msgf("Failed to open quic stream to unregister udp session with edge")
return
}
stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
defer stream.Close()
rpcClientStream, err := rpcquic.NewSessionClient(ctx, stream, q.rpcTimeout)
if err != nil {
// Log this at debug because this is not an error if session was closed due to lost connection
// with edge
q.logger.Err(err).Str("sessionID", sessionID.String()).
Msgf("Failed to open rpc stream to unregister udp session with edge")
return
}
defer rpcClientStream.Close()
if err := rpcClientStream.UnregisterUdpSession(ctx, sessionID, message); err != nil {
q.logger.Err(err).Str("sessionID", sessionID.String()).
Msgf("Failed to unregister udp session with edge")
}
}
// UnregisterUdpSession is the RPC method invoked by edge to unregister and terminate a sesssion
func (q *QUICConnection) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
return q.sessionManager.UnregisterSession(ctx, sessionID, message, true)
}
// UpdateConfiguration is the RPC method invoked by edge when there is a new configuration
func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
return q.orchestrator.UpdateConfig(version, config)
}
// streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
// the client.
type streamReadWriteAcker struct {
*rpcquic.RequestServerStream
connectResponseSent bool
}
// AckConnection acks response back to the proxy.
func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
metadata := []pogs.Metadata{}
// Only add tracing if provided by origintunneld
if tracePropagation != "" {
metadata = append(metadata, pogs.Metadata{
Key: tracing.CanonicalCloudflaredTracingHeader,
Val: tracePropagation,
})
}
s.connectResponseSent = true
return s.WriteConnectResponseData(nil, metadata...)
}
// httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
type httpResponseAdapter struct {
*rpcquic.RequestServerStream
headers http.Header
connectResponseSent bool
}
func newHTTPResponseAdapter(s *rpcquic.RequestServerStream) httpResponseAdapter {
return httpResponseAdapter{RequestServerStream: s, headers: make(http.Header)}
}
func (hrw *httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
// we do not support trailers over QUIC
}
func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
metadata := make([]pogs.Metadata, 0)
metadata = append(metadata, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
for k, vv := range header {
for _, v := range vv {
httpHeaderKey := fmt.Sprintf("%s:%s", HTTPHeaderKey, k)
metadata = append(metadata, pogs.Metadata{Key: httpHeaderKey, Val: v})
}
}
return hrw.WriteConnectResponseData(nil, metadata...)
}
func (hrw *httpResponseAdapter) Write(p []byte) (int, error) {
// Make sure to send WriteHeader response if not called yet
if !hrw.connectResponseSent {
hrw.WriteRespHeaders(http.StatusOK, hrw.headers)
}
return hrw.RequestServerStream.Write(p)
}
func (hrw *httpResponseAdapter) Header() http.Header {
return hrw.headers
}
// This is a no-op Flush because this adapter is over a quic.Stream and we don't need Flush here.
func (hrw *httpResponseAdapter) Flush() {}
func (hrw *httpResponseAdapter) WriteHeader(status int) {
hrw.WriteRespHeaders(status, hrw.headers)
}
func (hrw *httpResponseAdapter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
conn := &localProxyConnection{hrw.ReadWriteCloser}
readWriter := bufio.NewReadWriter(
bufio.NewReader(hrw.ReadWriteCloser),
bufio.NewWriter(hrw.ReadWriteCloser),
)
return conn, readWriter, nil
}
func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
hrw.WriteConnectResponseData(err, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
}
func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...pogs.Metadata) error {
hrw.connectResponseSent = true
return hrw.RequestServerStream.WriteConnectResponseData(respErr, metadata...)
}
func buildHTTPRequest(
ctx context.Context,
connectRequest *pogs.ConnectRequest,
body io.ReadCloser,
connIndex uint8,
log *zerolog.Logger,
) (*tracing.TracedHTTPRequest, error) {
metadata := connectRequest.MetadataMap()
dest := connectRequest.Dest
method := metadata[HTTPMethodKey]
host := metadata[HTTPHostKey]
req, err := http.NewRequestWithContext(ctx, method, dest, body)
if err != nil {
return nil, err
}
req.Host = host
for _, metadata := range connectRequest.Metadata {
if strings.Contains(metadata.Key, HTTPHeaderKey) {
// metadata.Key is off the format httpHeaderKey:<HTTPHeader>
httpHeaderKey := strings.Split(metadata.Key, ":")
if len(httpHeaderKey) != 2 {
return nil, fmt.Errorf("header Key: %s malformed", metadata.Key)
}
req.Header.Add(httpHeaderKey[1], metadata.Val)
}
}
// Go's http.Client automatically sends chunked request body if this value is not set on the
// *http.Request struct regardless of header:
// https://go.googlesource.com/go/+/go1.8rc2/src/net/http/transfer.go#154.
if err := setContentLength(req); err != nil {
return nil, fmt.Errorf("Error setting content-length: %w", err)
}
if shouldSetRequestBodyToEmpty(connectRequest, metadata, req) {
log.Debug().Str("host", req.Host).Str("method", req.Method).Msg("Set request to have no body")
req.Body = http.NoBody
}
stripWebsocketUpgradeHeader(req)
// Check for tracing on request
tracedReq := tracing.NewTracedHTTPRequest(req, connIndex, log)
return tracedReq, err
}
func setContentLength(req *http.Request) error {
var err error
if contentLengthStr := req.Header.Get("Content-Length"); contentLengthStr != "" {
req.ContentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
}
return err
}
func isTransferEncodingChunked(req *http.Request) bool {
transferEncodingVal := req.Header.Get("Transfer-Encoding")
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding suggests that this can be a comma
// separated value as well.
return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
}
// Borrowed from https://github.com/golang/go/blob/go1.22.6/src/net/http/request.go#L1541
func requestMethodUsuallyLacksBody(req *http.Request) bool {
switch strings.ToUpper(req.Method) {
case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH":
return true
}
return false
}
func shouldSetRequestBodyToEmpty(connectRequest *pogs.ConnectRequest, metadata map[string]string, req *http.Request) bool {
switch metadata[HTTPRequestBodyHintKey] {
case RequestBodyHintEmpty.String():
return true
case RequestBodyHintHasData.String():
return false
default:
}
isWebsocket := connectRequest.Type == pogs.ConnectionTypeWebsocket
// Go's client defaults to chunked encoding after a 200ms delay if the following cases are true:
// * the request body blocks
// * the content length is not set (or set to -1)
// * the method doesn't usually have a body (GET, HEAD, DELETE, ...)
// * there is no transfer-encoding=chunked already set.
// So, if transfer cannot be chunked and content length is 0, we dont set a request body.
// Reference: https://github.com/golang/go/blob/go1.22.2/src/net/http/transfer.go#L192-L206
return !isWebsocket && requestMethodUsuallyLacksBody(req) && !isTransferEncodingChunked(req) && req.ContentLength == 0
}
// A helper struct that guarantees a call to close only affects read side, but not write side.
type nopCloserReadWriter struct {
io.ReadWriteCloser
// for use by Read only
// we don't need a memory barrier here because there is an implicit assumption that
// Read calls can't happen concurrently by different go-routines.
sawEOF bool
// should be updated and read using atomic primitives.
// value is read in Read method and written in Close method, which could be done by different
// go-routines.
closed uint32
}
func (np *nopCloserReadWriter) Read(p []byte) (n int, err error) {
if np.sawEOF {
return 0, io.EOF
}
if atomic.LoadUint32(&np.closed) > 0 {
return 0, fmt.Errorf("closed by handler")
}
n, err = np.ReadWriteCloser.Read(p)
if err == io.EOF {
np.sawEOF = true
}
return
}
func (np *nopCloserReadWriter) Close() error {
atomic.StoreUint32(&np.closed, 1)
return nil
}
// muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
type muxerWrapper struct {
muxer *cfdquic.DatagramMuxerV2
}
func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
return rp.muxer.SendPacket(cfdquic.RawPacket(pk))
}
func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
pk, err := rp.muxer.ReceivePacket(ctx)
if err != nil {
return packet.RawPacket{}, err
}
rawPacket, ok := pk.(cfdquic.RawPacket)
if ok {
return packet.RawPacket(rawPacket), nil
}
return packet.RawPacket{}, fmt.Errorf("unexpected packet type %+v", pk)
}
func (rp *muxerWrapper) Close() error {
return nil
return conn, nil
}
func createUDPConnForConnIndex(connIndex uint8, localIP net.IP, logger *zerolog.Logger) (*net.UDPConn, error) {
func createUDPConnForConnIndex(connIndex uint8, localIP net.IP, edgeIP netip.AddrPort, logger *zerolog.Logger) (*net.UDPConn, error) {
portMapMutex.Lock()
defer portMapMutex.Unlock()
if localIP == nil {
localIP = net.IPv4zero
}
listenNetwork := "udp"
// https://github.com/quic-go/quic-go/issues/3793 DF bit cannot be set for dual stack listener on OSX
// https://github.com/quic-go/quic-go/issues/3793 DF bit cannot be set for dual stack listener ("udp") on macOS,
// to set the DF bit properly, the network string needs to be specific to the IP family.
if runtime.GOOS == "darwin" {
if localIP.To4() != nil {
if edgeIP.Addr().Is4() {
listenNetwork = "udp4"
} else {
listenNetwork = "udp6"

444
connection/quic_connection.go Normal file
View File

@ -0,0 +1,444 @@
package connection
import (
"bufio"
"context"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/pkg/errors"
"github.com/quic-go/quic-go"
"github.com/rs/zerolog"
"golang.org/x/sync/errgroup"
"github.com/cloudflare/cloudflared/packet"
cfdquic "github.com/cloudflare/cloudflared/quic"
"github.com/cloudflare/cloudflared/tracing"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
)
const (
// HTTPHeaderKey is used to get or set http headers in QUIC ALPN if the underlying proxy connection type is HTTP.
HTTPHeaderKey = "HttpHeader"
// HTTPMethodKey is used to get or set http method in QUIC ALPN if the underlying proxy connection type is HTTP.
HTTPMethodKey = "HttpMethod"
// HTTPHostKey is used to get or set http host in QUIC ALPN if the underlying proxy connection type is HTTP.
HTTPHostKey = "HttpHost"
QUICMetadataFlowID = "FlowID"
)
// quicConnection represents the type that facilitates Proxying via QUIC streams.
type quicConnection struct {
conn quic.Connection
logger *zerolog.Logger
orchestrator Orchestrator
datagramHandler DatagramSessionHandler
controlStreamHandler ControlStreamHandler
connOptions *tunnelpogs.ConnectionOptions
connIndex uint8
rpcTimeout time.Duration
streamWriteTimeout time.Duration
gracePeriod time.Duration
}
// NewTunnelConnection takes a [quic.Connection] to wrap it for use with cloudflared application logic.
func NewTunnelConnection(
ctx context.Context,
conn quic.Connection,
connIndex uint8,
orchestrator Orchestrator,
datagramSessionHandler DatagramSessionHandler,
controlStreamHandler ControlStreamHandler,
connOptions *pogs.ConnectionOptions,
rpcTimeout time.Duration,
streamWriteTimeout time.Duration,
gracePeriod time.Duration,
logger *zerolog.Logger,
) (TunnelConnection, error) {
return &quicConnection{
conn: conn,
logger: logger,
orchestrator: orchestrator,
datagramHandler: datagramSessionHandler,
controlStreamHandler: controlStreamHandler,
connOptions: connOptions,
connIndex: connIndex,
rpcTimeout: rpcTimeout,
streamWriteTimeout: streamWriteTimeout,
gracePeriod: gracePeriod,
}, nil
}
// Serve starts a QUIC connection that begins accepting streams.
func (q *quicConnection) Serve(ctx context.Context) error {
// The edge assumes the first stream is used for the control plane
controlStream, err := q.conn.OpenStream()
if err != nil {
return fmt.Errorf("failed to open a registration control stream: %w", err)
}
// If either goroutine returns nil error, we rely on this cancellation to make sure the other goroutine exits
// as fast as possible as well. Nil error means we want to exit for good (caller code won't retry serving this
// connection).
// If either goroutine returns a non nil error, then the error group cancels the context, thus also canceling the
// other goroutine as fast as possible.
ctx, cancel := context.WithCancel(ctx)
errGroup, ctx := errgroup.WithContext(ctx)
// In the future, if cloudflared can autonomously push traffic to the edge, we have to make sure the control
// stream is already fully registered before the other goroutines can proceed.
errGroup.Go(func() error {
// err is equal to nil if we exit due to unregistration. If that happens we want to wait the full
// amount of the grace period, allowing requests to finish before we cancel the context, which will
// make cloudflared exit.
if err := q.serveControlStream(ctx, controlStream); err == nil {
select {
case <-ctx.Done():
case <-time.Tick(q.gracePeriod):
}
}
cancel()
return err
})
errGroup.Go(func() error {
defer cancel()
return q.acceptStream(ctx)
})
errGroup.Go(func() error {
defer cancel()
return q.datagramHandler.Serve(ctx)
})
return errGroup.Wait()
}
// serveControlStream will serve the RPC; blocking until the control plane is done.
func (q *quicConnection) serveControlStream(ctx context.Context, controlStream quic.Stream) error {
return q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions, q.orchestrator)
}
// Close the connection with no errors specified.
func (q *quicConnection) Close() {
q.conn.CloseWithError(0, "")
}
func (q *quicConnection) acceptStream(ctx context.Context) error {
defer q.Close()
for {
quicStream, err := q.conn.AcceptStream(ctx)
if err != nil {
// context.Canceled is usually a user ctrl+c. We don't want to log an error here as it's intentional.
if errors.Is(err, context.Canceled) || q.controlStreamHandler.IsStopped() {
return nil
}
return fmt.Errorf("failed to accept QUIC stream: %w", err)
}
go q.runStream(quicStream)
}
}
func (q *quicConnection) runStream(quicStream quic.Stream) {
ctx := quicStream.Context()
stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
defer stream.Close()
// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
// code executed in the code path of handleStream don't trigger an earlier close to the downstream write stream.
// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
// A call to close will simulate a close to the read-side, which will fail subsequent reads.
noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
ss := rpcquic.NewCloudflaredServer(q.handleDataStream, q.datagramHandler, q, q.rpcTimeout)
if err := ss.Serve(ctx, noCloseStream); err != nil {
q.logger.Debug().Err(err).Msg("Failed to handle QUIC stream")
// if we received an error at this level, then close write side of stream with an error, which will result in
// RST_STREAM frame.
quicStream.CancelWrite(0)
}
}
func (q *quicConnection) handleDataStream(ctx context.Context, stream *rpcquic.RequestServerStream) error {
request, err := stream.ReadConnectRequestData()
if err != nil {
return err
}
if err, connectResponseSent := q.dispatchRequest(ctx, stream, request); err != nil {
q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
// if the connectResponse was already sent and we had an error, we need to propagate it up, so that the stream is
// closed with an RST_STREAM frame
if connectResponseSent {
return err
}
if writeRespErr := stream.WriteConnectResponseData(err); writeRespErr != nil {
return writeRespErr
}
}
return nil
}
// dispatchRequest will dispatch the request to the origin depending on the type and returns an error if it occurs.
// Also returns if the connect response was sent to the downstream during processing of the origin request.
func (q *quicConnection) dispatchRequest(ctx context.Context, stream *rpcquic.RequestServerStream, request *pogs.ConnectRequest) (err error, connectResponseSent bool) {
originProxy, err := q.orchestrator.GetOriginProxy()
if err != nil {
return err, false
}
switch request.Type {
case pogs.ConnectionTypeHTTP, pogs.ConnectionTypeWebsocket:
tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
if err != nil {
return err, false
}
w := newHTTPResponseAdapter(stream)
return originProxy.ProxyHTTP(&w, tracedReq, request.Type == pogs.ConnectionTypeWebsocket), w.connectResponseSent
case pogs.ConnectionTypeTCP:
rwa := &streamReadWriteAcker{RequestServerStream: stream}
metadata := request.MetadataMap()
return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
Dest: request.Dest,
FlowID: metadata[QUICMetadataFlowID],
CfTraceID: metadata[tracing.TracerContextName],
ConnIndex: q.connIndex,
}), rwa.connectResponseSent
default:
return errors.Errorf("unsupported error type: %s", request.Type), false
}
}
// UpdateConfiguration is the RPC method invoked by edge when there is a new configuration
func (q *quicConnection) UpdateConfiguration(ctx context.Context, version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
return q.orchestrator.UpdateConfig(version, config)
}
// streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
// the client.
type streamReadWriteAcker struct {
*rpcquic.RequestServerStream
connectResponseSent bool
}
// AckConnection acks response back to the proxy.
func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
metadata := []pogs.Metadata{}
// Only add tracing if provided by the edge request
if tracePropagation != "" {
metadata = append(metadata, pogs.Metadata{
Key: tracing.CanonicalCloudflaredTracingHeader,
Val: tracePropagation,
})
}
s.connectResponseSent = true
return s.WriteConnectResponseData(nil, metadata...)
}
// httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
type httpResponseAdapter struct {
*rpcquic.RequestServerStream
headers http.Header
connectResponseSent bool
}
func newHTTPResponseAdapter(s *rpcquic.RequestServerStream) httpResponseAdapter {
return httpResponseAdapter{RequestServerStream: s, headers: make(http.Header)}
}
func (hrw *httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
// we do not support trailers over QUIC
}
func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
metadata := make([]pogs.Metadata, 0)
metadata = append(metadata, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
for k, vv := range header {
for _, v := range vv {
httpHeaderKey := fmt.Sprintf("%s:%s", HTTPHeaderKey, k)
metadata = append(metadata, pogs.Metadata{Key: httpHeaderKey, Val: v})
}
}
return hrw.WriteConnectResponseData(nil, metadata...)
}
func (hrw *httpResponseAdapter) Write(p []byte) (int, error) {
// Make sure to send WriteHeader response if not called yet
if !hrw.connectResponseSent {
hrw.WriteRespHeaders(http.StatusOK, hrw.headers)
}
return hrw.RequestServerStream.Write(p)
}
func (hrw *httpResponseAdapter) Header() http.Header {
return hrw.headers
}
// This is a no-op Flush because this adapter is over a quic.Stream and we don't need Flush here.
func (hrw *httpResponseAdapter) Flush() {}
func (hrw *httpResponseAdapter) WriteHeader(status int) {
hrw.WriteRespHeaders(status, hrw.headers)
}
func (hrw *httpResponseAdapter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
conn := &localProxyConnection{hrw.ReadWriteCloser}
readWriter := bufio.NewReadWriter(
bufio.NewReader(hrw.ReadWriteCloser),
bufio.NewWriter(hrw.ReadWriteCloser),
)
return conn, readWriter, nil
}
func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
hrw.WriteConnectResponseData(err, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
}
func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...pogs.Metadata) error {
hrw.connectResponseSent = true
return hrw.RequestServerStream.WriteConnectResponseData(respErr, metadata...)
}
func buildHTTPRequest(
ctx context.Context,
connectRequest *pogs.ConnectRequest,
body io.ReadCloser,
connIndex uint8,
log *zerolog.Logger,
) (*tracing.TracedHTTPRequest, error) {
metadata := connectRequest.MetadataMap()
dest := connectRequest.Dest
method := metadata[HTTPMethodKey]
host := metadata[HTTPHostKey]
isWebsocket := connectRequest.Type == pogs.ConnectionTypeWebsocket
req, err := http.NewRequestWithContext(ctx, method, dest, body)
if err != nil {
return nil, err
}
req.Host = host
for _, metadata := range connectRequest.Metadata {
if strings.Contains(metadata.Key, HTTPHeaderKey) {
// metadata.Key is off the format httpHeaderKey:<HTTPHeader>
httpHeaderKey := strings.Split(metadata.Key, ":")
if len(httpHeaderKey) != 2 {
return nil, fmt.Errorf("header Key: %s malformed", metadata.Key)
}
req.Header.Add(httpHeaderKey[1], metadata.Val)
}
}
// Go's http.Client automatically sends chunked request body if this value is not set on the
// *http.Request struct regardless of header:
// https://go.googlesource.com/go/+/go1.8rc2/src/net/http/transfer.go#154.
if err := setContentLength(req); err != nil {
return nil, fmt.Errorf("Error setting content-length: %w", err)
}
// Go's client defaults to chunked encoding after a 200ms delay if the following cases are true:
// * the request body blocks
// * the content length is not set (or set to -1)
// * the method doesn't usually have a body (GET, HEAD, DELETE, ...)
// * there is no transfer-encoding=chunked already set.
// So, if transfer cannot be chunked and content length is 0, we dont set a request body.
if !isWebsocket && !isTransferEncodingChunked(req) && req.ContentLength == 0 {
req.Body = http.NoBody
}
stripWebsocketUpgradeHeader(req)
// Check for tracing on request
tracedReq := tracing.NewTracedHTTPRequest(req, connIndex, log)
return tracedReq, err
}
func setContentLength(req *http.Request) error {
var err error
if contentLengthStr := req.Header.Get("Content-Length"); contentLengthStr != "" {
req.ContentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
}
return err
}
func isTransferEncodingChunked(req *http.Request) bool {
transferEncodingVal := req.Header.Get("Transfer-Encoding")
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding suggests that this can be a comma
// separated value as well.
return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
}
// A helper struct that guarantees a call to close only affects read side, but not write side.
type nopCloserReadWriter struct {
io.ReadWriteCloser
// for use by Read only
// we don't need a memory barrier here because there is an implicit assumption that
// Read calls can't happen concurrently by different go-routines.
sawEOF bool
// should be updated and read using atomic primitives.
// value is read in Read method and written in Close method, which could be done by different
// go-routines.
closed uint32
}
func (np *nopCloserReadWriter) Read(p []byte) (n int, err error) {
if np.sawEOF {
return 0, io.EOF
}
if atomic.LoadUint32(&np.closed) > 0 {
return 0, fmt.Errorf("closed by handler")
}
n, err = np.ReadWriteCloser.Read(p)
if err == io.EOF {
np.sawEOF = true
}
return
}
func (np *nopCloserReadWriter) Close() error {
atomic.StoreUint32(&np.closed, 1)
return nil
}
// muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
type muxerWrapper struct {
muxer *cfdquic.DatagramMuxerV2
}
func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
return rp.muxer.SendPacket(cfdquic.RawPacket(pk))
}
func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
pk, err := rp.muxer.ReceivePacket(ctx)
if err != nil {
return packet.RawPacket{}, err
}
rawPacket, ok := pk.(cfdquic.RawPacket)
if ok {
return packet.RawPacket(rawPacket), nil
}
return packet.RawPacket{}, fmt.Errorf("unexpected packet type %+v", pk)
}
func (rp *muxerWrapper) Close() error {
return nil
}

216
connection/quic_test.go → connection/quic_connection_test.go
View File

@ -13,8 +13,8 @@ import (
"math/big"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"strings"
"testing"
"time"
@ -26,12 +26,14 @@ import (
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/nettest"
"github.com/cloudflare/cloudflared/datagramsession"
"github.com/cloudflare/cloudflared/ingress"
"github.com/cloudflare/cloudflared/packet"
cfdquic "github.com/cloudflare/cloudflared/quic"
"github.com/cloudflare/cloudflared/tracing"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
)
@ -162,11 +164,11 @@ func TestQUICServer(t *testing.T) {
close(serverDone)
}()
qc := testQUICConnection(udpListener.LocalAddr(), t, uint8(i))
tunnelConn, _ := testTunnelConnection(t, netip.MustParseAddrPort(udpListener.LocalAddr().String()), uint8(i))
connDone := make(chan struct{})
go func() {
qc.Serve(ctx)
tunnelConn.Serve(ctx)
close(connDone)
}()
@ -484,125 +486,6 @@ func TestBuildHTTPRequest(t *testing.T) {
},
body: io.NopCloser(&bytes.Buffer{}),
},
{
name: "if edge sends the body is empty hint, set body to empty",
connectRequest: &pogs.ConnectRequest{
Dest: "http://test.com",
Metadata: []pogs.Metadata{
{
Key: "HttpHeader:Another-Header",
Val: "Misc",
},
{
Key: "HttpHost",
Val: "cf.host",
},
{
Key: "HttpMethod",
Val: "put",
},
{
Key: HTTPRequestBodyHintKey,
Val: RequestBodyHintEmpty.String(),
},
},
},
req: &http.Request{
Method: "put",
URL: &url.URL{
Scheme: "http",
Host: "test.com",
},
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
Header: http.Header{
"Another-Header": []string{"Misc"},
},
ContentLength: 0,
Host: "cf.host",
Body: http.NoBody,
},
body: io.NopCloser(&bytes.Buffer{}),
},
{
name: "if edge sends the body has data hint, don't set body to empty",
connectRequest: &pogs.ConnectRequest{
Dest: "http://test.com",
Metadata: []pogs.Metadata{
{
Key: "HttpHeader:Another-Header",
Val: "Misc",
},
{
Key: "HttpHost",
Val: "cf.host",
},
{
Key: "HttpMethod",
Val: "put",
},
{
Key: HTTPRequestBodyHintKey,
Val: RequestBodyHintHasData.String(),
},
},
},
req: &http.Request{
Method: "put",
URL: &url.URL{
Scheme: "http",
Host: "test.com",
},
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
Header: http.Header{
"Another-Header": []string{"Misc"},
},
ContentLength: 0,
Host: "cf.host",
Body: io.NopCloser(&bytes.Buffer{}),
},
body: io.NopCloser(&bytes.Buffer{}),
},
{
name: "if the http method usually has body, don't set body to empty",
connectRequest: &pogs.ConnectRequest{
Dest: "http://test.com",
Metadata: []pogs.Metadata{
{
Key: "HttpHeader:Another-Header",
Val: "Misc",
},
{
Key: "HttpHost",
Val: "cf.host",
},
{
Key: "HttpMethod",
Val: "post",
},
},
},
req: &http.Request{
Method: "post",
URL: &url.URL{
Scheme: "http",
Host: "test.com",
},
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
Header: http.Header{
"Another-Header": []string{"Misc"},
},
ContentLength: 0,
Host: "cf.host",
Body: io.NopCloser(&bytes.Buffer{}),
},
body: io.NopCloser(&bytes.Buffer{}),
},
}
log := zerolog.Nop()
@ -632,7 +515,6 @@ func TestServeUDPSession(t *testing.T) {
defer udpListener.Close()
ctx, cancel := context.WithCancel(context.Background())
val := udpListener.LocalAddr()
// Establish QUIC connection with edge
edgeQUICSessionChan := make(chan quic.Connection)
@ -646,13 +528,14 @@ func TestServeUDPSession(t *testing.T) {
}()
// Random index to avoid reusing port
qc := testQUICConnection(val, t, 28)
go qc.Serve(ctx)
tunnelConn, datagramConn := testTunnelConnection(t, netip.MustParseAddrPort(udpListener.LocalAddr().String()), 28)
go tunnelConn.Serve(ctx)
edgeQUICSession := <-edgeQUICSessionChan
serveSession(ctx, qc, edgeQUICSession, closedByOrigin, io.EOF.Error(), t)
serveSession(ctx, qc, edgeQUICSession, closedByTimeout, datagramsession.SessionIdleErr(time.Millisecond*50).Error(), t)
serveSession(ctx, qc, edgeQUICSession, closedByRemote, "eyeball closed connection", t)
serveSession(ctx, datagramConn, edgeQUICSession, closedByOrigin, io.EOF.Error(), t)
serveSession(ctx, datagramConn, edgeQUICSession, closedByTimeout, datagramsession.SessionIdleErr(time.Millisecond*50).Error(), t)
serveSession(ctx, datagramConn, edgeQUICSession, closedByRemote, "eyeball closed connection", t)
cancel()
}
@ -695,8 +578,20 @@ func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
}
func TestCreateUDPConnReuseSourcePort(t *testing.T) {
edgeIPv4 := netip.MustParseAddrPort("0.0.0.0:0")
edgeIPv6 := netip.MustParseAddrPort("[::]:0")
// We assume the test environment has access to an IPv4 interface
testCreateUDPConnReuseSourcePortForEdgeIP(t, edgeIPv4)
if nettest.SupportsIPv6() {
testCreateUDPConnReuseSourcePortForEdgeIP(t, edgeIPv6)
}
}
func testCreateUDPConnReuseSourcePortForEdgeIP(t *testing.T, edgeIP netip.AddrPort) {
logger := zerolog.Nop()
conn, err := createUDPConnForConnIndex(0, nil, &logger)
conn, err := createUDPConnForConnIndex(0, nil, edgeIP, &logger)
require.NoError(t, err)
getPortFunc := func(conn *net.UDPConn) int {
@ -710,34 +605,34 @@ func TestCreateUDPConnReuseSourcePort(t *testing.T) {
conn.Close()
// should get the same port as before.
conn, err = createUDPConnForConnIndex(0, nil, &logger)
conn, err = createUDPConnForConnIndex(0, nil, edgeIP, &logger)
require.NoError(t, err)
require.Equal(t, initialPort, getPortFunc(conn))
// new index, should get a different port
conn1, err := createUDPConnForConnIndex(1, nil, &logger)
conn1, err := createUDPConnForConnIndex(1, nil, edgeIP, &logger)
require.NoError(t, err)
require.NotEqual(t, initialPort, getPortFunc(conn1))
// not closing the conn and trying to obtain a new conn for same index should give a different random port
conn, err = createUDPConnForConnIndex(0, nil, &logger)
conn, err = createUDPConnForConnIndex(0, nil, edgeIP, &logger)
require.NoError(t, err)
require.NotEqual(t, initialPort, getPortFunc(conn))
}
func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
func serveSession(ctx context.Context, datagramConn *datagramV2Connection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
var (
payload = []byte(t.Name())
)
sessionID := uuid.New()
cfdConn, originConn := net.Pipe()
// Registers and run a new session
session, err := qc.sessionManager.RegisterSession(ctx, sessionID, cfdConn)
session, err := datagramConn.sessionManager.RegisterSession(ctx, sessionID, cfdConn)
require.NoError(t, err)
sessionDone := make(chan struct{})
go func() {
qc.serveUDPSession(session, time.Millisecond*50)
datagramConn.serveUDPSession(session, time.Millisecond*50)
close(sessionDone)
}()
@ -761,7 +656,7 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
case closedByOrigin:
originConn.Close()
case closedByRemote:
err = qc.UnregisterUdpSession(ctx, sessionID, expectedReason)
err = datagramConn.UnregisterUdpSession(ctx, sessionID, expectedReason)
require.NoError(t, err)
case closedByTimeout:
}
@ -832,33 +727,58 @@ func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionI
return nil
}
func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QUICConnection {
func testTunnelConnection(t *testing.T, serverAddr netip.AddrPort, index uint8) (TunnelConnection, *datagramV2Connection) {
tlsClientConfig := &tls.Config{
InsecureSkipVerify: true,
NextProtos: []string{"argotunnel"},
}
// Start a mock httpProxy
log := zerolog.New(os.Stdout)
log := zerolog.New(io.Discard)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
qc, err := NewQUICConnection(
// Dial the QUIC connection to the edge
conn, err := DialQuic(
ctx,
testQUICConfig,
udpListenerAddr,
nil,
index,
tlsClientConfig,
&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
&tunnelpogs.ConnectionOptions{},
fakeControlStream{},
serverAddr,
nil, // connect on a random port
index,
&log,
nil,
)
// Start a session manager for the connection
sessionDemuxChan := make(chan *packet.Session, 4)
datagramMuxer := cfdquic.NewDatagramMuxerV2(conn, &log, sessionDemuxChan)
sessionManager := datagramsession.NewManager(&log, datagramMuxer.SendToSession, sessionDemuxChan)
packetRouter := ingress.NewPacketRouter(nil, datagramMuxer, &log)
datagramConn := &datagramV2Connection{
conn,
sessionManager,
datagramMuxer,
packetRouter,
15 * time.Second,
0 * time.Second,
&log,
}
tunnelConn, err := NewTunnelConnection(
ctx,
conn,
index,
&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
datagramConn,
fakeControlStream{},
&pogs.ConnectionOptions{},
15*time.Second,
0*time.Second,
0*time.Second,
&log,
)
require.NoError(t, err)
return qc
return tunnelConn, datagramConn
}
type mockReaderNoopWriter struct {

200
connection/quic_datagram_v2.go Normal file
View File

@ -0,0 +1,200 @@
package connection
import (
"context"
"fmt"
"net"
"time"
"github.com/google/uuid"
"github.com/quic-go/quic-go"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
"github.com/cloudflare/cloudflared/datagramsession"
"github.com/cloudflare/cloudflared/ingress"
"github.com/cloudflare/cloudflared/management"
"github.com/cloudflare/cloudflared/packet"
cfdquic "github.com/cloudflare/cloudflared/quic"
"github.com/cloudflare/cloudflared/tracing"
"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
)
const (
// emperically this capacity has been working well
demuxChanCapacity = 16
)
// DatagramSessionHandler is a service that can serve datagrams for a connection and handle sessions from incoming
// connection streams.
type DatagramSessionHandler interface {
Serve(context.Context) error
pogs.SessionManager
}
type datagramV2Connection struct {
conn quic.Connection
// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
sessionManager datagramsession.Manager
// datagramMuxer mux/demux datagrams from quic connection
datagramMuxer *cfdquic.DatagramMuxerV2
packetRouter *ingress.PacketRouter
rpcTimeout time.Duration
streamWriteTimeout time.Duration
logger *zerolog.Logger
}
func NewDatagramV2Connection(ctx context.Context,
conn quic.Connection,
packetConfig *ingress.GlobalRouterConfig,
rpcTimeout time.Duration,
streamWriteTimeout time.Duration,
logger *zerolog.Logger,
) DatagramSessionHandler {
sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
datagramMuxer := cfdquic.NewDatagramMuxerV2(conn, logger, sessionDemuxChan)
sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
packetRouter := ingress.NewPacketRouter(packetConfig, datagramMuxer, logger)
return &datagramV2Connection{
conn,
sessionManager,
datagramMuxer,
packetRouter,
rpcTimeout,
streamWriteTimeout,
logger,
}
}
func (d *datagramV2Connection) Serve(ctx context.Context) error {
// If either goroutine returns nil error, we rely on this cancellation to make sure the other goroutine exits
// as fast as possible as well. Nil error means we want to exit for good (caller code won't retry serving this
// connection).
// If either goroutine returns a non nil error, then the error group cancels the context, thus also canceling the
// other goroutine as fast as possible.
ctx, cancel := context.WithCancel(ctx)
errGroup, ctx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
defer cancel()
return d.sessionManager.Serve(ctx)
})
errGroup.Go(func() error {
defer cancel()
return d.datagramMuxer.ServeReceive(ctx)
})
errGroup.Go(func() error {
defer cancel()
return d.packetRouter.Serve(ctx)
})
return errGroup.Wait()
}
// RegisterUdpSession is the RPC method invoked by edge to register and run a session
func (q *datagramV2Connection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
traceCtx := tracing.NewTracedContext(ctx, traceContext, q.logger)
ctx, registerSpan := traceCtx.Tracer().Start(traceCtx, "register-session", trace.WithAttributes(
attribute.String("session-id", sessionID.String()),
attribute.String("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)),
))
log := q.logger.With().Int(management.EventTypeKey, int(management.UDP)).Logger()
// Each session is a series of datagram from an eyeball to a dstIP:dstPort.
// (src port, dst IP, dst port) uniquely identifies a session, so it needs a dedicated connected socket.
originProxy, err := ingress.DialUDP(dstIP, dstPort)
if err != nil {
log.Err(err).Msgf("Failed to create udp proxy to %s:%d", dstIP, dstPort)
tracing.EndWithErrorStatus(registerSpan, err)
return nil, err
}
registerSpan.SetAttributes(
attribute.Bool("socket-bind-success", true),
attribute.String("src", originProxy.LocalAddr().String()),
)
session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
if err != nil {
originProxy.Close()
log.Err(err).Str(datagramsession.LogFieldSessionID, datagramsession.FormatSessionID(sessionID)).Msgf("Failed to register udp session")
tracing.EndWithErrorStatus(registerSpan, err)
return nil, err
}
go q.serveUDPSession(session, closeAfterIdleHint)
log.Debug().
Str(datagramsession.LogFieldSessionID, datagramsession.FormatSessionID(sessionID)).
Str("src", originProxy.LocalAddr().String()).
Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).
Msgf("Registered session")
tracing.End(registerSpan)
resp := tunnelpogs.RegisterUdpSessionResponse{
Spans: traceCtx.GetProtoSpans(),
}
return &resp, nil
}
// UnregisterUdpSession is the RPC method invoked by edge to unregister and terminate a sesssion
func (q *datagramV2Connection) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
return q.sessionManager.UnregisterSession(ctx, sessionID, message, true)
}
func (q *datagramV2Connection) serveUDPSession(session *datagramsession.Session, closeAfterIdleHint time.Duration) {
ctx := q.conn.Context()
closedByRemote, err := session.Serve(ctx, closeAfterIdleHint)
// If session is terminated by remote, then we know it has been unregistered from session manager and edge
if !closedByRemote {
if err != nil {
q.closeUDPSession(ctx, session.ID, err.Error())
} else {
q.closeUDPSession(ctx, session.ID, "terminated without error")
}
}
q.logger.Debug().Err(err).
Int(management.EventTypeKey, int(management.UDP)).
Str(datagramsession.LogFieldSessionID, datagramsession.FormatSessionID(session.ID)).
Msg("Session terminated")
}
// closeUDPSession first unregisters the session from session manager, then it tries to unregister from edge
func (q *datagramV2Connection) closeUDPSession(ctx context.Context, sessionID uuid.UUID, message string) {
q.sessionManager.UnregisterSession(ctx, sessionID, message, false)
quicStream, err := q.conn.OpenStream()
if err != nil {
// Log this at debug because this is not an error if session was closed due to lost connection
// with edge
q.logger.Debug().Err(err).
Int(management.EventTypeKey, int(management.UDP)).
Str(datagramsession.LogFieldSessionID, datagramsession.FormatSessionID(sessionID)).
Msgf("Failed to open quic stream to unregister udp session with edge")
return
}
stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
defer stream.Close()
rpcClientStream, err := rpcquic.NewSessionClient(ctx, stream, q.rpcTimeout)
if err != nil {
// Log this at debug because this is not an error if session was closed due to lost connection
// with edge
q.logger.Err(err).Str(datagramsession.LogFieldSessionID, datagramsession.FormatSessionID(sessionID)).
Msgf("Failed to open rpc stream to unregister udp session with edge")
return
}
defer rpcClientStream.Close()
if err := rpcClientStream.UnregisterUdpSession(ctx, sessionID, message); err != nil {
q.logger.Err(err).Str(datagramsession.LogFieldSessionID, datagramsession.FormatSessionID(sessionID)).
Msgf("Failed to unregister udp session with edge")
}
}

12
datagramsession/manager.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"strings"
"time"
"github.com/google/uuid"
@ -20,8 +21,15 @@ const (
var (
errSessionManagerClosed = fmt.Errorf("session manager closed")
LogFieldSessionID = "sessionID"
)
func FormatSessionID(sessionID uuid.UUID) string {
sessionIDStr := sessionID.String()
sessionIDStr = strings.ReplaceAll(sessionIDStr, "-", "")
return sessionIDStr
}
// Manager defines the APIs to manage sessions from the same transport.
type Manager interface {
// Serve starts the event loop
@ -127,7 +135,7 @@ func (m *manager) registerSession(ctx context.Context, registration *registerSes
func (m *manager) newSession(id uuid.UUID, dstConn io.ReadWriteCloser) *Session {
logger := m.log.With().
Int(management.EventTypeKey, int(management.UDP)).
Str("sessionID", id.String()).Logger()
Str(LogFieldSessionID, FormatSessionID(id)).Logger()
return &Session{
ID: id,
sendFunc: m.sendFunc,
@ -174,7 +182,7 @@ func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
func (m *manager) sendToSession(datagram *packet.Session) {
session, ok := m.sessions[datagram.ID]
if !ok {
m.log.Error().Str("sessionID", datagram.ID.String()).Msg("session not found")
m.log.Error().Str(LogFieldSessionID, FormatSessionID(datagram.ID)).Msg("session not found")
return
}
// session writes to destination over a connected UDP socket, which should not be blocking, so this call doesn't

4
edgediscovery/dial.go
View File

@ -9,7 +9,7 @@ import (
"github.com/pkg/errors"
)
// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node
// DialEdge makes a TLS connection to a Cloudflare edge node
func DialEdge(
ctx context.Context,
timeout time.Duration,
@ -36,7 +36,7 @@ func DialEdge(
if err = tlsEdgeConn.Handshake(); err != nil {
return nil, newDialError(err, "TLS handshake with edge error")
}
// clear the deadline on the conn; h2mux has its own timeouts
// clear the deadline on the conn; http2 has its own timeouts
tlsEdgeConn.SetDeadline(time.Time{})
return tlsEdgeConn, nil
}

7
github_release.py
View File

@ -254,7 +254,7 @@ def assert_asset_version(binary_path, release_version):
binary_version = get_binary_version(binary_path)
elif ext == '.tgz':
tar = tarfile.open(binary_path, "r:gz")
tar.extractall("tmp", filter='data')
tar.extractall("tmp")
tar.close()
binary_path = os.path.join(os.getcwd(), 'tmp', 'cloudflared')
binary_version = get_binary_version(binary_path)
@ -286,21 +286,20 @@ def main():
else:
client = Github(args.api_key)
repo = client.get_repo(CLOUDFLARED_REPO)
release = get_or_create_release(repo, args.release_version, args.dry_run)
if os.path.isdir(args.path):
onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))]
for filename in onlyfiles:
binary_path = os.path.join(args.path, filename)
assert_asset_version(binary_path, args.release_version)
release = get_or_create_release(repo, args.release_version, args.dry_run)
for filename in onlyfiles:
binary_path = os.path.join(args.path, filename)
upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,
args.kv_api_token)
move_asset(binary_path, filename)
else:
upload_asset(release, args.path, args.name, args.release_version, args.kv_account_id, args.namespace_id,
args.kv_api_token)
raise Exception("the argument path must be a directory")
except Exception as e:
logging.exception(e)

29
go.mod
View File

@ -3,7 +3,7 @@ module github.com/cloudflare/cloudflared
go 1.22
require (
github.com/coredns/coredns v1.10.0
github.com/coredns/coredns v1.11.3
github.com/coreos/go-oidc/v3 v3.10.0
github.com/coreos/go-systemd/v22 v22.5.0
github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@ -13,18 +13,17 @@ require (
github.com/go-chi/chi/v5 v5.0.8
github.com/go-chi/cors v1.2.1
github.com/go-jose/go-jose/v4 v4.0.1
github.com/gobwas/ws v1.0.4
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
github.com/gobwas/ws v1.2.1
github.com/google/gopacket v1.1.19
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.4.2
github.com/json-iterator/go v1.1.12
github.com/mattn/go-colorable v0.1.13
github.com/miekg/dns v1.1.50
github.com/miekg/dns v1.1.58
github.com/mitchellh/go-homedir v1.1.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/client_model v0.5.0
github.com/prometheus/client_model v0.6.0
github.com/quic-go/quic-go v0.45.0
github.com/rs/zerolog v1.20.0
github.com/stretchr/testify v1.9.0
@ -55,7 +54,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coredns/caddy v1.1.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
@ -64,36 +63,36 @@ require (
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/onsi/ginkgo/v2 v2.9.5 // indirect
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/common v0.53.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
go.opentelemetry.io/otel/metric v1.26.0 // indirect
go.uber.org/mock v0.4.0 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/oauth2 v0.17.0 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/tools v0.21.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/grpc v1.63.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/grpc v1.63.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

91
go.sum
View File

@ -7,13 +7,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
github.com/coredns/coredns v1.10.0 h1:jCfuWsBjTs0dapkkhISfPCzn5LqvSRtrFtaf/Tjj4DI=
github.com/coredns/coredns v1.10.0/go.mod h1:CIfRU5TgpuoIiJBJ4XrofQzfFQpPFh32ERpUevrSlaw=
github.com/coredns/coredns v1.11.3 h1:8RjnpZc42db5th84/QJKH2i137ecJdzZK1HJwhetSPk=
github.com/coredns/coredns v1.11.3/go.mod h1:lqFkDsHjEUdY7LJ75Nib3lwqJGip6ewWOqNIf8OavIQ=
github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU=
github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@ -24,8 +21,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
@ -75,19 +73,18 @@ github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk=
github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@ -102,8 +99,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo=
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -114,7 +111,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0Q
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM=
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d h1:PRDnysJ9dF1vUMmEzBu6aHQeUluSQy4eWH3RsSSy/vI=
github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -140,10 +136,10 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -152,16 +148,16 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg=
github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas=
github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -171,10 +167,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE=
github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/quic-go/quic-go v0.45.0 h1:OHmkQGM37luZITyTSu6ff03HP/2IrwDX1ZFiNEhSFUE=
@ -195,14 +191,13 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
@ -233,39 +228,32 @@ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJ
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -275,7 +263,6 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
@ -287,24 +274,20 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo=
google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0=
google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
google.golang.org/grpc v1.63.0 h1:WjKe+dnvABXyPJMD7KDNLxtoGk5tgk+YFWN6cBWjZE8=
google.golang.org/grpc v1.63.0/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ=
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=

195
h2mux/activestreammap.go
View File

@ -1,195 +0,0 @@
package h2mux
import (
"sync"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/net/http2"
)
var (
ActiveStreams = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "cloudflared",
Subsystem: "tunnel",
Name: "active_streams",
Help: "Number of active streams created by all muxers.",
})
)
func init() {
prometheus.MustRegister(ActiveStreams)
}
// activeStreamMap is used to moderate access to active streams between the read and write
// threads, and deny access to new peer streams while shutting down.
type activeStreamMap struct {
sync.RWMutex
// streams tracks open streams.
streams map[uint32]*MuxedStream
// nextStreamID is the next ID to use on our side of the connection.
// This is odd for clients, even for servers.
nextStreamID uint32
// maxPeerStreamID is the ID of the most recent stream opened by the peer.
maxPeerStreamID uint32
// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
activeStreams prometheus.Gauge
// ignoreNewStreams is true when the connection is being shut down. New streams
// cannot be registered.
ignoreNewStreams bool
// streamsEmpty is a chan that will be closed when no more streams are open.
streamsEmptyChan chan struct{}
closeOnce sync.Once
}
func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Gauge) *activeStreamMap {
m := &activeStreamMap{
streams: make(map[uint32]*MuxedStream),
streamsEmptyChan: make(chan struct{}),
nextStreamID: 1,
activeStreams: activeStreams,
}
// Client initiated stream uses odd stream ID, server initiated stream uses even stream ID
if !useClientStreamNumbers {
m.nextStreamID = 2
}
return m
}
// This function should be called while `m` is locked.
func (m *activeStreamMap) notifyStreamsEmpty() {
m.closeOnce.Do(func() {
close(m.streamsEmptyChan)
})
}
// Len returns the number of active streams.
func (m *activeStreamMap) Len() int {
m.RLock()
defer m.RUnlock()
return len(m.streams)
}
func (m *activeStreamMap) Get(streamID uint32) (*MuxedStream, bool) {
m.RLock()
defer m.RUnlock()
stream, ok := m.streams[streamID]
return stream, ok
}
// Set returns true if the stream was assigned successfully. If a stream
// already existed with that ID or we are shutting down, return false.
func (m *activeStreamMap) Set(newStream *MuxedStream) bool {
m.Lock()
defer m.Unlock()
if _, ok := m.streams[newStream.streamID]; ok {
return false
}
if m.ignoreNewStreams {
return false
}
m.streams[newStream.streamID] = newStream
m.activeStreams.Inc()
return true
}
// Delete stops tracking the stream. It should be called only after it is closed and reset.
func (m *activeStreamMap) Delete(streamID uint32) {
m.Lock()
defer m.Unlock()
if _, ok := m.streams[streamID]; ok {
delete(m.streams, streamID)
m.activeStreams.Dec()
}
// shutting down, and now the map is empty
if m.ignoreNewStreams && len(m.streams) == 0 {
m.notifyStreamsEmpty()
}
}
// Shutdown blocks new streams from being created.
// It returns `done`, a channel that is closed once the last stream has closed
// and `progress`, whether a shutdown was already in progress
func (m *activeStreamMap) Shutdown() (done <-chan struct{}, alreadyInProgress bool) {
m.Lock()
defer m.Unlock()
if m.ignoreNewStreams {
// already shutting down
return m.streamsEmptyChan, true
}
m.ignoreNewStreams = true
if len(m.streams) == 0 {
// there are no streams to wait for
m.notifyStreamsEmpty()
}
return m.streamsEmptyChan, false
}
// AcquireLocalID acquires a new stream ID for a stream you're opening.
func (m *activeStreamMap) AcquireLocalID() uint32 {
m.Lock()
defer m.Unlock()
x := m.nextStreamID
m.nextStreamID += 2
return x
}
// ObservePeerID observes the ID of a stream opened by the peer. It returns true if we should accept
// the new stream, or false to reject it. The ErrCode gives the reason why.
func (m *activeStreamMap) AcquirePeerID(streamID uint32) (bool, http2.ErrCode) {
m.Lock()
defer m.Unlock()
switch {
case m.ignoreNewStreams:
return false, http2.ErrCodeStreamClosed
case streamID > m.maxPeerStreamID:
m.maxPeerStreamID = streamID
return true, http2.ErrCodeNo
default:
return false, http2.ErrCodeStreamClosed
}
}
// IsPeerStreamID is true if the stream ID belongs to the peer.
func (m *activeStreamMap) IsPeerStreamID(streamID uint32) bool {
m.RLock()
defer m.RUnlock()
return (streamID % 2) != (m.nextStreamID % 2)
}
// IsLocalStreamID is true if it is a stream we have opened, even if it is now closed.
func (m *activeStreamMap) IsLocalStreamID(streamID uint32) bool {
m.RLock()
defer m.RUnlock()
return (streamID%2) == (m.nextStreamID%2) && streamID < m.nextStreamID
}
// LastPeerStreamID returns the most recently opened peer stream ID.
func (m *activeStreamMap) LastPeerStreamID() uint32 {
m.RLock()
defer m.RUnlock()
return m.maxPeerStreamID
}
// LastLocalStreamID returns the most recently opened local stream ID.
func (m *activeStreamMap) LastLocalStreamID() uint32 {
m.RLock()
defer m.RUnlock()
if m.nextStreamID > 1 {
return m.nextStreamID - 2
}
return 0
}
// Abort closes every active stream and prevents new ones being created. This should be used to
// return errors in pending read/writes when the underlying connection goes away.
func (m *activeStreamMap) Abort() {
m.Lock()
defer m.Unlock()
for _, stream := range m.streams {
stream.Close()
}
m.ignoreNewStreams = true
m.notifyStreamsEmpty()
}

195
h2mux/activestreammap_test.go
View File

@ -1,195 +0,0 @@
package h2mux
import (
"sync"
"testing"
"github.com/stretchr/testify/assert"
)
func TestShutdown(t *testing.T) {
const numStreams = 1000
m := newActiveStreamMap(true, ActiveStreams)
// Add all the streams
{
var wg sync.WaitGroup
wg.Add(numStreams)
for i := 0; i < numStreams; i++ {
go func(streamID int) {
defer wg.Done()
stream := &MuxedStream{streamID: uint32(streamID)}
ok := m.Set(stream)
assert.True(t, ok)
}(i)
}
wg.Wait()
}
assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
shutdownChan, alreadyInProgress := m.Shutdown()
select {
case <-shutdownChan:
assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
default:
}
assert.False(t, alreadyInProgress)
shutdownChan2, alreadyInProgress2 := m.Shutdown()
assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
// Delete all the streams
{
var wg sync.WaitGroup
wg.Add(numStreams)
for i := 0; i < numStreams; i++ {
go func(streamID int) {
defer wg.Done()
m.Delete(uint32(streamID))
}(i)
}
wg.Wait()
}
assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
select {
case <-shutdownChan:
default:
assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
}
}
func TestEmptyBeforeShutdown(t *testing.T) {
const numStreams = 1000
m := newActiveStreamMap(true, ActiveStreams)
// Add all the streams
{
var wg sync.WaitGroup
wg.Add(numStreams)
for i := 0; i < numStreams; i++ {
go func(streamID int) {
defer wg.Done()
stream := &MuxedStream{streamID: uint32(streamID)}
ok := m.Set(stream)
assert.True(t, ok)
}(i)
}
wg.Wait()
}
assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
// Delete all the streams, bringing m to size 0
{
var wg sync.WaitGroup
wg.Add(numStreams)
for i := 0; i < numStreams; i++ {
go func(streamID int) {
defer wg.Done()
m.Delete(uint32(streamID))
}(i)
}
wg.Wait()
}
assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
// Add one stream back
const soloStreamID = uint32(0)
ok := m.Set(&MuxedStream{streamID: soloStreamID})
assert.True(t, ok)
shutdownChan, alreadyInProgress := m.Shutdown()
select {
case <-shutdownChan:
assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
default:
}
assert.False(t, alreadyInProgress)
shutdownChan2, alreadyInProgress2 := m.Shutdown()
assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
// Remove the remaining stream
m.Delete(soloStreamID)
select {
case <-shutdownChan:
default:
assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
}
}
type noopBuffer struct {
isClosed bool
}
func (t *noopBuffer) Read(p []byte) (n int, err error) { return len(p), nil }
func (t *noopBuffer) Write(p []byte) (n int, err error) { return len(p), nil }
func (t *noopBuffer) Reset() {}
func (t *noopBuffer) Len() int { return 0 }
func (t *noopBuffer) Close() error { t.isClosed = true; return nil }
func (t *noopBuffer) Closed() bool { return t.isClosed }
type noopReadyList struct{}
func (_ *noopReadyList) Signal(streamID uint32) {}
func TestAbort(t *testing.T) {
const numStreams = 1000
m := newActiveStreamMap(true, ActiveStreams)
var openedStreams sync.Map
// Add all the streams
{
var wg sync.WaitGroup
wg.Add(numStreams)
for i := 0; i < numStreams; i++ {
go func(streamID int) {
defer wg.Done()
stream := &MuxedStream{
streamID: uint32(streamID),
readBuffer: &noopBuffer{},
writeBuffer: &noopBuffer{},
readyList: &noopReadyList{},
}
ok := m.Set(stream)
assert.True(t, ok)
openedStreams.Store(stream.streamID, stream)
}(i)
}
wg.Wait()
}
assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
shutdownChan, alreadyInProgress := m.Shutdown()
select {
case <-shutdownChan:
assert.Fail(t, "before Abort(), shutdownChan shouldn't be closed")
default:
}
assert.False(t, alreadyInProgress)
m.Abort()
assert.Equal(t, numStreams, m.Len(), "Abort() shouldn't delete any streams")
openedStreams.Range(func(key interface{}, value interface{}) bool {
stream := value.(*MuxedStream)
readBuffer := stream.readBuffer.(*noopBuffer)
writeBuffer := stream.writeBuffer.(*noopBuffer)
return assert.True(t, readBuffer.isClosed && writeBuffer.isClosed, "Abort() should have closed all the streams")
})
select {
case <-shutdownChan:
default:
assert.Fail(t, "after Abort(), shutdownChan should have been closed")
}
// multiple aborts shouldn't cause any issues
m.Abort()
m.Abort()
m.Abort()
}

27
h2mux/bytes_counter.go
View File

@ -1,27 +0,0 @@
package h2mux
import (
"sync/atomic"
)
type AtomicCounter struct {
count uint64
}
func NewAtomicCounter(initCount uint64) *AtomicCounter {
return &AtomicCounter{count: initCount}
}
func (c *AtomicCounter) IncrementBy(number uint64) {
atomic.AddUint64(&c.count, number)
}
// Count returns the current value of counter and reset it to 0
func (c *AtomicCounter) Count() uint64 {
return atomic.SwapUint64(&c.count, 0)
}
// Value returns the current value of counter
func (c *AtomicCounter) Value() uint64 {
return atomic.LoadUint64(&c.count)
}

23
h2mux/bytes_counter_test.go
View File

@ -1,23 +0,0 @@
package h2mux
import (
"sync"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCounter(t *testing.T) {
var wg sync.WaitGroup
wg.Add(dataPoints)
c := AtomicCounter{}
for i := 0; i < dataPoints; i++ {
go func() {
defer wg.Done()
c.IncrementBy(uint64(1))
}()
}
wg.Wait()
assert.Equal(t, uint64(dataPoints), c.Count())
assert.Equal(t, uint64(0), c.Count())
}

66
h2mux/error.go
View File

@ -1,66 +0,0 @@
package h2mux
import (
"fmt"
"golang.org/x/net/http2"
)
var (
// HTTP2 error codes: https://http2.github.io/http2-spec/#ErrorCodes
ErrHandshakeTimeout = MuxerHandshakeError{"1000 handshake timeout"}
ErrBadHandshakeNotSettings = MuxerHandshakeError{"1001 unexpected response"}
ErrBadHandshakeUnexpectedAck = MuxerHandshakeError{"1002 unexpected response"}
ErrBadHandshakeNoMagic = MuxerHandshakeError{"1003 unexpected response"}
ErrBadHandshakeWrongMagic = MuxerHandshakeError{"1004 connected to endpoint of wrong type"}
ErrBadHandshakeNotSettingsAck = MuxerHandshakeError{"1005 unexpected response"}
ErrBadHandshakeUnexpectedSettings = MuxerHandshakeError{"1006 unexpected response"}
ErrUnexpectedFrameType = MuxerProtocolError{"2001 unexpected frame type", http2.ErrCodeProtocol}
ErrUnknownStream = MuxerProtocolError{"2002 unknown stream", http2.ErrCodeProtocol}
ErrInvalidStream = MuxerProtocolError{"2003 invalid stream", http2.ErrCodeProtocol}
ErrNotRPCStream = MuxerProtocolError{"2004 not RPC stream", http2.ErrCodeProtocol}
ErrStreamHeadersSent = MuxerApplicationError{"3000 headers already sent"}
ErrStreamRequestConnectionClosed = MuxerApplicationError{"3001 connection closed while opening stream"}
ErrConnectionDropped = MuxerApplicationError{"3002 connection dropped"}
ErrStreamRequestTimeout = MuxerApplicationError{"3003 open stream timeout"}
ErrResponseHeadersTimeout = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
ErrResponseHeadersConnectionClosed = MuxerApplicationError{"3005 connection closed while waiting for initial response headers"}
ErrClosedStream = MuxerStreamError{"4000 stream closed", http2.ErrCodeStreamClosed}
)
type MuxerHandshakeError struct {
cause string
}
func (e MuxerHandshakeError) Error() string {
return fmt.Sprintf("Handshake error: %s", e.cause)
}
type MuxerProtocolError struct {
cause string
h2code http2.ErrCode
}
func (e MuxerProtocolError) Error() string {
return fmt.Sprintf("Protocol error: %s", e.cause)
}
type MuxerApplicationError struct {
cause string
}
func (e MuxerApplicationError) Error() string {
return fmt.Sprintf("Application error: %s", e.cause)
}
type MuxerStreamError struct {
cause string
h2code http2.ErrCode
}
func (e MuxerStreamError) Error() string {
return fmt.Sprintf("Stream error: %s", e.cause)
}

17
h2mux/h2_compressor.go
View File

@ -1,17 +0,0 @@
package h2mux
import (
"io"
)
func CompressionIsSupported() bool {
return false
}
func newDecompressor(src io.Reader) decompressor {
return nil
}
func newCompressor(dst io.Writer, quality, lgwin int) compressor {
return nil
}

596
h2mux/h2_dictionaries.go
View File

@ -1,596 +0,0 @@
package h2mux
import (
"bytes"
"io"
"strings"
"sync"
"golang.org/x/net/http2"
)
/* This is an implementation of https://github.com/vkrasnov/h2-compression-dictionaries
but modified for tunnels in a few key ways:
Since tunnels is a server-to-server service, some aspects of the spec would cause
unnecessary head-of-line blocking on the CPU and on the network, hence this implementation
allows for parallel compression on the "client", and buffering on the "server" to solve
this problem. */
// Assign temporary values
const SettingCompression http2.SettingID = 0xff20
const (
FrameSetCompressionContext http2.FrameType = 0xf0
FrameUseDictionary http2.FrameType = 0xf1
FrameSetDictionary http2.FrameType = 0xf2
)
const (
FlagSetDictionaryAppend http2.Flags = 0x1
FlagSetDictionaryOffset http2.Flags = 0x2
)
const compressionVersion = uint8(1)
const compressionFormat = uint8(2)
type CompressionSetting uint
const (
CompressionNone CompressionSetting = iota
CompressionLow
CompressionMedium
CompressionMax
)
type CompressionPreset struct {
nDicts, dictSize, quality uint8
}
type compressor interface {
Write([]byte) (int, error)
Flush() error
SetDictionary([]byte)
Close() error
}
type decompressor interface {
Read([]byte) (int, error)
SetDictionary([]byte)
Close() error
}
var compressionPresets = map[CompressionSetting]CompressionPreset{
CompressionNone: {0, 0, 0},
CompressionLow: {32, 17, 5},
CompressionMedium: {64, 18, 6},
CompressionMax: {255, 19, 9},
}
func compressionSettingVal(version, fmt, sz, nd uint8) uint32 {
// Currently the compression settings are include:
// * version: only 1 is supported
// * fmt: only 2 for brotli is supported
// * sz: log2 of the maximal allowed dictionary size
// * nd: max allowed number of dictionaries
return uint32(version)<<24 + uint32(fmt)<<16 + uint32(sz)<<8 + uint32(nd)
}
func parseCompressionSettingVal(setting uint32) (version, fmt, sz, nd uint8) {
version = uint8(setting >> 24)
fmt = uint8(setting >> 16)
sz = uint8(setting >> 8)
nd = uint8(setting)
return
}
func (c CompressionSetting) toH2Setting() uint32 {
p, ok := compressionPresets[c]
if !ok {
return 0
}
return compressionSettingVal(compressionVersion, compressionFormat, p.dictSize, p.nDicts)
}
func (c CompressionSetting) getPreset() CompressionPreset {
return compressionPresets[c]
}
type dictUpdate struct {
reader *h2DictionaryReader
dictionary *h2ReadDictionary
buff []byte
isReady bool
isUse bool
s setDictRequest
}
type h2ReadDictionary struct {
dictionary []byte
queue []*dictUpdate
maxSize int
}
type h2ReadDictionaries struct {
d []h2ReadDictionary
maxSize int
}
type h2DictionaryReader struct {
*SharedBuffer // Propagate the decompressed output into the original buffer
decompBuffer *bytes.Buffer // Intermediate buffer for the brotli compressor
dictionary []byte // The content of the dictionary being used by this reader
internalBuffer []byte
s, e int // Start and end of the buffer
decomp decompressor // The brotli compressor
isClosed bool // Indicates that Close was called for this reader
queue []*dictUpdate // List of dictionaries to update, when the data is available
}
type h2WriteDictionary []byte
type setDictRequest struct {
streamID uint32
dictID uint8
dictSZ uint64
truncate, offset uint64
P, E, D bool
}
type useDictRequest struct {
dictID uint8
streamID uint32
setDict []setDictRequest
}
type h2WriteDictionaries struct {
dictLock sync.Mutex
dictChan chan useDictRequest
dictionaries []h2WriteDictionary
nextAvail int // next unused dictionary slot
maxAvail int // max ID, defined by SETTINGS
maxSize int // max size, defined by SETTINGS
typeToDict map[string]uint8 // map from content type to dictionary that encodes it
pathToDict map[string]uint8 // map from path to dictionary that encodes it
quality int
window int
compIn, compOut *AtomicCounter
}
type h2DictWriter struct {
*bytes.Buffer
comp compressor
dicts *h2WriteDictionaries
writerLock sync.Mutex
streamID uint32
path string
contentType string
}
type h2Dictionaries struct {
write *h2WriteDictionaries
read *h2ReadDictionaries
}
func (o *dictUpdate) update(buff []byte) {
o.buff = make([]byte, len(buff))
copy(o.buff, buff)
o.isReady = true
}
func (d *h2ReadDictionary) update() {
for len(d.queue) > 0 {
o := d.queue[0]
if !o.isReady {
break
}
if o.isUse {
reader := o.reader
reader.dictionary = make([]byte, len(d.dictionary))
copy(reader.dictionary, d.dictionary)
reader.decomp = newDecompressor(reader.decompBuffer)
if len(reader.dictionary) > 0 {
reader.decomp.SetDictionary(reader.dictionary)
}
reader.Write([]byte{})
} else {
d.dictionary = adjustDictionary(d.dictionary, o.buff, o.s, d.maxSize)
}
d.queue = d.queue[1:]
}
}
func newH2ReadDictionaries(nd, sz uint8) h2ReadDictionaries {
d := make([]h2ReadDictionary, int(nd))
for i := range d {
d[i].maxSize = 1 << uint(sz)
}
return h2ReadDictionaries{d: d, maxSize: 1 << uint(sz)}
}
func (dicts *h2ReadDictionaries) getDictByID(dictID uint8) (*h2ReadDictionary, error) {
if int(dictID) > len(dicts.d) {
return nil, MuxerStreamError{"dictID too big", http2.ErrCodeProtocol}
}
return &dicts.d[dictID], nil
}
func (dicts *h2ReadDictionaries) newReader(b *SharedBuffer, dictID uint8) *h2DictionaryReader {
if int(dictID) > len(dicts.d) {
return nil
}
dictionary := &dicts.d[dictID]
reader := &h2DictionaryReader{SharedBuffer: b, decompBuffer: &bytes.Buffer{}, internalBuffer: make([]byte, dicts.maxSize)}
if len(dictionary.queue) == 0 {
reader.dictionary = make([]byte, len(dictionary.dictionary))
copy(reader.dictionary, dictionary.dictionary)
reader.decomp = newDecompressor(reader.decompBuffer)
if len(reader.dictionary) > 0 {
reader.decomp.SetDictionary(reader.dictionary)
}
} else {
dictionary.queue = append(dictionary.queue, &dictUpdate{isUse: true, isReady: true, reader: reader})
}
return reader
}
func (r *h2DictionaryReader) updateWaitingDictionaries() {
// Update all the waiting dictionaries
for _, o := range r.queue {
if o.isReady {
continue
}
if r.isClosed || uint64(r.e) >= o.s.dictSZ {
o.update(r.internalBuffer[:r.e])
if o == o.dictionary.queue[0] {
defer o.dictionary.update()
}
}
}
}
// Write actually happens when reading from network, this is therefore the stage where we decompress the buffer
func (r *h2DictionaryReader) Write(p []byte) (n int, err error) {
// Every write goes into brotli buffer first
n, err = r.decompBuffer.Write(p)
if err != nil {
return
}
if r.decomp == nil {
return
}
for {
m, err := r.decomp.Read(r.internalBuffer[r.e:])
if err != nil && err != io.EOF {
r.SharedBuffer.Close()
r.decomp.Close()
return n, err
}
r.SharedBuffer.Write(r.internalBuffer[r.e : r.e+m])
r.e += m
if m == 0 {
break
}
if r.e == len(r.internalBuffer) {
r.updateWaitingDictionaries()
r.e = 0
}
}
r.updateWaitingDictionaries()
if r.isClosed {
r.SharedBuffer.Close()
r.decomp.Close()
}
return
}
func (r *h2DictionaryReader) Close() error {
if r.isClosed {
return nil
}
r.isClosed = true
r.Write([]byte{})
return nil
}
var compressibleTypes = map[string]bool{
"application/atom+xml": true,
"application/javascript": true,
"application/json": true,
"application/ld+json": true,
"application/manifest+json": true,
"application/rss+xml": true,
"application/vnd.geo+json": true,
"application/vnd.ms-fontobject": true,
"application/x-font-ttf": true,
"application/x-yaml": true,
"application/x-web-app-manifest+json": true,
"application/xhtml+xml": true,
"application/xml": true,
"font/opentype": true,
"image/bmp": true,
"image/svg+xml": true,
"image/x-icon": true,
"text/cache-manifest": true,
"text/css": true,
"text/html": true,
"text/plain": true,
"text/vcard": true,
"text/vnd.rim.location.xloc": true,
"text/vtt": true,
"text/x-component": true,
"text/x-cross-domain-policy": true,
"text/x-yaml": true,
}
func getContentType(headers []Header) string {
for _, h := range headers {
if strings.ToLower(h.Name) == "content-type" {
val := strings.ToLower(h.Value)
sep := strings.IndexRune(val, ';')
if sep != -1 {
return val[:sep]
}
return val
}
}
return ""
}
func newH2WriteDictionaries(nd, sz, quality uint8, compIn, compOut *AtomicCounter) (*h2WriteDictionaries, chan useDictRequest) {
useDictChan := make(chan useDictRequest)
return &h2WriteDictionaries{
dictionaries: make([]h2WriteDictionary, nd),
nextAvail: 0,
maxAvail: int(nd),
maxSize: 1 << uint(sz),
dictChan: useDictChan,
typeToDict: make(map[string]uint8),
pathToDict: make(map[string]uint8),
quality: int(quality),
window: 1 << uint(sz+1),
compIn: compIn,
compOut: compOut,
}, useDictChan
}
func adjustDictionary(currentDictionary, newData []byte, set setDictRequest, maxSize int) []byte {
currentDictionary = append(currentDictionary, newData[:set.dictSZ]...)
if len(currentDictionary) > maxSize {
currentDictionary = currentDictionary[len(currentDictionary)-maxSize:]
}
return currentDictionary
}
func (h2d *h2WriteDictionaries) getNextDictID() (dictID uint8, ok bool) {
if h2d.nextAvail < h2d.maxAvail {
dictID, ok = uint8(h2d.nextAvail), true
h2d.nextAvail++
return
}
return 0, false
}
func (h2d *h2WriteDictionaries) getGenericDictID() (dictID uint8, ok bool) {
if h2d.maxAvail == 0 {
return 0, false
}
return uint8(h2d.maxAvail - 1), true
}
func (h2d *h2WriteDictionaries) getDictWriter(s *MuxedStream, headers []Header) *h2DictWriter {
w := s.writeBuffer
if w == nil {
return nil
}
if s.method != "GET" && s.method != "POST" {
return nil
}
s.contentType = getContentType(headers)
if _, ok := compressibleTypes[s.contentType]; !ok && !strings.HasPrefix(s.contentType, "text") {
return nil
}
return &h2DictWriter{
Buffer: w.(*bytes.Buffer),
path: s.path,
contentType: s.contentType,
streamID: s.streamID,
dicts: h2d,
}
}
func assignDictToStream(s *MuxedStream, p []byte) bool {
// On first write to stream:
// * assign the right dictionary
// * update relevant dictionaries
// * send the required USE_DICT and SET_DICT frames
h2d := s.dictionaries.write
if h2d == nil {
return false
}
w, ok := s.writeBuffer.(*h2DictWriter)
if !ok || w.comp != nil {
return false
}
h2d.dictLock.Lock()
if w.comp != nil {
// Check again with lock, in therory the interface allows for unordered writes
h2d.dictLock.Unlock()
return false
}
// The logic of dictionary generation is below
// Is there a dictionary for the exact path or content-type?
var useID uint8
pathID, pathFound := h2d.pathToDict[w.path]
typeID, typeFound := h2d.typeToDict[w.contentType]
if pathFound {
// Use dictionary for path as top priority
useID = pathID
if !typeFound { // Shouldn't really happen, unless type changes between requests
typeID, typeFound = h2d.getNextDictID()
if typeFound {
h2d.typeToDict[w.contentType] = typeID
}
}
} else if typeFound {
// Use dictionary for same content type as second priority
useID = typeID
pathID, pathFound = h2d.getNextDictID()
if pathFound { // If a slot is available, generate new dictionary for path
h2d.pathToDict[w.path] = pathID
}
} else {
// Use the overflow dictionary as last resort
// If slots are available generate new dictionaries for path and content-type
useID, _ = h2d.getGenericDictID()
pathID, pathFound = h2d.getNextDictID()
if pathFound {
h2d.pathToDict[w.path] = pathID
}
typeID, typeFound = h2d.getNextDictID()
if typeFound {
h2d.typeToDict[w.contentType] = typeID
}
}
useLen := h2d.maxSize
if len(p) < useLen {
useLen = len(p)
}
// Update all the dictionaries using the new data
setDicts := make([]setDictRequest, 0, 3)
setDict := setDictRequest{
streamID: w.streamID,
dictID: useID,
dictSZ: uint64(useLen),
}
setDicts = append(setDicts, setDict)
if pathID != useID {
setDict.dictID = pathID
setDicts = append(setDicts, setDict)
}
if typeID != useID {
setDict.dictID = typeID
setDicts = append(setDicts, setDict)
}
h2d.dictChan <- useDictRequest{streamID: w.streamID, dictID: uint8(useID), setDict: setDicts}
dict := h2d.dictionaries[useID]
// Brolti requires the dictionary to be immutable
copyDict := make([]byte, len(dict))
copy(copyDict, dict)
for _, set := range setDicts {
h2d.dictionaries[set.dictID] = adjustDictionary(h2d.dictionaries[set.dictID], p, set, h2d.maxSize)
}
w.comp = newCompressor(w.Buffer, h2d.quality, h2d.window)
s.writeLock.Lock()
h2d.dictLock.Unlock()
if len(copyDict) > 0 {
w.comp.SetDictionary(copyDict)
}
return true
}
func (w *h2DictWriter) Write(p []byte) (n int, err error) {
bufLen := w.Buffer.Len()
if w.comp != nil {
n, err = w.comp.Write(p)
if err != nil {
return
}
err = w.comp.Flush()
w.dicts.compIn.IncrementBy(uint64(n))
w.dicts.compOut.IncrementBy(uint64(w.Buffer.Len() - bufLen))
return
}
return w.Buffer.Write(p)
}
func (w *h2DictWriter) Close() error {
if w.comp != nil {
return w.comp.Close()
}
return nil
}
// From http2/hpack
func http2ReadVarInt(n byte, p []byte) (remain []byte, v uint64, err error) {
if n < 1 || n > 8 {
panic("bad n")
}
if len(p) == 0 {
return nil, 0, MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
v = uint64(p[0])
if n < 8 {
v &= (1 << uint64(n)) - 1
}
if v < (1<<uint64(n))-1 {
return p[1:], v, nil
}
origP := p
p = p[1:]
var m uint64
for len(p) > 0 {
b := p[0]
p = p[1:]
v += uint64(b&127) << m
if b&128 == 0 {
return p, v, nil
}
m += 7
if m >= 63 {
return origP, 0, MuxerStreamError{"invalid integer", http2.ErrCodeProtocol}
}
}
return nil, 0, MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
func appendVarInt(dst []byte, n byte, i uint64) []byte {
k := uint64((1 << n) - 1)
if i < k {
return append(dst, byte(i))
}
dst = append(dst, byte(k))
i -= k
for ; i >= 128; i >>= 7 {
dst = append(dst, byte(0x80|(i&0x7f)))
}
return append(dst, byte(i))
}

506
h2mux/h2mux.go
View File

@ -1,506 +0,0 @@
package h2mux
import (
"context"
"io"
"strings"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog"
"golang.org/x/net/http2"
"golang.org/x/net/http2/hpack"
"golang.org/x/sync/errgroup"
)
const (
defaultFrameSize uint32 = 1 << 14 // Minimum frame size in http2 spec
defaultWindowSize uint32 = (1 << 16) - 1 // Minimum window size in http2 spec
maxWindowSize uint32 = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size in http2 spec
defaultTimeout time.Duration = 5 * time.Second
defaultRetries uint64 = 5
defaultWriteBufferMaxLen int = 1024 * 1024 // 1mb
writeBufferInitialSize int = 16 * 1024 // 16KB
SettingMuxerMagic http2.SettingID = 0x42db
MuxerMagicOrigin uint32 = 0xa2e43c8b
MuxerMagicEdge uint32 = 0x1088ebf9
)
type MuxedStreamHandler interface {
ServeStream(*MuxedStream) error
}
type MuxedStreamFunc func(stream *MuxedStream) error
func (f MuxedStreamFunc) ServeStream(stream *MuxedStream) error {
return f(stream)
}
type MuxerConfig struct {
Timeout time.Duration
Handler MuxedStreamHandler
IsClient bool
// Name is used to identify this muxer instance when logging.
Name string
// The minimum time this connection can be idle before sending a heartbeat.
HeartbeatInterval time.Duration
// The minimum number of heartbeats to send before terminating the connection.
MaxHeartbeats uint64
// Logger to use
Log *zerolog.Logger
CompressionQuality CompressionSetting
// Initial size for HTTP2 flow control windows
DefaultWindowSize uint32
// Largest allowable size for HTTP2 flow control windows
MaxWindowSize uint32
// Largest allowable capacity for the buffer of data to be sent
StreamWriteBufferMaxLen int
}
type Muxer struct {
// f is used to read and write HTTP2 frames on the wire.
f *http2.Framer
// config is the MuxerConfig given in Handshake.
config MuxerConfig
// w, r are references to the underlying connection used.
w io.WriteCloser
r io.ReadCloser
// muxReader is the read process.
muxReader *MuxReader
// muxWriter is the write process.
muxWriter *MuxWriter
// muxMetricsUpdater is the process to update metrics
muxMetricsUpdater muxMetricsUpdater
// newStreamChan is used to create new streams on the writer thread.
// The writer will assign the next available stream ID.
newStreamChan chan MuxedStreamRequest
// abortChan is used to abort the writer event loop.
abortChan chan struct{}
// abortOnce is used to ensure abortChan is closed once only.
abortOnce sync.Once
// readyList is used to signal writable streams.
readyList *ReadyList
// streams tracks currently-open streams.
streams *activeStreamMap
// explicitShutdown records whether the Muxer is closing because Shutdown was called, or due to another
// error.
explicitShutdown *BooleanFuse
compressionQuality CompressionPreset
}
func RPCHeaders() []Header {
return []Header{
{Name: ":method", Value: "RPC"},
{Name: ":scheme", Value: "capnp"},
{Name: ":path", Value: "*"},
}
}
// Handshake establishes a muxed connection with the peer.
// After the handshake completes, it is possible to open and accept streams.
func Handshake(
w io.WriteCloser,
r io.ReadCloser,
config MuxerConfig,
activeStreamsMetrics prometheus.Gauge,
) (*Muxer, error) {
// Set default config values
if config.Timeout == 0 {
config.Timeout = defaultTimeout
}
if config.DefaultWindowSize == 0 {
config.DefaultWindowSize = defaultWindowSize
}
if config.MaxWindowSize == 0 {
config.MaxWindowSize = maxWindowSize
}
if config.StreamWriteBufferMaxLen == 0 {
config.StreamWriteBufferMaxLen = defaultWriteBufferMaxLen
}
// Initialise connection state fields
m := &Muxer{
f: http2.NewFramer(w, r), // A framer that writes to w and reads from r
config: config,
w: w,
r: r,
newStreamChan: make(chan MuxedStreamRequest),
abortChan: make(chan struct{}),
readyList: NewReadyList(),
streams: newActiveStreamMap(config.IsClient, activeStreamsMetrics),
}
m.f.ReadMetaHeaders = hpack.NewDecoder(4096, func(hpack.HeaderField) {})
// Initialise the settings to identify this connection and confirm the other end is sane.
handshakeSetting := http2.Setting{ID: SettingMuxerMagic, Val: MuxerMagicEdge}
compressionSetting := http2.Setting{ID: SettingCompression, Val: 0}
expectedMagic := MuxerMagicOrigin
if config.IsClient {
handshakeSetting.Val = MuxerMagicOrigin
expectedMagic = MuxerMagicEdge
}
errChan := make(chan error, 2)
// Simultaneously send our settings and verify the peer's settings.
go func() { errChan <- m.f.WriteSettings(handshakeSetting, compressionSetting) }()
go func() { errChan <- m.readPeerSettings(expectedMagic) }()
err := joinErrorsWithTimeout(errChan, 2, config.Timeout, ErrHandshakeTimeout)
if err != nil {
return nil, err
}
// Confirm sanity by ACKing the frame and expecting an ACK for our frame.
// Not strictly necessary, but let's pretend to be H2-like.
go func() { errChan <- m.f.WriteSettingsAck() }()
go func() { errChan <- m.readPeerSettingsAck() }()
err = joinErrorsWithTimeout(errChan, 2, config.Timeout, ErrHandshakeTimeout)
if err != nil {
return nil, err
}
// set up reader/writer pair ready for serve
streamErrors := NewStreamErrorMap()
goAwayChan := make(chan http2.ErrCode, 1)
inBoundCounter := NewAtomicCounter(0)
outBoundCounter := NewAtomicCounter(0)
pingTimestamp := NewPingTimestamp()
connActive := NewSignal()
idleDuration := config.HeartbeatInterval
// Sanity check to ensure idelDuration is sane
if idleDuration == 0 || idleDuration < defaultTimeout {
idleDuration = defaultTimeout
config.Log.Info().Msgf("muxer: Minimum idle time has been adjusted to %d", defaultTimeout)
}
maxRetries := config.MaxHeartbeats
if maxRetries == 0 {
maxRetries = defaultRetries
config.Log.Info().Msgf("muxer: Minimum number of unacked heartbeats to send before closing the connection has been adjusted to %d", maxRetries)
}
compBytesBefore, compBytesAfter := NewAtomicCounter(0), NewAtomicCounter(0)
m.muxMetricsUpdater = newMuxMetricsUpdater(
m.abortChan,
compBytesBefore,
compBytesAfter,
)
m.explicitShutdown = NewBooleanFuse()
m.muxReader = &MuxReader{
f: m.f,
handler: m.config.Handler,
streams: m.streams,
readyList: m.readyList,
streamErrors: streamErrors,
goAwayChan: goAwayChan,
abortChan: m.abortChan,
pingTimestamp: pingTimestamp,
connActive: connActive,
initialStreamWindow: m.config.DefaultWindowSize,
streamWindowMax: m.config.MaxWindowSize,
streamWriteBufferMaxLen: m.config.StreamWriteBufferMaxLen,
r: m.r,
metricsUpdater: m.muxMetricsUpdater,
bytesRead: inBoundCounter,
}
m.muxWriter = &MuxWriter{
f: m.f,
streams: m.streams,
streamErrors: streamErrors,
readyStreamChan: m.readyList.ReadyChannel(),
newStreamChan: m.newStreamChan,
goAwayChan: goAwayChan,
abortChan: m.abortChan,
pingTimestamp: pingTimestamp,
idleTimer: NewIdleTimer(idleDuration, maxRetries),
connActiveChan: connActive.WaitChannel(),
maxFrameSize: defaultFrameSize,
metricsUpdater: m.muxMetricsUpdater,
bytesWrote: outBoundCounter,
}
m.muxWriter.headerEncoder = hpack.NewEncoder(&m.muxWriter.headerBuffer)
if m.compressionQuality.dictSize > 0 && m.compressionQuality.nDicts > 0 {
nd, sz := m.compressionQuality.nDicts, m.compressionQuality.dictSize
writeDicts, dictChan := newH2WriteDictionaries(
nd,
sz,
m.compressionQuality.quality,
compBytesBefore,
compBytesAfter,
)
readDicts := newH2ReadDictionaries(nd, sz)
m.muxReader.dictionaries = h2Dictionaries{read: &readDicts, write: writeDicts}
m.muxWriter.useDictChan = dictChan
}
return m, nil
}
func (m *Muxer) readPeerSettings(magic uint32) error {
frame, err := m.f.ReadFrame()
if err != nil {
return err
}
settingsFrame, ok := frame.(*http2.SettingsFrame)
if !ok {
return ErrBadHandshakeNotSettings
}
if settingsFrame.Header().Flags != 0 {
return ErrBadHandshakeUnexpectedAck
}
peerMagic, ok := settingsFrame.Value(SettingMuxerMagic)
if !ok {
return ErrBadHandshakeNoMagic
}
if magic != peerMagic {
return ErrBadHandshakeWrongMagic
}
peerCompression, ok := settingsFrame.Value(SettingCompression)
if !ok {
m.compressionQuality = compressionPresets[CompressionNone]
return nil
}
ver, fmt, sz, nd := parseCompressionSettingVal(peerCompression)
if ver != compressionVersion || fmt != compressionFormat || sz == 0 || nd == 0 {
m.compressionQuality = compressionPresets[CompressionNone]
return nil
}
// Values used for compression are the minimum between the two peers
if sz < m.compressionQuality.dictSize {
m.compressionQuality.dictSize = sz
}
if nd < m.compressionQuality.nDicts {
m.compressionQuality.nDicts = nd
}
return nil
}
func (m *Muxer) readPeerSettingsAck() error {
frame, err := m.f.ReadFrame()
if err != nil {
return err
}
settingsFrame, ok := frame.(*http2.SettingsFrame)
if !ok {
return ErrBadHandshakeNotSettingsAck
}
if settingsFrame.Header().Flags != http2.FlagSettingsAck {
return ErrBadHandshakeUnexpectedSettings
}
return nil
}
func joinErrorsWithTimeout(errChan <-chan error, receiveCount int, timeout time.Duration, timeoutError error) error {
for i := 0; i < receiveCount; i++ {
select {
case err := <-errChan:
if err != nil {
return err
}
case <-time.After(timeout):
return timeoutError
}
}
return nil
}
// Serve runs the event loops that comprise h2mux:
// - MuxReader.run()
// - MuxWriter.run()
// - muxMetricsUpdater.run()
// In the normal case, Shutdown() is called concurrently with Serve() to stop
// these loops.
func (m *Muxer) Serve(ctx context.Context) error {
errGroup, _ := errgroup.WithContext(ctx)
errGroup.Go(func() error {
ch := make(chan error)
go func() {
err := m.muxReader.run(m.config.Log)
m.explicitShutdown.Fuse(false)
m.r.Close()
m.abort()
// don't block if parent goroutine quit early
select {
case ch <- err:
default:
}
}()
select {
case err := <-ch:
return err
case <-ctx.Done():
return ctx.Err()
}
})
errGroup.Go(func() error {
ch := make(chan error)
go func() {
err := m.muxWriter.run(m.config.Log)
m.explicitShutdown.Fuse(false)
m.w.Close()
m.abort()
// don't block if parent goroutine quit early
select {
case ch <- err:
default:
}
}()
select {
case err := <-ch:
return err
case <-ctx.Done():
return ctx.Err()
}
})
errGroup.Go(func() error {
ch := make(chan error)
go func() {
err := m.muxMetricsUpdater.run(m.config.Log)
// don't block if parent goroutine quit early
select {
case ch <- err:
default:
}
}()
select {
case err := <-ch:
return err
case <-ctx.Done():
return ctx.Err()
}
})
err := errGroup.Wait()
if isUnexpectedTunnelError(err, m.explicitShutdown.Value()) {
return err
}
return nil
}
// Shutdown is called to initiate the "happy path" of muxer termination.
// It blocks new streams from being created.
// It returns a channel that is closed when the last stream has been closed.
func (m *Muxer) Shutdown() <-chan struct{} {
m.explicitShutdown.Fuse(true)
return m.muxReader.Shutdown()
}
// IsUnexpectedTunnelError identifies errors that are expected when shutting down the h2mux tunnel.
// The set of expected errors change depending on whether we initiated shutdown or not.
func isUnexpectedTunnelError(err error, expectedShutdown bool) bool {
if err == nil {
return false
}
if !expectedShutdown {
return true
}
return !isConnectionClosedError(err)
}
func isConnectionClosedError(err error) bool {
if err == io.EOF {
return true
}
if err == io.ErrClosedPipe {
return true
}
if err.Error() == "tls: use of closed connection" {
return true
}
if strings.HasSuffix(err.Error(), "use of closed network connection") {
return true
}
return false
}
// OpenStream opens a new data stream with the given headers.
// Called by proxy server and tunnel
func (m *Muxer) OpenStream(ctx context.Context, headers []Header, body io.Reader) (*MuxedStream, error) {
stream := m.NewStream(headers)
if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, body)); err != nil {
return nil, err
}
if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
return nil, err
}
return stream, nil
}
func (m *Muxer) OpenRPCStream(ctx context.Context) (*MuxedStream, error) {
stream := m.NewStream(RPCHeaders())
if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, nil)); err != nil {
stream.Close()
return nil, err
}
if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
stream.Close()
return nil, err
}
if !IsRPCStreamResponse(stream) {
stream.Close()
return nil, ErrNotRPCStream
}
return stream, nil
}
func (m *Muxer) NewStream(headers []Header) *MuxedStream {
return NewStream(m.config, headers, m.readyList, m.muxReader.dictionaries)
}
func (m *Muxer) MakeMuxedStreamRequest(ctx context.Context, request MuxedStreamRequest) error {
select {
case <-ctx.Done():
return ErrStreamRequestTimeout
case <-m.abortChan:
return ErrStreamRequestConnectionClosed
// Will be received by mux writer
case m.newStreamChan <- request:
return nil
}
}
func (m *Muxer) CloseStreamRead(stream *MuxedStream) {
stream.CloseRead()
if stream.WriteClosed() {
m.streams.Delete(stream.streamID)
}
}
func (m *Muxer) AwaitResponseHeaders(ctx context.Context, stream *MuxedStream) error {
select {
case <-ctx.Done():
return ErrResponseHeadersTimeout
case <-m.abortChan:
return ErrResponseHeadersConnectionClosed
case <-stream.responseHeadersReceived:
return nil
}
}
func (m *Muxer) Metrics() *MuxerMetrics {
return m.muxMetricsUpdater.metrics()
}
func (m *Muxer) abort() {
m.abortOnce.Do(func() {
close(m.abortChan)
m.readyList.Close()
m.streams.Abort()
})
}
// Return how many retries/ticks since the connection was last marked active
func (m *Muxer) TimerRetries() uint64 {
return m.muxWriter.idleTimer.RetryCount()
}
func IsRPCStreamResponse(stream *MuxedStream) bool {
headers := stream.Headers
return len(headers) == 1 &&
headers[0].Name == ":status" &&
headers[0].Value == "200"
}

909
h2mux/h2mux_test.go
View File

@ -1,909 +0,0 @@
package h2mux
import (
"bytes"
"context"
"fmt"
"io"
"math/rand"
"net"
"os"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
"golang.org/x/sync/errgroup"
)
const (
testOpenStreamTimeout = time.Millisecond * 5000
testHandshakeTimeout = time.Millisecond * 1000
)
var log = zerolog.Nop()
func TestMain(m *testing.M) {
if os.Getenv("VERBOSE") == "1" {
//TODO: set log level
}
os.Exit(m.Run())
}
type DefaultMuxerPair struct {
OriginMuxConfig MuxerConfig
OriginMux *Muxer
OriginConn net.Conn
EdgeMuxConfig MuxerConfig
EdgeMux *Muxer
EdgeConn net.Conn
doneC chan struct{}
}
func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc) *DefaultMuxerPair {
origin, edge := net.Pipe()
p := &DefaultMuxerPair{
OriginMuxConfig: MuxerConfig{
Timeout: testHandshakeTimeout,
Handler: f,
IsClient: true,
Name: "origin",
Log: &log,
DefaultWindowSize: (1 << 8) - 1,
MaxWindowSize: (1 << 15) - 1,
StreamWriteBufferMaxLen: 1024,
HeartbeatInterval: defaultTimeout,
MaxHeartbeats: defaultRetries,
},
OriginConn: origin,
EdgeMuxConfig: MuxerConfig{
Timeout: testHandshakeTimeout,
IsClient: false,
Name: "edge",
Log: &log,
DefaultWindowSize: (1 << 8) - 1,
MaxWindowSize: (1 << 15) - 1,
StreamWriteBufferMaxLen: 1024,
HeartbeatInterval: defaultTimeout,
MaxHeartbeats: defaultRetries,
},
EdgeConn: edge,
doneC: make(chan struct{}),
}
assert.NoError(t, p.Handshake(testName))
return p
}
func NewCompressedMuxerPair(t assert.TestingT, testName string, quality CompressionSetting, f MuxedStreamFunc) *DefaultMuxerPair {
origin, edge := net.Pipe()
p := &DefaultMuxerPair{
OriginMuxConfig: MuxerConfig{
Timeout: time.Second,
Handler: f,
IsClient: true,
Name: "origin",
CompressionQuality: quality,
Log: &log,
HeartbeatInterval: defaultTimeout,
MaxHeartbeats: defaultRetries,
},
OriginConn: origin,
EdgeMuxConfig: MuxerConfig{
Timeout: time.Second,
IsClient: false,
Name: "edge",
CompressionQuality: quality,
Log: &log,
HeartbeatInterval: defaultTimeout,
MaxHeartbeats: defaultRetries,
},
EdgeConn: edge,
doneC: make(chan struct{}),
}
assert.NoError(t, p.Handshake(testName))
return p
}
func (p *DefaultMuxerPair) Handshake(testName string) error {
ctx, cancel := context.WithTimeout(context.Background(), testHandshakeTimeout)
defer cancel()
errGroup, _ := errgroup.WithContext(ctx)
errGroup.Go(func() (err error) {
p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, ActiveStreams)
return errors.Wrap(err, "edge handshake failure")
})
errGroup.Go(func() (err error) {
p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, ActiveStreams)
return errors.Wrap(err, "origin handshake failure")
})
return errGroup.Wait()
}
func (p *DefaultMuxerPair) Serve(t assert.TestingT) {
ctx := context.Background()
var wg sync.WaitGroup
wg.Add(2)
go func() {
err := p.EdgeMux.Serve(ctx)
if err != nil && err != io.EOF && err != io.ErrClosedPipe {
t.Errorf("error in edge muxer Serve(): %s", err)
}
p.OriginMux.Shutdown()
wg.Done()
}()
go func() {
err := p.OriginMux.Serve(ctx)
if err != nil && err != io.EOF && err != io.ErrClosedPipe {
t.Errorf("error in origin muxer Serve(): %s", err)
}
p.EdgeMux.Shutdown()
wg.Done()
}()
go func() {
// notify when both muxes have stopped serving
wg.Wait()
close(p.doneC)
}()
}
func (p *DefaultMuxerPair) Wait(t *testing.T) {
select {
case <-p.doneC:
return
case <-time.After(5 * time.Second):
t.Fatal("timeout waiting for shutdown")
}
}
func (p *DefaultMuxerPair) OpenEdgeMuxStream(headers []Header, body io.Reader) (*MuxedStream, error) {
ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
defer cancel()
return p.EdgeMux.OpenStream(ctx, headers, body)
}
func TestHandshake(t *testing.T) {
f := func(stream *MuxedStream) error {
return nil
}
muxPair := NewDefaultMuxerPair(t, t.Name(), f)
AssertIfPipeReadable(t, muxPair.OriginConn)
AssertIfPipeReadable(t, muxPair.EdgeConn)
}
func TestSingleStream(t *testing.T) {
f := MuxedStreamFunc(func(stream *MuxedStream) error {
if len(stream.Headers) != 1 {
t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "test-header" {
t.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "headerValue" {
t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
}
_ = stream.WriteHeaders([]Header{
{Name: "response-header", Value: "responseValue"},
})
buf := []byte("Hello world")
_, _ = stream.Write(buf)
n, err := io.ReadFull(stream, buf)
if n > 0 {
t.Fatalf("read %d bytes after EOF", n)
}
if err != io.EOF {
t.Fatalf("expected EOF, got %s", err)
}
return nil
})
muxPair := NewDefaultMuxerPair(t, t.Name(), f)
muxPair.Serve(t)
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
if err != nil {
t.Fatalf("error in OpenStream: %s", err)
}
if len(stream.Headers) != 1 {
t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "response-header" {
t.Fatalf("expected header name %s, got %s", "response-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "responseValue" {
t.Fatalf("expected header value %s, got %s", "responseValue", stream.Headers[0].Value)
}
responseBody := make([]byte, 11)
n, err := io.ReadFull(stream, responseBody)
if err != nil {
t.Fatalf("error from (*MuxedStream).Read: %s", err)
}
if n != len(responseBody) {
t.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n)
}
if string(responseBody) != "Hello world" {
t.Fatalf("expected response body %s, got %s", "Hello world", responseBody)
}
_ = stream.Close()
n, err = stream.Write([]byte("aaaaa"))
if n > 0 {
t.Fatalf("wrote %d bytes after EOF", n)
}
if err != io.EOF {
t.Fatalf("expected EOF, got %s", err)
}
}
func TestSingleStreamLargeResponseBody(t *testing.T) {
bodySize := 1 << 24
f := MuxedStreamFunc(func(stream *MuxedStream) error {
if len(stream.Headers) != 1 {
t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "test-header" {
t.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "headerValue" {
t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
}
_ = stream.WriteHeaders([]Header{
{Name: "response-header", Value: "responseValue"},
})
payload := make([]byte, bodySize)
for i := range payload {
payload[i] = byte(i % 256)
}
t.Log("Writing payload...")
n, err := stream.Write(payload)
t.Logf("Wrote %d bytes into the stream", n)
if err != nil {
t.Fatalf("origin write error: %s", err)
}
if n != len(payload) {
t.Fatalf("origin short write: %d/%d bytes", n, len(payload))
}
return nil
})
muxPair := NewDefaultMuxerPair(t, t.Name(), f)
muxPair.Serve(t)
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
if err != nil {
t.Fatalf("error in OpenStream: %s", err)
}
if len(stream.Headers) != 1 {
t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "response-header" {
t.Fatalf("expected header name %s, got %s", "response-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "responseValue" {
t.Fatalf("expected header value %s, got %s", "responseValue", stream.Headers[0].Value)
}
responseBody := make([]byte, bodySize)
n, err := io.ReadFull(stream, responseBody)
if err != nil {
t.Fatalf("error from (*MuxedStream).Read: %s", err)
}
if n != len(responseBody) {
t.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n)
}
}
func TestMultipleStreams(t *testing.T) {
f := MuxedStreamFunc(func(stream *MuxedStream) error {
if len(stream.Headers) != 1 {
t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "client-token" {
t.Fatalf("expected header name %s, got %s", "client-token", stream.Headers[0].Name)
}
log.Debug().Msgf("Got request for stream %s", stream.Headers[0].Value)
_ = stream.WriteHeaders([]Header{
{Name: "response-token", Value: stream.Headers[0].Value},
})
log.Debug().Msgf("Wrote headers for stream %s", stream.Headers[0].Value)
_, _ = stream.Write([]byte("OK"))
log.Debug().Msgf("Wrote body for stream %s", stream.Headers[0].Value)
return nil
})
muxPair := NewDefaultMuxerPair(t, t.Name(), f)
muxPair.Serve(t)
maxStreams := 64
errorsC := make(chan error, maxStreams)
var wg sync.WaitGroup
wg.Add(maxStreams)
for i := 0; i < maxStreams; i++ {
go func(tokenId int) {
defer wg.Done()
tokenString := fmt.Sprintf("%d", tokenId)
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "client-token", Value: tokenString}},
nil,
)
log.Debug().Msgf("Got headers for stream %d", tokenId)
if err != nil {
errorsC <- err
return
}
if len(stream.Headers) != 1 {
errorsC <- fmt.Errorf("stream %d has error: expected %d headers, got %d", stream.streamID, 1, len(stream.Headers))
return
}
if stream.Headers[0].Name != "response-token" {
errorsC <- fmt.Errorf("stream %d has error: expected header name %s, got %s", stream.streamID, "response-token", stream.Headers[0].Name)
return
}
if stream.Headers[0].Value != tokenString {
errorsC <- fmt.Errorf("stream %d has error: expected header value %s, got %s", stream.streamID, tokenString, stream.Headers[0].Value)
return
}
responseBody := make([]byte, 2)
n, err := io.ReadFull(stream, responseBody)
if err != nil {
errorsC <- fmt.Errorf("stream %d has error: error from (*MuxedStream).Read: %s", stream.streamID, err)
return
}
if n != len(responseBody) {
errorsC <- fmt.Errorf("stream %d has error: expected response body to have %d bytes, got %d", stream.streamID, len(responseBody), n)
return
}
if string(responseBody) != "OK" {
errorsC <- fmt.Errorf("stream %d has error: expected response body %s, got %s", stream.streamID, "OK", responseBody)
return
}
}(i)
}
wg.Wait()
close(errorsC)
testFail := false
for err := range errorsC {
testFail = true
log.Error().Msgf("%s", err)
}
if testFail {
t.Fatalf("TestMultipleStreams failed")
}
}
func TestMultipleStreamsFlowControl(t *testing.T) {
maxStreams := 32
responseSizes := make([]int32, maxStreams)
for i := 0; i < maxStreams; i++ {
responseSizes[i] = rand.Int31n(int32(defaultWindowSize << 4))
}
f := MuxedStreamFunc(func(stream *MuxedStream) error {
if len(stream.Headers) != 1 {
t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "test-header" {
t.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "headerValue" {
t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
}
_ = stream.WriteHeaders([]Header{
{Name: "response-header", Value: "responseValue"},
})
payload := make([]byte, responseSizes[(stream.streamID-2)/2])
for i := range payload {
payload[i] = byte(i % 256)
}
n, err := stream.Write(payload)
if err != nil {
t.Fatalf("origin write error: %s", err)
}
if n != len(payload) {
t.Fatalf("origin short write: %d/%d bytes", n, len(payload))
}
return nil
})
muxPair := NewDefaultMuxerPair(t, t.Name(), f)
muxPair.Serve(t)
errGroup, _ := errgroup.WithContext(context.Background())
for i := 0; i < maxStreams; i++ {
errGroup.Go(func() error {
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
if err != nil {
return fmt.Errorf("error in OpenStream: %d %s", stream.streamID, err)
}
if len(stream.Headers) != 1 {
return fmt.Errorf("stream %d expected %d headers, got %d", stream.streamID, 1, len(stream.Headers))
}
if stream.Headers[0].Name != "response-header" {
return fmt.Errorf("stream %d expected header name %s, got %s", stream.streamID, "response-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "responseValue" {
return fmt.Errorf("stream %d expected header value %s, got %s", stream.streamID, "responseValue", stream.Headers[0].Value)
}
responseBody := make([]byte, responseSizes[(stream.streamID-2)/2])
n, err := io.ReadFull(stream, responseBody)
if err != nil {
return fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err)
}
if n != len(responseBody) {
return fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(responseBody), n)
}
return nil
})
}
assert.NoError(t, errGroup.Wait())
}
func TestGracefulShutdown(t *testing.T) {
sendC := make(chan struct{})
responseBuf := bytes.Repeat([]byte("Hello world"), 65536)
f := MuxedStreamFunc(func(stream *MuxedStream) error {
_ = stream.WriteHeaders([]Header{
{Name: "response-header", Value: "responseValue"},
})
<-sendC
log.Debug().Msgf("Writing %d bytes", len(responseBuf))
_, _ = stream.Write(responseBuf)
_ = stream.CloseWrite()
log.Debug().Msgf("Wrote %d bytes", len(responseBuf))
// Reading from the stream will block until the edge closes its end of the stream.
// Otherwise, we'll close the whole connection before receiving the 'stream closed'
// message from the edge.
// Graceful shutdown works if you omit this, it just gives spurious errors for now -
// TODO ignore errors when writing 'stream closed' and we're shutting down.
_, _ = stream.Read([]byte{0})
log.Debug().Msgf("Handler ends")
return nil
})
muxPair := NewDefaultMuxerPair(t, t.Name(), f)
muxPair.Serve(t)
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
if err != nil {
t.Fatalf("error in OpenStream: %s", err)
}
// Start graceful shutdown of the edge mux - this should also close the origin mux when done
muxPair.EdgeMux.Shutdown()
close(sendC)
responseBody := make([]byte, len(responseBuf))
log.Debug().Msgf("Waiting for %d bytes", len(responseBuf))
n, err := io.ReadFull(stream, responseBody)
if err != nil {
t.Fatalf("error from (*MuxedStream).Read with %d bytes read: %s", n, err)
}
if n != len(responseBody) {
t.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n)
}
if !bytes.Equal(responseBuf, responseBody) {
t.Fatalf("response body mismatch")
}
_ = stream.Close()
muxPair.Wait(t)
}
func TestUnexpectedShutdown(t *testing.T) {
sendC := make(chan struct{})
handlerFinishC := make(chan struct{})
responseBuf := bytes.Repeat([]byte("Hello world"), 65536)
f := MuxedStreamFunc(func(stream *MuxedStream) error {
defer close(handlerFinishC)
_ = stream.WriteHeaders([]Header{
{Name: "response-header", Value: "responseValue"},
})
<-sendC
n, err := stream.Read([]byte{0})
if err != io.EOF {
t.Fatalf("unexpected error from (*MuxedStream).Read: %s", err)
}
if n != 0 {
t.Fatalf("expected empty read, got %d bytes", n)
}
// Write comes after read, because write buffers data before it is flushed. It wouldn't know about EOF
// until some time later. Calling read first forces it to know about EOF now.
_, err = stream.Write(responseBuf)
if err != io.EOF {
t.Fatalf("unexpected error from (*MuxedStream).Write: %s", err)
}
return nil
})
muxPair := NewDefaultMuxerPair(t, t.Name(), f)
muxPair.Serve(t)
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
// Close the underlying connection before telling the origin to write.
_ = muxPair.EdgeConn.Close()
close(sendC)
if err != nil {
t.Fatalf("error in OpenStream: %s", err)
}
responseBody := make([]byte, len(responseBuf))
n, err := io.ReadFull(stream, responseBody)
if err != io.EOF {
t.Fatalf("unexpected error from (*MuxedStream).Read: %s", err)
}
if n != 0 {
t.Fatalf("expected response body to have %d bytes, got %d", 0, n)
}
// The write ordering requirement explained in the origin handler applies here too.
_, err = stream.Write(responseBuf)
if err != io.EOF {
t.Fatalf("unexpected error from (*MuxedStream).Write: %s", err)
}
<-handlerFinishC
}
func EchoHandler(stream *MuxedStream) error {
var buf bytes.Buffer
_, _ = fmt.Fprintf(&buf, "Hello, world!\n\n# REQUEST HEADERS:\n\n")
for _, header := range stream.Headers {
_, _ = fmt.Fprintf(&buf, "[%s] = %s\n", header.Name, header.Value)
}
_ = stream.WriteHeaders([]Header{
{Name: ":status", Value: "200"},
{Name: "server", Value: "Echo-server/1.0"},
{Name: "date", Value: time.Now().Format(time.RFC850)},
{Name: "content-type", Value: "text/html; charset=utf-8"},
{Name: "content-length", Value: strconv.Itoa(buf.Len())},
})
_, _ = buf.WriteTo(stream)
return nil
}
func TestOpenAfterDisconnect(t *testing.T) {
for i := 0; i < 3; i++ {
muxPair := NewDefaultMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), i), EchoHandler)
muxPair.Serve(t)
switch i {
case 0:
// Close both directions of the connection to cause EOF on both peers.
_ = muxPair.OriginConn.Close()
_ = muxPair.EdgeConn.Close()
case 1:
// Close origin conn to cause EOF on origin first.
_ = muxPair.OriginConn.Close()
case 2:
// Close edge conn to cause EOF on edge first.
_ = muxPair.EdgeConn.Close()
}
_, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
if err != ErrStreamRequestConnectionClosed && err != ErrResponseHeadersConnectionClosed {
t.Fatalf("case %v: unexpected error in OpenStream: %v", i, err)
}
}
}
func TestHPACK(t *testing.T) {
muxPair := NewDefaultMuxerPair(t, t.Name(), EchoHandler)
muxPair.Serve(t)
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{
{Name: ":method", Value: "RPC"},
{Name: ":scheme", Value: "capnp"},
{Name: ":path", Value: "*"},
},
nil,
)
if err != nil {
t.Fatalf("error in OpenStream: %s", err)
}
_ = stream.Close()
for i := 0; i < 3; i++ {
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{
{Name: ":method", Value: "GET"},
{Name: ":scheme", Value: "https"},
{Name: ":authority", Value: "tunnel.otterlyadorable.co.uk"},
{Name: ":path", Value: "/get"},
{Name: "accept-encoding", Value: "gzip"},
{Name: "cf-ray", Value: "378948953f044408-SFO-DOG"},
{Name: "cf-visitor", Value: "{\"scheme\":\"https\"}"},
{Name: "cf-connecting-ip", Value: "2400:cb00:0025:010d:0000:0000:0000:0001"},
{Name: "x-forwarded-for", Value: "2400:cb00:0025:010d:0000:0000:0000:0001"},
{Name: "x-forwarded-proto", Value: "https"},
{Name: "accept-language", Value: "en-gb"},
{Name: "referer", Value: "https://tunnel.otterlyadorable.co.uk/"},
{Name: "cookie", Value: "__cfduid=d4555095065f92daedc059490771967d81493032162"},
{Name: "connection", Value: "Keep-Alive"},
{Name: "cf-ipcountry", Value: "US"},
{Name: "accept", Value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
{Name: "user-agent", Value: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4"},
},
nil,
)
if err != nil {
t.Fatalf("error in OpenStream: %s", err)
}
if len(stream.Headers) == 0 {
t.Fatal("response has no headers")
}
if stream.Headers[0].Name != ":status" {
t.Fatalf("first header should be status, found %s instead", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "200" {
t.Fatalf("expected status 200, got %s", stream.Headers[0].Value)
}
_, _ = io.ReadAll(stream)
_ = stream.Close()
}
}
func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
errC := make(chan error)
go func() {
b := []byte{0}
n, err := pipe.Read(b)
if n > 0 {
t.Errorf("read pipe was not empty")
return
}
errC <- err
}()
select {
case err := <-errC:
if err != nil {
t.Fatalf("read error: %s", err)
}
case <-time.After(100 * time.Millisecond):
// nothing to read
}
}
func sampleSiteHandler(files map[string][]byte) MuxedStreamFunc {
return func(stream *MuxedStream) error {
var contentType string
var pathHeader Header
for _, h := range stream.Headers {
if h.Name == ":path" {
pathHeader = h
break
}
}
if pathHeader.Name != ":path" {
return fmt.Errorf("Couldn't find :path header in test")
}
if strings.Contains(pathHeader.Value, "html") {
contentType = "text/html; charset=utf-8"
} else if strings.Contains(pathHeader.Value, "js") {
contentType = "application/javascript"
} else if strings.Contains(pathHeader.Value, "css") {
contentType = "text/css"
} else {
contentType = "img/gif"
}
_ = stream.WriteHeaders([]Header{
{Name: "content-type", Value: contentType},
})
log.Debug().Msgf("Wrote headers for stream %s", pathHeader.Value)
file, ok := files[pathHeader.Value]
if !ok {
return fmt.Errorf("%s content is not preloaded", pathHeader.Value)
}
_, _ = stream.Write(file)
log.Debug().Msgf("Wrote body for stream %s", pathHeader.Value)
return nil
}
}
func sampleSiteTest(muxPair *DefaultMuxerPair, path string, files map[string][]byte) error {
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{
{Name: ":method", Value: "GET"},
{Name: ":scheme", Value: "https"},
{Name: ":authority", Value: "tunnel.otterlyadorable.co.uk"},
{Name: ":path", Value: path},
{Name: "accept-encoding", Value: "br, gzip"},
{Name: "cf-ray", Value: "378948953f044408-SFO-DOG"},
},
nil,
)
if err != nil {
return fmt.Errorf("error in OpenStream: %v", err)
}
file, ok := files[path]
if !ok {
return fmt.Errorf("%s content is not preloaded", path)
}
responseBody := make([]byte, len(file))
n, err := io.ReadFull(stream, responseBody)
if err != nil {
return fmt.Errorf("error from (*MuxedStream).Read: %v", err)
}
if n != len(file) {
return fmt.Errorf("expected response body to have %d bytes, got %d", len(file), n)
}
if string(responseBody[:n]) != string(file) {
return fmt.Errorf("expected response body %s, got %s", file, responseBody[:n])
}
return nil
}
func loadSampleFiles(paths []string) (map[string][]byte, error) {
files := make(map[string][]byte)
for _, path := range paths {
if _, ok := files[path]; !ok {
expectBody, err := os.ReadFile(path)
if err != nil {
return nil, err
}
files[path] = expectBody
}
}
return files, nil
}
func BenchmarkOpenStream(b *testing.B) {
const streams = 5000
for i := 0; i < b.N; i++ {
b.StopTimer()
f := MuxedStreamFunc(func(stream *MuxedStream) error {
if len(stream.Headers) != 1 {
b.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "test-header" {
b.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "headerValue" {
b.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
}
_ = stream.WriteHeaders([]Header{
{Name: "response-header", Value: "responseValue"},
})
return nil
})
muxPair := NewDefaultMuxerPair(b, fmt.Sprintf("%s_%d", b.Name(), i), f)
muxPair.Serve(b)
b.StartTimer()
openStreams(b, muxPair, streams)
}
}
func openStreams(b *testing.B, muxPair *DefaultMuxerPair, n int) {
errGroup, _ := errgroup.WithContext(context.Background())
for i := 0; i < n; i++ {
errGroup.Go(func() error {
_, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
return err
})
}
assert.NoError(b, errGroup.Wait())
}
func BenchmarkSingleStreamLargeResponseBody(b *testing.B) {
const bodySize = 1 << 24
const writeBufferSize = 16 << 10
const writeN = bodySize / writeBufferSize
payload := make([]byte, writeBufferSize)
for i := range payload {
payload[i] = byte(i % 256)
}
const readBufferSize = 16 << 10
const readN = bodySize / readBufferSize
responseBody := make([]byte, readBufferSize)
f := MuxedStreamFunc(func(stream *MuxedStream) error {
if len(stream.Headers) != 1 {
b.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "test-header" {
b.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "headerValue" {
b.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
}
_ = stream.WriteHeaders([]Header{
{Name: "response-header", Value: "responseValue"},
})
for i := 0; i < writeN; i++ {
n, err := stream.Write(payload)
if err != nil {
b.Fatalf("origin write error: %s", err)
}
if n != len(payload) {
b.Fatalf("origin short write: %d/%d bytes", n, len(payload))
}
}
return nil
})
name := fmt.Sprintf("%s_%d", b.Name(), rand.Int())
origin, edge := net.Pipe()
muxPair := &DefaultMuxerPair{
OriginMuxConfig: MuxerConfig{
Timeout: testHandshakeTimeout,
Handler: f,
IsClient: true,
Name: "origin",
Log: &log,
DefaultWindowSize: defaultWindowSize,
MaxWindowSize: maxWindowSize,
StreamWriteBufferMaxLen: defaultWriteBufferMaxLen,
HeartbeatInterval: defaultTimeout,
MaxHeartbeats: defaultRetries,
},
OriginConn: origin,
EdgeMuxConfig: MuxerConfig{
Timeout: testHandshakeTimeout,
IsClient: false,
Name: "edge",
Log: &log,
DefaultWindowSize: defaultWindowSize,
MaxWindowSize: maxWindowSize,
StreamWriteBufferMaxLen: defaultWriteBufferMaxLen,
HeartbeatInterval: defaultTimeout,
MaxHeartbeats: defaultRetries,
},
EdgeConn: edge,
doneC: make(chan struct{}),
}
assert.NoError(b, muxPair.Handshake(name))
muxPair.Serve(b)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
stream, err := muxPair.OpenEdgeMuxStream(
[]Header{{Name: "test-header", Value: "headerValue"}},
nil,
)
if err != nil {
b.Fatalf("error in OpenStream: %s", err)
}
if len(stream.Headers) != 1 {
b.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
}
if stream.Headers[0].Name != "response-header" {
b.Fatalf("expected header name %s, got %s", "response-header", stream.Headers[0].Name)
}
if stream.Headers[0].Value != "responseValue" {
b.Fatalf("expected header value %s, got %s", "responseValue", stream.Headers[0].Value)
}
for k := 0; k < readN; k++ {
n, err := io.ReadFull(stream, responseBody)
if err != nil {
b.Fatalf("error from (*MuxedStream).Read: %s", err)
}
if n != len(responseBody) {
b.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n)
}
}
}
}

81
h2mux/idletimer.go
View File

@ -1,81 +0,0 @@
package h2mux
import (
"math/rand"
"sync"
"time"
)
// IdleTimer is a type of Timer designed for managing heartbeats on an idle connection.
// The timer ticks on an interval with added jitter to avoid accidental synchronisation
// between two endpoints. It tracks the number of retries/ticks since the connection was
// last marked active.
//
// The methods of IdleTimer must not be called while a goroutine is reading from C.
type IdleTimer struct {
// The channel on which ticks are delivered.
C <-chan time.Time
// A timer used to measure idle connection time. Reset after sending data.
idleTimer *time.Timer
// The maximum length of time a connection is idle before sending a ping.
idleDuration time.Duration
// A pseudorandom source used to add jitter to the idle duration.
randomSource *rand.Rand
// The maximum number of retries allowed.
maxRetries uint64
// The number of retries since the connection was last marked active.
retries uint64
// A lock to prevent race condition while checking retries
stateLock sync.RWMutex
}
func NewIdleTimer(idleDuration time.Duration, maxRetries uint64) *IdleTimer {
t := &IdleTimer{
idleTimer: time.NewTimer(idleDuration),
idleDuration: idleDuration,
randomSource: rand.New(rand.NewSource(time.Now().Unix())),
maxRetries: maxRetries,
}
t.C = t.idleTimer.C
return t
}
// Retry should be called when retrying the idle timeout. If the maximum number of retries
// has been met, returns false.
// After calling this function and sending a heartbeat, call ResetTimer. Since sending the
// heartbeat could be a blocking operation, we resetting the timer after the write completes
// to avoid it expiring during the write.
func (t *IdleTimer) Retry() bool {
t.stateLock.Lock()
defer t.stateLock.Unlock()
if t.retries >= t.maxRetries {
return false
}
t.retries++
return true
}
func (t *IdleTimer) RetryCount() uint64 {
t.stateLock.RLock()
defer t.stateLock.RUnlock()
return t.retries
}
// MarkActive resets the idle connection timer and suppresses any outstanding idle events.
func (t *IdleTimer) MarkActive() {
if !t.idleTimer.Stop() {
// eat the timer event to prevent spurious pings
<-t.idleTimer.C
}
t.stateLock.Lock()
t.retries = 0
t.stateLock.Unlock()
t.ResetTimer()
}
// Reset the idle timer according to the configured duration, with some added jitter.
func (t *IdleTimer) ResetTimer() {
jitter := time.Duration(t.randomSource.Int63n(int64(t.idleDuration)))
t.idleTimer.Reset(t.idleDuration + jitter)
}

31
h2mux/idletimer_test.go
View File

@ -1,31 +0,0 @@
package h2mux
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestRetry(t *testing.T) {
timer := NewIdleTimer(time.Second, 2)
assert.Equal(t, uint64(0), timer.RetryCount())
ok := timer.Retry()
assert.True(t, ok)
assert.Equal(t, uint64(1), timer.RetryCount())
ok = timer.Retry()
assert.True(t, ok)
assert.Equal(t, uint64(2), timer.RetryCount())
ok = timer.Retry()
assert.False(t, ok)
}
func TestMarkActive(t *testing.T) {
timer := NewIdleTimer(time.Second, 2)
assert.Equal(t, uint64(0), timer.RetryCount())
ok := timer.Retry()
assert.True(t, ok)
assert.Equal(t, uint64(1), timer.RetryCount())
timer.MarkActive()
assert.Equal(t, uint64(0), timer.RetryCount())
}

457
h2mux/muxedstream.go
View File

@ -1,457 +0,0 @@
package h2mux
import (
"bytes"
"io"
"sync"
)
type ReadWriteLengther interface {
io.ReadWriter
Reset()
Len() int
}
type ReadWriteClosedCloser interface {
io.ReadWriteCloser
Closed() bool
}
// MuxedStreamDataSignaller is a write-only *ReadyList
type MuxedStreamDataSignaller interface {
// Non-blocking: call this when data is ready to be sent for the given stream ID.
Signal(ID uint32)
}
type Header struct {
Name, Value string
}
// MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
type MuxedStream struct {
streamID uint32
// The "Receive" end of the stream
readBufferLock sync.RWMutex
readBuffer ReadWriteClosedCloser
// This is the amount of bytes that are in our receive window
// (how much data we can receive into this stream).
receiveWindow uint32
// current receive window size limit. Exponentially increase it when it's exhausted
receiveWindowCurrentMax uint32
// hard limit set in http2 spec. 2^31-1
receiveWindowMax uint32
// The desired size increment for receiveWindow.
// If this is nonzero, a WINDOW_UPDATE frame needs to be sent.
windowUpdate uint32
// The headers that were most recently received.
// Particularly:
// * for an eyeball-initiated stream (as passed to TunnelHandler::ServeStream),
// these are the request headers
// * for a cloudflared-initiated stream (as created by Register/UnregisterTunnel),
// these are the response headers.
// They are useful in both of these contexts; hence `Headers` is public.
Headers []Header
// For use in the context of a cloudflared-initiated stream.
responseHeadersReceived chan struct{}
// The "Send" end of the stream
writeLock sync.Mutex
writeBuffer ReadWriteLengther
// The maximum capacity that the send buffer should grow to.
writeBufferMaxLen int
// A channel to be notified when the send buffer is not full.
writeBufferHasSpace chan struct{}
// This is the amount of bytes that are in the peer's receive window
// (how much data we can send from this stream).
sendWindow uint32
// The muxer's readyList
readyList MuxedStreamDataSignaller
// The headers that should be sent, and a flag so we only send them once.
headersSent bool
writeHeaders []Header
// EOF-related fields
// true if the write end of this stream has been closed
writeEOF bool
// true if we have sent EOF to the peer
sentEOF bool
// true if the peer sent us an EOF
receivedEOF bool
// Compression-related fields
receivedUseDict bool
method string
contentType string
path string
dictionaries h2Dictionaries
}
type TunnelHostname string
func (th TunnelHostname) String() string {
return string(th)
}
func (th TunnelHostname) IsSet() bool {
return th != ""
}
func NewStream(config MuxerConfig, writeHeaders []Header, readyList MuxedStreamDataSignaller, dictionaries h2Dictionaries) *MuxedStream {
return &MuxedStream{
responseHeadersReceived: make(chan struct{}),
readBuffer: NewSharedBuffer(),
writeBuffer: &bytes.Buffer{},
writeBufferMaxLen: config.StreamWriteBufferMaxLen,
writeBufferHasSpace: make(chan struct{}, 1),
receiveWindow: config.DefaultWindowSize,
receiveWindowCurrentMax: config.DefaultWindowSize,
receiveWindowMax: config.MaxWindowSize,
sendWindow: config.DefaultWindowSize,
readyList: readyList,
writeHeaders: writeHeaders,
dictionaries: dictionaries,
}
}
func (s *MuxedStream) Read(p []byte) (n int, err error) {
var readBuffer ReadWriteClosedCloser
if s.dictionaries.read != nil {
s.readBufferLock.RLock()
readBuffer = s.readBuffer
s.readBufferLock.RUnlock()
} else {
readBuffer = s.readBuffer
}
n, err = readBuffer.Read(p)
s.replenishReceiveWindow(uint32(n))
return
}
// Blocks until len(p) bytes have been written to the buffer
func (s *MuxedStream) Write(p []byte) (int, error) {
// If assignDictToStream returns success, then it will have acquired the
// writeLock. Otherwise we must acquire it ourselves.
ok := assignDictToStream(s, p)
if !ok {
s.writeLock.Lock()
}
defer s.writeLock.Unlock()
if s.writeEOF {
return 0, io.EOF
}
// pre-allocate some space in the write buffer if possible
if buffer, ok := s.writeBuffer.(*bytes.Buffer); ok {
if buffer.Cap() == 0 {
buffer.Grow(writeBufferInitialSize)
}
}
totalWritten := 0
for totalWritten < len(p) {
// If the buffer is full, block till there is more room.
// Use a loop to recheck the buffer size after the lock is reacquired.
for s.writeBufferMaxLen <= s.writeBuffer.Len() {
s.awaitWriteBufferHasSpace()
if s.writeEOF {
return totalWritten, io.EOF
}
}
amountToWrite := len(p) - totalWritten
spaceAvailable := s.writeBufferMaxLen - s.writeBuffer.Len()
if spaceAvailable < amountToWrite {
amountToWrite = spaceAvailable
}
amountWritten, err := s.writeBuffer.Write(p[totalWritten : totalWritten+amountToWrite])
totalWritten += amountWritten
if err != nil {
return totalWritten, err
}
s.writeNotify()
}
return totalWritten, nil
}
func (s *MuxedStream) Close() error {
// TUN-115: Close the write buffer before the read buffer.
// In the case of shutdown, read will not get new data, but the write buffer can still receive
// new data. Closing read before write allows application to race between a failed read and a
// successful write, even though this close should appear to be atomic.
// This can't happen the other way because reads may succeed after a failed write; if we read
// past EOF the application will block until we close the buffer.
err := s.CloseWrite()
if err != nil {
if s.CloseRead() == nil {
// don't bother the caller with errors if at least one close succeeded
return nil
}
return err
}
return s.CloseRead()
}
func (s *MuxedStream) CloseRead() error {
return s.readBuffer.Close()
}
func (s *MuxedStream) CloseWrite() error {
s.writeLock.Lock()
defer s.writeLock.Unlock()
if s.writeEOF {
return io.EOF
}
s.writeEOF = true
if c, ok := s.writeBuffer.(io.Closer); ok {
c.Close()
}
// Allow MuxedStream::Write() to terminate its loop with err=io.EOF, if needed
s.notifyWriteBufferHasSpace()
// We need to send something over the wire, even if it's an END_STREAM with no data
s.writeNotify()
return nil
}
func (s *MuxedStream) WriteClosed() bool {
s.writeLock.Lock()
defer s.writeLock.Unlock()
return s.writeEOF
}
func (s *MuxedStream) WriteHeaders(headers []Header) error {
s.writeLock.Lock()
defer s.writeLock.Unlock()
if s.writeHeaders != nil {
return ErrStreamHeadersSent
}
if s.dictionaries.write != nil {
dictWriter := s.dictionaries.write.getDictWriter(s, headers)
if dictWriter != nil {
s.writeBuffer = dictWriter
}
}
s.writeHeaders = headers
s.headersSent = false
s.writeNotify()
return nil
}
// IsRPCStream returns if the stream is used to transport RPC.
func (s *MuxedStream) IsRPCStream() bool {
rpcHeaders := RPCHeaders()
if len(s.Headers) != len(rpcHeaders) {
return false
}
// The headers order matters, so RPC stream should be opened with OpenRPCStream method and let MuxWriter serializes the headers.
for i, rpcHeader := range rpcHeaders {
if s.Headers[i] != rpcHeader {
return false
}
}
return true
}
// Block until a value is sent on writeBufferHasSpace.
// Must be called while holding writeLock
func (s *MuxedStream) awaitWriteBufferHasSpace() {
s.writeLock.Unlock()
<-s.writeBufferHasSpace
s.writeLock.Lock()
}
// Send a value on writeBufferHasSpace without blocking.
// Must be called while holding writeLock
func (s *MuxedStream) notifyWriteBufferHasSpace() {
select {
case s.writeBufferHasSpace <- struct{}{}:
default:
}
}
func (s *MuxedStream) getReceiveWindow() uint32 {
s.writeLock.Lock()
defer s.writeLock.Unlock()
return s.receiveWindow
}
func (s *MuxedStream) getSendWindow() uint32 {
s.writeLock.Lock()
defer s.writeLock.Unlock()
return s.sendWindow
}
// writeNotify must happen while holding writeLock.
func (s *MuxedStream) writeNotify() {
s.readyList.Signal(s.streamID)
}
// Call by muxreader when it gets a WindowUpdateFrame. This is an update of the peer's
// receive window (how much data we can send).
func (s *MuxedStream) replenishSendWindow(bytes uint32) {
s.writeLock.Lock()
defer s.writeLock.Unlock()
s.sendWindow += bytes
s.writeNotify()
}
// Call by muxreader when it receives a data frame
func (s *MuxedStream) consumeReceiveWindow(bytes uint32) bool {
s.writeLock.Lock()
defer s.writeLock.Unlock()
// received data size is greater than receive window/buffer
if s.receiveWindow < bytes {
return false
}
s.receiveWindow -= bytes
if s.receiveWindow < s.receiveWindowCurrentMax/2 && s.receiveWindowCurrentMax < s.receiveWindowMax {
// exhausting client send window (how much data client can send)
// and there is room to grow the receive window
newMax := s.receiveWindowCurrentMax << 1
if newMax > s.receiveWindowMax {
newMax = s.receiveWindowMax
}
s.windowUpdate += newMax - s.receiveWindowCurrentMax
s.receiveWindowCurrentMax = newMax
// notify MuxWriter to write WINDOW_UPDATE frame
s.writeNotify()
}
return true
}
// Arranges for the MuxWriter to send a WINDOW_UPDATE
// Called by MuxedStream::Read when data has left the read buffer.
func (s *MuxedStream) replenishReceiveWindow(bytes uint32) {
s.writeLock.Lock()
defer s.writeLock.Unlock()
s.windowUpdate += bytes
s.writeNotify()
}
// receiveEOF should be called when the peer indicates no more data will be sent.
// Returns true if the socket is now closed (i.e. the write side is already closed).
func (s *MuxedStream) receiveEOF() (closed bool) {
s.writeLock.Lock()
defer s.writeLock.Unlock()
s.receivedEOF = true
s.CloseRead()
return s.writeEOF && s.writeBuffer.Len() == 0
}
func (s *MuxedStream) gotReceiveEOF() bool {
s.writeLock.Lock()
defer s.writeLock.Unlock()
return s.receivedEOF
}
// MuxedStreamReader implements io.ReadCloser for the read end of the stream.
// This is useful for passing to functions that close the object after it is done reading,
// but you still want to be able to write data afterwards (e.g. http.Client).
type MuxedStreamReader struct {
*MuxedStream
}
func (s MuxedStreamReader) Read(p []byte) (n int, err error) {
return s.MuxedStream.Read(p)
}
func (s MuxedStreamReader) Close() error {
return s.MuxedStream.CloseRead()
}
// streamChunk represents a chunk of data to be written.
type streamChunk struct {
streamID uint32
// true if a HEADERS frame should be sent
sendHeaders bool
headers []Header
// nonzero if a WINDOW_UPDATE frame should be sent;
// in that case, it is the increment value to use
windowUpdate uint32
// true if data frames should be sent
sendData bool
eof bool
buffer []byte
offset int
}
// getChunk atomically extracts a chunk of data to be written by MuxWriter.
// The data returned will not exceed the send window for this stream.
func (s *MuxedStream) getChunk() *streamChunk {
s.writeLock.Lock()
defer s.writeLock.Unlock()
chunk := &streamChunk{
streamID: s.streamID,
sendHeaders: !s.headersSent,
headers: s.writeHeaders,
windowUpdate: s.windowUpdate,
sendData: !s.sentEOF,
eof: s.writeEOF && uint32(s.writeBuffer.Len()) <= s.sendWindow,
}
// Copy at most s.sendWindow bytes, adjust the sendWindow accordingly
toCopy := int(s.sendWindow)
if toCopy > s.writeBuffer.Len() {
toCopy = s.writeBuffer.Len()
}
if toCopy > 0 {
buf := make([]byte, toCopy)
writeLen, _ := s.writeBuffer.Read(buf)
chunk.buffer = buf[:writeLen]
s.sendWindow -= uint32(writeLen)
}
// Allow MuxedStream::Write() to continue, if needed
if s.writeBuffer.Len() < s.writeBufferMaxLen {
s.notifyWriteBufferHasSpace()
}
// When we write the chunk, we'll write the WINDOW_UPDATE frame if needed
s.receiveWindow += s.windowUpdate
s.windowUpdate = 0
// When we write the chunk, we'll write the headers if needed
s.headersSent = true
// if this chunk contains the end of the stream, close the stream now
if chunk.sendData && chunk.eof {
s.sentEOF = true
}
return chunk
}
func (c *streamChunk) sendHeadersFrame() bool {
return c.sendHeaders
}
func (c *streamChunk) sendWindowUpdateFrame() bool {
return c.windowUpdate > 0
}
func (c *streamChunk) sendDataFrame() bool {
return c.sendData
}
func (c *streamChunk) nextDataFrame(frameSize int) (payload []byte, endStream bool) {
bytesLeft := len(c.buffer) - c.offset
if frameSize > bytesLeft {
frameSize = bytesLeft
}
nextOffset := c.offset + frameSize
payload = c.buffer[c.offset:nextOffset]
c.offset = nextOffset
if c.offset == len(c.buffer) {
// this is the last data frame in this chunk
c.sendData = false
if c.eof {
endStream = true
}
}
return
}

127
h2mux/muxedstream_test.go
View File

@ -1,127 +0,0 @@
package h2mux
import (
"bytes"
"io"
"testing"
"github.com/stretchr/testify/assert"
)
const testWindowSize uint32 = 65535
const testMaxWindowSize uint32 = testWindowSize << 2
// Only sending WINDOW_UPDATE frame, so sendWindow should never change
func TestFlowControlSingleStream(t *testing.T) {
stream := &MuxedStream{
responseHeadersReceived: make(chan struct{}),
readBuffer: NewSharedBuffer(),
writeBuffer: &bytes.Buffer{},
receiveWindow: testWindowSize,
receiveWindowCurrentMax: testWindowSize,
receiveWindowMax: testMaxWindowSize,
sendWindow: testWindowSize,
readyList: NewReadyList(),
}
var tempWindowUpdate uint32
var tempStreamChunk *streamChunk
assert.True(t, stream.consumeReceiveWindow(testWindowSize/2))
dataSent := testWindowSize / 2
assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax)
assert.Equal(t, testWindowSize, stream.sendWindow)
assert.Equal(t, uint32(0), stream.windowUpdate)
tempStreamChunk = stream.getChunk()
assert.Equal(t, uint32(0), tempStreamChunk.windowUpdate)
assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax)
assert.Equal(t, testWindowSize, stream.sendWindow)
assert.Equal(t, uint32(0), stream.windowUpdate)
assert.True(t, stream.consumeReceiveWindow(2))
dataSent += 2
assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax)
assert.Equal(t, testWindowSize, stream.sendWindow)
assert.Equal(t, testWindowSize, stream.windowUpdate)
tempWindowUpdate = stream.windowUpdate
tempStreamChunk = stream.getChunk()
assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate)
assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow)
assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax)
assert.Equal(t, testWindowSize, stream.sendWindow)
assert.Equal(t, uint32(0), stream.windowUpdate)
assert.True(t, stream.consumeReceiveWindow(testWindowSize+10))
dataSent += testWindowSize + 10
assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow)
assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax)
assert.Equal(t, testWindowSize, stream.sendWindow)
assert.Equal(t, testWindowSize<<1, stream.windowUpdate)
tempWindowUpdate = stream.windowUpdate
tempStreamChunk = stream.getChunk()
assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate)
assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow)
assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax)
assert.Equal(t, testWindowSize, stream.sendWindow)
assert.Equal(t, uint32(0), stream.windowUpdate)
assert.False(t, stream.consumeReceiveWindow(testMaxWindowSize+1))
assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow)
assert.Equal(t, testMaxWindowSize, stream.receiveWindowCurrentMax)
}
func TestMuxedStreamEOF(t *testing.T) {
for i := 0; i < 4096; i++ {
readyList := NewReadyList()
stream := &MuxedStream{
streamID: 1,
readBuffer: NewSharedBuffer(),
receiveWindow: 65536,
receiveWindowMax: 65536,
sendWindow: 65536,
readyList: readyList,
}
go func() { stream.Close() }()
n, err := stream.Read([]byte{0})
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, n)
// Write comes after read, because write buffers data before it is flushed. It wouldn't know about EOF
// until some time later. Calling read first forces it to know about EOF now.
n, err = stream.Write([]byte{1})
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, n)
}
}
func TestIsRPCStream(t *testing.T) {
tests := []struct {
stream *MuxedStream
isRPCStream bool
}{
{
stream: &MuxedStream{},
isRPCStream: false,
},
{
stream: &MuxedStream{Headers: RPCHeaders()},
isRPCStream: true,
},
{
stream: &MuxedStream{Headers: []Header{
{Name: ":method", Value: "rpc"},
{Name: ":scheme", Value: "Capnp"},
{Name: ":path", Value: "/"},
}},
isRPCStream: false,
},
}
for _, test := range tests {
assert.Equal(t, test.isRPCStream, test.stream.IsRPCStream())
}
}

296
h2mux/muxmetrics.go
View File

@ -1,296 +0,0 @@
package h2mux
import (
"sync"
"time"
"github.com/golang-collections/collections/queue"
"github.com/rs/zerolog"
)
// data points used to compute average receive window and send window size
const (
// data points used to compute average receive window and send window size
dataPoints = 100
// updateFreq is set to 1 sec so we can get inbound & outbound byes/sec
updateFreq = time.Second
)
type muxMetricsUpdater interface {
// metrics returns the latest metrics
metrics() *MuxerMetrics
// run is a blocking call to start the event loop
run(log *zerolog.Logger) error
// updateRTTChan is called by muxReader to report new RTT measurements
updateRTT(rtt *roundTripMeasurement)
//updateReceiveWindowChan is called by muxReader and muxWriter when receiveWindow size is updated
updateReceiveWindow(receiveWindow uint32)
//updateSendWindowChan is called by muxReader and muxWriter when sendWindow size is updated
updateSendWindow(sendWindow uint32)
// updateInBoundBytesChan is called periodicallyby muxReader to report bytesRead
updateInBoundBytes(inBoundBytes uint64)
// updateOutBoundBytesChan is called periodically by muxWriter to report bytesWrote
updateOutBoundBytes(outBoundBytes uint64)
}
type muxMetricsUpdaterImpl struct {
// rttData keeps record of rtt, rttMin, rttMax and last measured time
rttData *rttData
// receiveWindowData keeps record of receive window measurement
receiveWindowData *flowControlData
// sendWindowData keeps record of send window measurement
sendWindowData *flowControlData
// inBoundRate is incoming bytes/sec
inBoundRate *rate
// outBoundRate is outgoing bytes/sec
outBoundRate *rate
// updateRTTChan is the channel to receive new RTT measurement
updateRTTChan chan *roundTripMeasurement
//updateReceiveWindowChan is the channel to receive updated receiveWindow size
updateReceiveWindowChan chan uint32
//updateSendWindowChan is the channel to receive updated sendWindow size
updateSendWindowChan chan uint32
// updateInBoundBytesChan us the channel to receive bytesRead
updateInBoundBytesChan chan uint64
// updateOutBoundBytesChan us the channel to receive bytesWrote
updateOutBoundBytesChan chan uint64
// shutdownC is to signal the muxerMetricsUpdater to shutdown
abortChan <-chan struct{}
compBytesBefore, compBytesAfter *AtomicCounter
}
type MuxerMetrics struct {
RTT, RTTMin, RTTMax time.Duration
ReceiveWindowAve, SendWindowAve float64
ReceiveWindowMin, ReceiveWindowMax, SendWindowMin, SendWindowMax uint32
InBoundRateCurr, InBoundRateMin, InBoundRateMax uint64
OutBoundRateCurr, OutBoundRateMin, OutBoundRateMax uint64
CompBytesBefore, CompBytesAfter *AtomicCounter
}
func (m *MuxerMetrics) CompRateAve() float64 {
if m.CompBytesBefore.Value() == 0 {
return 1.
}
return float64(m.CompBytesAfter.Value()) / float64(m.CompBytesBefore.Value())
}
type roundTripMeasurement struct {
receiveTime, sendTime time.Time
}
type rttData struct {
rtt, rttMin, rttMax time.Duration
lastMeasurementTime time.Time
lock sync.RWMutex
}
type flowControlData struct {
sum uint64
min, max uint32
queue *queue.Queue
lock sync.RWMutex
}
type rate struct {
curr uint64
min, max uint64
lock sync.RWMutex
}
func newMuxMetricsUpdater(
abortChan <-chan struct{},
compBytesBefore, compBytesAfter *AtomicCounter,
) muxMetricsUpdater {
updateRTTChan := make(chan *roundTripMeasurement, 1)
updateReceiveWindowChan := make(chan uint32, 1)
updateSendWindowChan := make(chan uint32, 1)
updateInBoundBytesChan := make(chan uint64)
updateOutBoundBytesChan := make(chan uint64)
return &muxMetricsUpdaterImpl{
rttData: newRTTData(),
receiveWindowData: newFlowControlData(),
sendWindowData: newFlowControlData(),
inBoundRate: newRate(),
outBoundRate: newRate(),
updateRTTChan: updateRTTChan,
updateReceiveWindowChan: updateReceiveWindowChan,
updateSendWindowChan: updateSendWindowChan,
updateInBoundBytesChan: updateInBoundBytesChan,
updateOutBoundBytesChan: updateOutBoundBytesChan,
abortChan: abortChan,
compBytesBefore: compBytesBefore,
compBytesAfter: compBytesAfter,
}
}
func (updater *muxMetricsUpdaterImpl) metrics() *MuxerMetrics {
m := &MuxerMetrics{}
m.RTT, m.RTTMin, m.RTTMax = updater.rttData.metrics()
m.ReceiveWindowAve, m.ReceiveWindowMin, m.ReceiveWindowMax = updater.receiveWindowData.metrics()
m.SendWindowAve, m.SendWindowMin, m.SendWindowMax = updater.sendWindowData.metrics()
m.InBoundRateCurr, m.InBoundRateMin, m.InBoundRateMax = updater.inBoundRate.get()
m.OutBoundRateCurr, m.OutBoundRateMin, m.OutBoundRateMax = updater.outBoundRate.get()
m.CompBytesBefore, m.CompBytesAfter = updater.compBytesBefore, updater.compBytesAfter
return m
}
func (updater *muxMetricsUpdaterImpl) run(log *zerolog.Logger) error {
defer log.Debug().Msg("mux - metrics: event loop finished")
for {
select {
case <-updater.abortChan:
log.Debug().Msgf("mux - metrics: Stopping mux metrics updater")
return nil
case roundTripMeasurement := <-updater.updateRTTChan:
go updater.rttData.update(roundTripMeasurement)
log.Debug().Msg("mux - metrics: Update rtt")
case receiveWindow := <-updater.updateReceiveWindowChan:
go updater.receiveWindowData.update(receiveWindow)
log.Debug().Msg("mux - metrics: Update receive window")
case sendWindow := <-updater.updateSendWindowChan:
go updater.sendWindowData.update(sendWindow)
log.Debug().Msg("mux - metrics: Update send window")
case inBoundBytes := <-updater.updateInBoundBytesChan:
// inBoundBytes is bytes/sec because the update interval is 1 sec
go updater.inBoundRate.update(inBoundBytes)
log.Debug().Msgf("mux - metrics: Inbound bytes %d", inBoundBytes)
case outBoundBytes := <-updater.updateOutBoundBytesChan:
// outBoundBytes is bytes/sec because the update interval is 1 sec
go updater.outBoundRate.update(outBoundBytes)
log.Debug().Msgf("mux - metrics: Outbound bytes %d", outBoundBytes)
}
}
}
func (updater *muxMetricsUpdaterImpl) updateRTT(rtt *roundTripMeasurement) {
select {
case updater.updateRTTChan <- rtt:
case <-updater.abortChan:
}
}
func (updater *muxMetricsUpdaterImpl) updateReceiveWindow(receiveWindow uint32) {
select {
case updater.updateReceiveWindowChan <- receiveWindow:
case <-updater.abortChan:
}
}
func (updater *muxMetricsUpdaterImpl) updateSendWindow(sendWindow uint32) {
select {
case updater.updateSendWindowChan <- sendWindow:
case <-updater.abortChan:
}
}
func (updater *muxMetricsUpdaterImpl) updateInBoundBytes(inBoundBytes uint64) {
select {
case updater.updateInBoundBytesChan <- inBoundBytes:
case <-updater.abortChan:
}
}
func (updater *muxMetricsUpdaterImpl) updateOutBoundBytes(outBoundBytes uint64) {
select {
case updater.updateOutBoundBytesChan <- outBoundBytes:
case <-updater.abortChan:
}
}
func newRTTData() *rttData {
return &rttData{}
}
func (r *rttData) update(measurement *roundTripMeasurement) {
r.lock.Lock()
defer r.lock.Unlock()
// discard pings before lastMeasurementTime
if r.lastMeasurementTime.After(measurement.sendTime) {
return
}
r.lastMeasurementTime = measurement.sendTime
r.rtt = measurement.receiveTime.Sub(measurement.sendTime)
if r.rttMax < r.rtt {
r.rttMax = r.rtt
}
if r.rttMin == 0 || r.rttMin > r.rtt {
r.rttMin = r.rtt
}
}
func (r *rttData) metrics() (rtt, rttMin, rttMax time.Duration) {
r.lock.RLock()
defer r.lock.RUnlock()
return r.rtt, r.rttMin, r.rttMax
}
func newFlowControlData() *flowControlData {
return &flowControlData{queue: queue.New()}
}
func (f *flowControlData) update(measurement uint32) {
f.lock.Lock()
defer f.lock.Unlock()
var firstItem uint32
// store new data into queue, remove oldest data if queue is full
f.queue.Enqueue(measurement)
if f.queue.Len() > dataPoints {
// data type should always be uint32
firstItem = f.queue.Dequeue().(uint32)
}
// if (measurement - firstItem) < 0, uint64(measurement - firstItem)
// will overflow and become a large positive number
f.sum += uint64(measurement)
f.sum -= uint64(firstItem)
if measurement > f.max {
f.max = measurement
}
if f.min == 0 || measurement < f.min {
f.min = measurement
}
}
// caller of ave() should acquire lock first
func (f *flowControlData) ave() float64 {
if f.queue.Len() == 0 {
return 0
}
return float64(f.sum) / float64(f.queue.Len())
}
func (f *flowControlData) metrics() (ave float64, min, max uint32) {
f.lock.RLock()
defer f.lock.RUnlock()
return f.ave(), f.min, f.max
}
func newRate() *rate {
return &rate{}
}
func (r *rate) update(measurement uint64) {
r.lock.Lock()
defer r.lock.Unlock()
r.curr = measurement
// if measurement is 0, then there is no incoming/outgoing connection, don't update min/max
if r.curr == 0 {
return
}
if measurement > r.max {
r.max = measurement
}
if r.min == 0 || measurement < r.min {
r.min = measurement
}
}
func (r *rate) get() (curr, min, max uint64) {
r.lock.RLock()
defer r.lock.RUnlock()
return r.curr, r.min, r.max
}

169
h2mux/muxmetrics_test.go
View File

@ -1,169 +0,0 @@
package h2mux
import (
"sync"
"testing"
"time"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
)
func ave(sum uint64, len int) float64 {
return float64(sum) / float64(len)
}
func TestRTTUpdate(t *testing.T) {
r := newRTTData()
start := time.Now()
// send at 0 ms, receive at 2 ms, RTT = 2ms
m := &roundTripMeasurement{receiveTime: start.Add(2 * time.Millisecond), sendTime: start}
r.update(m)
assert.Equal(t, start, r.lastMeasurementTime)
assert.Equal(t, 2*time.Millisecond, r.rtt)
assert.Equal(t, 2*time.Millisecond, r.rttMin)
assert.Equal(t, 2*time.Millisecond, r.rttMax)
// send at 3 ms, receive at 6 ms, RTT = 3ms
m = &roundTripMeasurement{receiveTime: start.Add(6 * time.Millisecond), sendTime: start.Add(3 * time.Millisecond)}
r.update(m)
assert.Equal(t, start.Add(3*time.Millisecond), r.lastMeasurementTime)
assert.Equal(t, 3*time.Millisecond, r.rtt)
assert.Equal(t, 2*time.Millisecond, r.rttMin)
assert.Equal(t, 3*time.Millisecond, r.rttMax)
// send at 7 ms, receive at 8 ms, RTT = 1ms
m = &roundTripMeasurement{receiveTime: start.Add(8 * time.Millisecond), sendTime: start.Add(7 * time.Millisecond)}
r.update(m)
assert.Equal(t, start.Add(7*time.Millisecond), r.lastMeasurementTime)
assert.Equal(t, 1*time.Millisecond, r.rtt)
assert.Equal(t, 1*time.Millisecond, r.rttMin)
assert.Equal(t, 3*time.Millisecond, r.rttMax)
// send at -4 ms, receive at 0 ms, RTT = 4ms, but this ping is before last measurement
// so it will be discarded
m = &roundTripMeasurement{receiveTime: start, sendTime: start.Add(-2 * time.Millisecond)}
r.update(m)
assert.Equal(t, start.Add(7*time.Millisecond), r.lastMeasurementTime)
assert.Equal(t, 1*time.Millisecond, r.rtt)
assert.Equal(t, 1*time.Millisecond, r.rttMin)
assert.Equal(t, 3*time.Millisecond, r.rttMax)
}
func TestFlowControlDataUpdate(t *testing.T) {
f := newFlowControlData()
assert.Equal(t, 0, f.queue.Len())
assert.Equal(t, float64(0), f.ave())
var sum uint64
min := maxWindowSize - dataPoints
max := maxWindowSize
for i := 1; i <= dataPoints; i++ {
size := maxWindowSize - uint32(i)
f.update(size)
assert.Equal(t, max-uint32(1), f.max)
assert.Equal(t, size, f.min)
assert.Equal(t, i, f.queue.Len())
sum += uint64(size)
assert.Equal(t, sum, f.sum)
assert.Equal(t, ave(sum, f.queue.Len()), f.ave())
}
// queue is full, should start to dequeue first element
for i := 1; i <= dataPoints; i++ {
f.update(max)
assert.Equal(t, max, f.max)
assert.Equal(t, min, f.min)
assert.Equal(t, dataPoints, f.queue.Len())
sum += uint64(i)
assert.Equal(t, sum, f.sum)
assert.Equal(t, ave(sum, dataPoints), f.ave())
}
}
func TestMuxMetricsUpdater(t *testing.T) {
t.Skip("Inherently racy test due to muxMetricsUpdaterImpl.run()")
errChan := make(chan error)
abortChan := make(chan struct{})
compBefore, compAfter := NewAtomicCounter(0), NewAtomicCounter(0)
m := newMuxMetricsUpdater(abortChan, compBefore, compAfter)
log := zerolog.Nop()
go func() {
errChan <- m.run(&log)
}()
var wg sync.WaitGroup
wg.Add(2)
// mock muxReader
readerStart := time.Now()
rm := &roundTripMeasurement{receiveTime: readerStart, sendTime: readerStart}
m.updateRTT(rm)
go func() {
defer wg.Done()
assert.Equal(t, 0, dataPoints%4,
"dataPoints is not divisible by 4; this test should be adjusted accordingly")
readerSend := readerStart.Add(time.Millisecond)
for i := 1; i <= dataPoints/4; i++ {
readerReceive := readerSend.Add(time.Duration(i) * time.Millisecond)
rm := &roundTripMeasurement{receiveTime: readerReceive, sendTime: readerSend}
m.updateRTT(rm)
readerSend = readerReceive.Add(time.Millisecond)
m.updateReceiveWindow(uint32(i))
m.updateSendWindow(uint32(i))
m.updateInBoundBytes(uint64(i))
}
}()
// mock muxWriter
go func() {
defer wg.Done()
assert.Equal(t, 0, dataPoints%4,
"dataPoints is not divisible by 4; this test should be adjusted accordingly")
for j := dataPoints/4 + 1; j <= dataPoints/2; j++ {
m.updateReceiveWindow(uint32(j))
m.updateSendWindow(uint32(j))
// should always be discarded since the send time is before readerSend
rm := &roundTripMeasurement{receiveTime: readerStart, sendTime: readerStart.Add(-time.Duration(j*dataPoints) * time.Millisecond)}
m.updateRTT(rm)
m.updateOutBoundBytes(uint64(j))
}
}()
wg.Wait()
metrics := m.metrics()
points := dataPoints / 2
assert.Equal(t, time.Millisecond, metrics.RTTMin)
assert.Equal(t, time.Duration(dataPoints/4)*time.Millisecond, metrics.RTTMax)
// sum(1..i) = i*(i+1)/2, ave(1..i) = i*(i+1)/2/i = (i+1)/2
assert.Equal(t, float64(points+1)/float64(2), metrics.ReceiveWindowAve)
assert.Equal(t, uint32(1), metrics.ReceiveWindowMin)
assert.Equal(t, uint32(points), metrics.ReceiveWindowMax)
assert.Equal(t, float64(points+1)/float64(2), metrics.SendWindowAve)
assert.Equal(t, uint32(1), metrics.SendWindowMin)
assert.Equal(t, uint32(points), metrics.SendWindowMax)
assert.Equal(t, uint64(dataPoints/4), metrics.InBoundRateCurr)
assert.Equal(t, uint64(1), metrics.InBoundRateMin)
assert.Equal(t, uint64(dataPoints/4), metrics.InBoundRateMax)
assert.Equal(t, uint64(dataPoints/2), metrics.OutBoundRateCurr)
assert.Equal(t, uint64(dataPoints/4+1), metrics.OutBoundRateMin)
assert.Equal(t, uint64(dataPoints/2), metrics.OutBoundRateMax)
close(abortChan)
assert.Nil(t, <-errChan)
close(errChan)
}

508
h2mux/muxreader.go
View File

@ -1,508 +0,0 @@
package h2mux
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"net/url"
"time"
"github.com/rs/zerolog"
"golang.org/x/net/http2"
)
type MuxReader struct {
// f is used to read HTTP2 frames.
f *http2.Framer
// handler provides a callback to receive new streams. if nil, new streams cannot be accepted.
handler MuxedStreamHandler
// streams tracks currently-open streams.
streams *activeStreamMap
// readyList is used to signal writable streams.
readyList *ReadyList
// streamErrors lets us report stream errors to the MuxWriter.
streamErrors *StreamErrorMap
// goAwayChan is used to tell the writer to send a GOAWAY message.
goAwayChan chan<- http2.ErrCode
// abortChan is used when shutting down ungracefully. When this becomes readable, all activity should stop.
abortChan <-chan struct{}
// pingTimestamp is an atomic value containing the latest received ping timestamp.
pingTimestamp *PingTimestamp
// connActive is used to signal to the writer that something happened on the connection.
// This is used to clear idle timeout disconnection deadlines.
connActive Signal
// The initial value for the send and receive window of a new stream.
initialStreamWindow uint32
// The max value for the send window of a stream.
streamWindowMax uint32
// The max size for the write buffer of a stream
streamWriteBufferMaxLen int
// r is a reference to the underlying connection used when shutting down.
r io.Closer
// metricsUpdater is used to report metrics
metricsUpdater muxMetricsUpdater
// bytesRead is the amount of bytes read from data frames since the last time we called metricsUpdater.updateInBoundBytes()
bytesRead *AtomicCounter
// dictionaries holds the h2 cross-stream compression dictionaries
dictionaries h2Dictionaries
}
// Shutdown blocks new streams from being created.
// It returns a channel that is closed once the last stream has closed.
func (r *MuxReader) Shutdown() <-chan struct{} {
done, alreadyInProgress := r.streams.Shutdown()
if alreadyInProgress {
return done
}
r.sendGoAway(http2.ErrCodeNo)
go func() {
// close reader side when last stream ends; this will cause the writer to abort
<-done
r.r.Close()
}()
return done
}
func (r *MuxReader) run(log *zerolog.Logger) error {
defer log.Debug().Msg("mux - read: event loop finished")
// routine to periodically update bytesRead
go func() {
ticker := time.NewTicker(updateFreq)
defer ticker.Stop()
for {
select {
case <-r.abortChan:
return
case <-ticker.C:
r.metricsUpdater.updateInBoundBytes(r.bytesRead.Count())
}
}
}()
for {
frame, err := r.f.ReadFrame()
if err != nil {
errorString := fmt.Sprintf("mux - read: %s", err)
if errorDetail := r.f.ErrorDetail(); errorDetail != nil {
errorString = fmt.Sprintf("%s: errorDetail: %s", errorString, errorDetail)
}
switch e := err.(type) {
case http2.StreamError:
log.Info().Msgf("%s: stream error", errorString)
// Ideally we wouldn't return here, since that aborts the muxer.
// We should communicate the error to the relevant MuxedStream
// data structure, so that callers of MuxedStream.Read() and
// MuxedStream.Write() would see it. Then we could `continue`
// and keep the muxer going.
return r.streamError(e.StreamID, e.Code)
case http2.ConnectionError:
log.Info().Msgf("%s: stream error", errorString)
return r.connectionError(err)
default:
if isConnectionClosedError(err) {
if r.streams.Len() == 0 {
// don't log the error here -- that would just be extra noise
log.Debug().Msg("mux - read: shutting down")
return nil
}
log.Info().Msgf("%s: connection closed unexpectedly", errorString)
return err
} else {
log.Info().Msgf("%s: frame read error", errorString)
return r.connectionError(err)
}
}
}
r.connActive.Signal()
log.Debug().Msgf("mux - read: read frame: data %v", frame)
switch f := frame.(type) {
case *http2.DataFrame:
err = r.receiveFrameData(f, log)
case *http2.MetaHeadersFrame:
err = r.receiveHeaderData(f)
case *http2.RSTStreamFrame:
streamID := f.Header().StreamID
if streamID == 0 {
return ErrInvalidStream
}
if stream, ok := r.streams.Get(streamID); ok {
stream.Close()
}
r.streams.Delete(streamID)
case *http2.PingFrame:
r.receivePingData(f)
case *http2.GoAwayFrame:
err = r.receiveGoAway(f)
// The receiver of a flow-controlled frame sends a WINDOW_UPDATE frame as it
// consumes data and frees up space in flow-control windows
case *http2.WindowUpdateFrame:
err = r.updateStreamWindow(f)
case *http2.UnknownFrame:
switch f.Header().Type {
case FrameUseDictionary:
err = r.receiveUseDictionary(f)
case FrameSetDictionary:
err = r.receiveSetDictionary(f)
default:
err = ErrUnexpectedFrameType
}
default:
err = ErrUnexpectedFrameType
}
if err != nil {
log.Debug().Msgf("mux - read: read error: data %v", frame)
return r.connectionError(err)
}
}
}
func (r *MuxReader) newMuxedStream(streamID uint32) *MuxedStream {
return &MuxedStream{
streamID: streamID,
readBuffer: NewSharedBuffer(),
writeBuffer: &bytes.Buffer{},
writeBufferMaxLen: r.streamWriteBufferMaxLen,
writeBufferHasSpace: make(chan struct{}, 1),
receiveWindow: r.initialStreamWindow,
receiveWindowCurrentMax: r.initialStreamWindow,
receiveWindowMax: r.streamWindowMax,
sendWindow: r.initialStreamWindow,
readyList: r.readyList,
dictionaries: r.dictionaries,
}
}
// getStreamForFrame returns a stream if valid, or an error describing why the stream could not be returned.
func (r *MuxReader) getStreamForFrame(frame http2.Frame) (*MuxedStream, error) {
sid := frame.Header().StreamID
if sid == 0 {
return nil, ErrUnexpectedFrameType
}
if stream, ok := r.streams.Get(sid); ok {
return stream, nil
}
if r.streams.IsLocalStreamID(sid) {
// no stream available, but no error
return nil, ErrClosedStream
}
if sid < r.streams.LastPeerStreamID() {
// no stream available, stream closed error
return nil, ErrClosedStream
}
return nil, ErrUnknownStream
}
func (r *MuxReader) defaultStreamErrorHandler(err error, header http2.FrameHeader) error {
if header.Flags.Has(http2.FlagHeadersEndStream) {
return nil
} else if err == ErrUnknownStream || err == ErrClosedStream {
return r.streamError(header.StreamID, http2.ErrCodeStreamClosed)
} else {
return err
}
}
// Receives header frames from a stream. A non-nil error is a connection error.
func (r *MuxReader) receiveHeaderData(frame *http2.MetaHeadersFrame) error {
var stream *MuxedStream
sid := frame.Header().StreamID
if sid == 0 {
return ErrUnexpectedFrameType
}
newStream := r.streams.IsPeerStreamID(sid)
if newStream {
// header request
// TODO support trailers (if stream exists)
ok, err := r.streams.AcquirePeerID(sid)
if !ok {
// ignore new streams while shutting down
return r.streamError(sid, err)
}
stream = r.newMuxedStream(sid)
// Set stream. Returns false if a stream already existed with that ID or we are shutting down, return false.
if !r.streams.Set(stream) {
// got HEADERS frame for an existing stream
// TODO support trailers
return r.streamError(sid, http2.ErrCodeInternal)
}
} else {
// header response
var err error
if stream, err = r.getStreamForFrame(frame); err != nil {
return r.defaultStreamErrorHandler(err, frame.Header())
}
}
headers := make([]Header, 0, len(frame.Fields))
for _, header := range frame.Fields {
switch header.Name {
case ":method":
stream.method = header.Value
case ":path":
u, err := url.Parse(header.Value)
if err == nil {
stream.path = u.Path
}
case "accept-encoding":
// remove accept-encoding if dictionaries are enabled
if r.dictionaries.write != nil {
continue
}
}
headers = append(headers, Header{Name: header.Name, Value: header.Value})
}
stream.Headers = headers
if frame.Header().Flags.Has(http2.FlagHeadersEndStream) {
stream.receiveEOF()
return nil
}
if newStream {
go r.handleStream(stream)
} else {
close(stream.responseHeadersReceived)
}
return nil
}
func (r *MuxReader) handleStream(stream *MuxedStream) {
defer stream.Close()
r.handler.ServeStream(stream)
}
// Receives a data frame from a stream. A non-nil error is a connection error.
func (r *MuxReader) receiveFrameData(frame *http2.DataFrame, log *zerolog.Logger) error {
stream, err := r.getStreamForFrame(frame)
if err != nil {
return r.defaultStreamErrorHandler(err, frame.Header())
}
data := frame.Data()
if len(data) > 0 {
n, err := stream.readBuffer.Write(data)
if err != nil {
return r.streamError(stream.streamID, http2.ErrCodeInternal)
}
r.bytesRead.IncrementBy(uint64(n))
}
if frame.Header().Flags.Has(http2.FlagDataEndStream) {
if stream.receiveEOF() {
r.streams.Delete(stream.streamID)
log.Debug().Msgf("mux - read: stream closed: streamID: %d", frame.Header().StreamID)
} else {
log.Debug().Msgf("mux - read: shutdown receive side: streamID: %d", frame.Header().StreamID)
}
return nil
}
if !stream.consumeReceiveWindow(uint32(len(data))) {
return r.streamError(stream.streamID, http2.ErrCodeFlowControl)
}
r.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow())
return nil
}
// Receive a PING from the peer. Update RTT and send/receive window metrics if it's an ACK.
func (r *MuxReader) receivePingData(frame *http2.PingFrame) {
ts := int64(binary.LittleEndian.Uint64(frame.Data[:]))
if !frame.IsAck() {
r.pingTimestamp.Set(ts)
return
}
// Update the computed RTT aggregations with a new measurement.
// `ts` is the time that the probe was sent.
// We assume that `time.Now()` is the time we received that probe.
r.metricsUpdater.updateRTT(&roundTripMeasurement{
receiveTime: time.Now(),
sendTime: time.Unix(0, ts),
})
}
// Receive a GOAWAY from the peer. Gracefully shut down our connection.
func (r *MuxReader) receiveGoAway(frame *http2.GoAwayFrame) error {
r.Shutdown()
// Close all streams above the last processed stream
lastStream := r.streams.LastLocalStreamID()
for i := frame.LastStreamID + 2; i <= lastStream; i++ {
if stream, ok := r.streams.Get(i); ok {
stream.Close()
}
}
return nil
}
// Receive a USE_DICTIONARY from the peer. Setup dictionary for stream.
func (r *MuxReader) receiveUseDictionary(frame *http2.UnknownFrame) error {
payload := frame.Payload()
streamID := frame.StreamID
// Check frame is formatted properly
if len(payload) != 1 {
return r.streamError(streamID, http2.ErrCodeProtocol)
}
stream, err := r.getStreamForFrame(frame)
if err != nil {
return err
}
if stream.receivedUseDict == true || stream.dictionaries.read == nil {
return r.streamError(streamID, http2.ErrCodeInternal)
}
stream.receivedUseDict = true
dictID := payload[0]
dictReader := stream.dictionaries.read.newReader(stream.readBuffer.(*SharedBuffer), dictID)
if dictReader == nil {
return r.streamError(streamID, http2.ErrCodeInternal)
}
stream.readBufferLock.Lock()
stream.readBuffer = dictReader
stream.readBufferLock.Unlock()
return nil
}
// Receive a SET_DICTIONARY from the peer. Update dictionaries accordingly.
func (r *MuxReader) receiveSetDictionary(frame *http2.UnknownFrame) (err error) {
payload := frame.Payload()
flags := frame.Flags
stream, err := r.getStreamForFrame(frame)
if err != nil && err != ErrClosedStream {
return err
}
reader, ok := stream.readBuffer.(*h2DictionaryReader)
if !ok {
return r.streamError(frame.StreamID, http2.ErrCodeProtocol)
}
// A SetDictionary frame consists of several
// Dictionary-Entries that specify how existing dictionaries
// are to be updated using the current stream data
// +---------------+---------------+
// | Dictionary-Entry (+) ...
// +---------------+---------------+
for {
// Each Dictionary-Entry is formatted as follows:
// +-------------------------------+
// | Dictionary-ID (8) |
// +---+---------------------------+
// | P | Size (7+) |
// +---+---------------------------+
// | E?| D?| Truncate? (6+) |
// +---+---------------------------+
// | Offset? (8+) |
// +-------------------------------+
var size, truncate, offset uint64
var p, e, d bool
// Parse a single Dictionary-Entry
if len(payload) < 2 { // Must have at least id and size
return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
dictID := uint8(payload[0])
p = (uint8(payload[1]) >> 7) == 1
payload, size, err = http2ReadVarInt(7, payload[1:])
if err != nil {
return
}
if flags.Has(FlagSetDictionaryAppend) {
// Presence of FlagSetDictionaryAppend means we expect e, d and truncate
if len(payload) < 1 {
return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
e = (uint8(payload[0]) >> 7) == 1
d = (uint8((payload[0])>>6) & 1) == 1
payload, truncate, err = http2ReadVarInt(6, payload)
if err != nil {
return
}
}
if flags.Has(FlagSetDictionaryOffset) {
// Presence of FlagSetDictionaryOffset means we expect offset
if len(payload) < 1 {
return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
payload, offset, err = http2ReadVarInt(8, payload)
if err != nil {
return
}
}
setdict := setDictRequest{streamID: stream.streamID,
dictID: dictID,
dictSZ: size,
truncate: truncate,
offset: offset,
P: p,
E: e,
D: d}
// Find the right dictionary
dict, err := r.dictionaries.read.getDictByID(dictID)
if err != nil {
return err
}
// Register a dictionary update order for the dictionary and reader
updateEntry := &dictUpdate{reader: reader, dictionary: dict, s: setdict}
dict.queue = append(dict.queue, updateEntry)
reader.queue = append(reader.queue, updateEntry)
// End of frame
if len(payload) == 0 {
break
}
}
return nil
}
// Receives header frames from a stream. A non-nil error is a connection error.
func (r *MuxReader) updateStreamWindow(frame *http2.WindowUpdateFrame) error {
stream, err := r.getStreamForFrame(frame)
if err != nil && err != ErrUnknownStream && err != ErrClosedStream {
return err
}
if stream == nil {
// ignore window updates on closed streams
return nil
}
stream.replenishSendWindow(frame.Increment)
r.metricsUpdater.updateSendWindow(stream.getSendWindow())
return nil
}
// Raise a stream processing error, closing the stream. Runs on the write thread.
func (r *MuxReader) streamError(streamID uint32, e http2.ErrCode) error {
r.streamErrors.RaiseError(streamID, e)
return nil
}
func (r *MuxReader) connectionError(err error) error {
http2Code := http2.ErrCodeInternal
switch e := err.(type) {
case http2.ConnectionError:
http2Code = http2.ErrCode(e)
case MuxerProtocolError:
http2Code = e.h2code
}
r.sendGoAway(http2Code)
return err
}
// Instruct the writer to send a GOAWAY message if possible. This may fail in
// the case where an existing GOAWAY message is in flight or the writer event
// loop already ended.
func (r *MuxReader) sendGoAway(errCode http2.ErrCode) {
select {
case r.goAwayChan <- errCode:
default:
}
}

88
h2mux/muxreader_test.go
View File

@ -1,88 +0,0 @@
package h2mux
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var (
methodHeader = Header{
Name: ":method",
Value: "GET",
}
schemeHeader = Header{
Name: ":scheme",
Value: "https",
}
pathHeader = Header{
Name: ":path",
Value: "/api/tunnels",
}
respStatusHeader = Header{
Name: ":status",
Value: "200",
}
)
type mockOriginStreamHandler struct {
stream *MuxedStream
}
func (mosh *mockOriginStreamHandler) ServeStream(stream *MuxedStream) error {
mosh.stream = stream
// Echo tunnel hostname in header
stream.WriteHeaders([]Header{respStatusHeader})
return nil
}
func assertOpenStreamSucceed(t *testing.T, stream *MuxedStream, err error) {
assert.NoError(t, err)
assert.Len(t, stream.Headers, 1)
assert.Equal(t, respStatusHeader, stream.Headers[0])
}
func TestMissingHeaders(t *testing.T) {
originHandler := &mockOriginStreamHandler{}
muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream)
muxPair.Serve(t)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
reqHeaders := []Header{
{
Name: "content-type",
Value: "application/json",
},
}
stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
assertOpenStreamSucceed(t, stream, err)
assert.Empty(t, originHandler.stream.method)
assert.Empty(t, originHandler.stream.path)
}
func TestReceiveHeaderData(t *testing.T) {
originHandler := &mockOriginStreamHandler{}
muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream)
muxPair.Serve(t)
reqHeaders := []Header{
methodHeader,
schemeHeader,
pathHeader,
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
assertOpenStreamSucceed(t, stream, err)
assert.Equal(t, methodHeader.Value, originHandler.stream.method)
assert.Equal(t, pathHeader.Value, originHandler.stream.path)
}

311
h2mux/muxwriter.go
View File

@ -1,311 +0,0 @@
package h2mux
import (
"bytes"
"encoding/binary"
"io"
"time"
"github.com/rs/zerolog"
"golang.org/x/net/http2"
"golang.org/x/net/http2/hpack"
)
type MuxWriter struct {
// f is used to write HTTP2 frames.
f *http2.Framer
// streams tracks currently-open streams.
streams *activeStreamMap
// streamErrors receives stream errors raised by the MuxReader.
streamErrors *StreamErrorMap
// readyStreamChan is used to multiplex writable streams onto the single connection.
// When a stream becomes writable its ID is sent on this channel.
readyStreamChan <-chan uint32
// newStreamChan is used to create new streams with a given set of headers.
newStreamChan <-chan MuxedStreamRequest
// goAwayChan is used to send a single GOAWAY message to the peer. The element received
// is the HTTP/2 error code to send.
goAwayChan <-chan http2.ErrCode
// abortChan is used when shutting down ungracefully. When this becomes readable, all activity should stop.
abortChan <-chan struct{}
// pingTimestamp is an atomic value containing the latest received ping timestamp.
pingTimestamp *PingTimestamp
// A timer used to measure idle connection time. Reset after sending data.
idleTimer *IdleTimer
// connActiveChan receives a signal that the connection received some (read) activity.
connActiveChan <-chan struct{}
// Maximum size of all frames that can be sent on this connection.
maxFrameSize uint32
// headerEncoder is the stateful header encoder for this connection
headerEncoder *hpack.Encoder
// headerBuffer is the temporary buffer used by headerEncoder.
headerBuffer bytes.Buffer
// metricsUpdater is used to report metrics
metricsUpdater muxMetricsUpdater
// bytesWrote is the amount of bytes written to data frames since the last time we called metricsUpdater.updateOutBoundBytes()
bytesWrote *AtomicCounter
useDictChan <-chan useDictRequest
}
type MuxedStreamRequest struct {
stream *MuxedStream
body io.Reader
}
func NewMuxedStreamRequest(stream *MuxedStream, body io.Reader) MuxedStreamRequest {
return MuxedStreamRequest{
stream: stream,
body: body,
}
}
func (r *MuxedStreamRequest) flushBody() {
io.Copy(r.stream, r.body)
r.stream.CloseWrite()
}
func tsToPingData(ts int64) [8]byte {
pingData := [8]byte{}
binary.LittleEndian.PutUint64(pingData[:], uint64(ts))
return pingData
}
func (w *MuxWriter) run(log *zerolog.Logger) error {
defer log.Debug().Msg("mux - write: event loop finished")
// routine to periodically communicate bytesWrote
go func() {
ticker := time.NewTicker(updateFreq)
defer ticker.Stop()
for {
select {
case <-w.abortChan:
return
case <-ticker.C:
w.metricsUpdater.updateOutBoundBytes(w.bytesWrote.Count())
}
}
}()
for {
select {
case <-w.abortChan:
log.Debug().Msg("mux - write: aborting writer thread")
return nil
case errCode := <-w.goAwayChan:
log.Debug().Msgf("mux - write: sending GOAWAY code %v", errCode)
err := w.f.WriteGoAway(w.streams.LastPeerStreamID(), errCode, []byte{})
if err != nil {
return err
}
w.idleTimer.MarkActive()
case <-w.pingTimestamp.GetUpdateChan():
log.Debug().Msg("mux - write: sending PING ACK")
err := w.f.WritePing(true, tsToPingData(w.pingTimestamp.Get()))
if err != nil {
return err
}
w.idleTimer.MarkActive()
case <-w.idleTimer.C:
if !w.idleTimer.Retry() {
return ErrConnectionDropped
}
log.Debug().Msg("mux - write: sending PING")
err := w.f.WritePing(false, tsToPingData(time.Now().UnixNano()))
if err != nil {
return err
}
w.idleTimer.ResetTimer()
case <-w.connActiveChan:
w.idleTimer.MarkActive()
case <-w.streamErrors.GetSignalChan():
for streamID, errCode := range w.streamErrors.GetErrors() {
log.Debug().Msgf("mux - write: resetting stream with code: %v streamID: %d", errCode, streamID)
err := w.f.WriteRSTStream(streamID, errCode)
if err != nil {
return err
}
}
w.idleTimer.MarkActive()
case streamRequest := <-w.newStreamChan:
streamID := w.streams.AcquireLocalID()
streamRequest.stream.streamID = streamID
if !w.streams.Set(streamRequest.stream) {
// Race between OpenStream and Shutdown, and Shutdown won. Let Shutdown (and the eventual abort) take
// care of this stream. Ideally we'd pass the error directly to the stream object somehow so the
// caller can be unblocked sooner, but the value of that optimisation is minimal for most of the
// reasons why you'd call Shutdown anyway.
continue
}
if streamRequest.body != nil {
go streamRequest.flushBody()
}
err := w.writeStreamData(streamRequest.stream, log)
if err != nil {
return err
}
w.idleTimer.MarkActive()
case streamID := <-w.readyStreamChan:
stream, ok := w.streams.Get(streamID)
if !ok {
continue
}
err := w.writeStreamData(stream, log)
if err != nil {
return err
}
w.idleTimer.MarkActive()
case useDict := <-w.useDictChan:
err := w.writeUseDictionary(useDict)
if err != nil {
log.Error().Msgf("mux - write: error writing use dictionary: %s", err)
return err
}
w.idleTimer.MarkActive()
}
}
}
func (w *MuxWriter) writeStreamData(stream *MuxedStream, log *zerolog.Logger) error {
log.Debug().Msgf("mux - write: writable: streamID: %d", stream.streamID)
chunk := stream.getChunk()
w.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow())
w.metricsUpdater.updateSendWindow(stream.getSendWindow())
if chunk.sendHeadersFrame() {
err := w.writeHeaders(chunk.streamID, chunk.headers)
if err != nil {
log.Error().Msgf("mux - write: error writing headers: %s: streamID: %d", err, stream.streamID)
return err
}
log.Debug().Msgf("mux - write: output headers: streamID: %d", stream.streamID)
}
if chunk.sendWindowUpdateFrame() {
// Send a WINDOW_UPDATE frame to update our receive window.
// If the Stream ID is zero, the window update applies to the connection as a whole
// RFC7540 section-6.9.1 "A receiver that receives a flow-controlled frame MUST
// always account for its contribution against the connection flow-control
// window, unless the receiver treats this as a connection error"
err := w.f.WriteWindowUpdate(chunk.streamID, chunk.windowUpdate)
if err != nil {
log.Error().Msgf("mux - write: error writing window update: %s: streamID: %d", err, stream.streamID)
return err
}
log.Debug().Msgf("mux - write: increment receive window by %d streamID: %d", chunk.windowUpdate, stream.streamID)
}
for chunk.sendDataFrame() {
payload, sentEOF := chunk.nextDataFrame(int(w.maxFrameSize))
err := w.f.WriteData(chunk.streamID, sentEOF, payload)
if err != nil {
log.Error().Msgf("mux - write: error writing data: %s: streamID: %d", err, stream.streamID)
return err
}
// update the amount of data wrote
w.bytesWrote.IncrementBy(uint64(len(payload)))
log.Debug().Msgf("mux - write: output data: %d: streamID: %d", len(payload), stream.streamID)
if sentEOF {
if stream.readBuffer.Closed() {
// transition into closed state
if !stream.gotReceiveEOF() {
// the peer may send data that we no longer want to receive. Force them into the
// closed state.
log.Debug().Msgf("mux - write: resetting stream: streamID: %d", stream.streamID)
w.f.WriteRSTStream(chunk.streamID, http2.ErrCodeNo)
} else {
// Half-open stream transitioned into closed
log.Debug().Msgf("mux - write: closing stream: streamID: %d", stream.streamID)
}
w.streams.Delete(chunk.streamID)
} else {
log.Debug().Msgf("mux - write: closing stream write side: streamID: %d", stream.streamID)
}
}
}
return nil
}
func (w *MuxWriter) encodeHeaders(headers []Header) ([]byte, error) {
w.headerBuffer.Reset()
for _, header := range headers {
err := w.headerEncoder.WriteField(hpack.HeaderField{
Name: header.Name,
Value: header.Value,
})
if err != nil {
return nil, err
}
}
return w.headerBuffer.Bytes(), nil
}
// writeHeaders writes a block of encoded headers, splitting it into multiple frames if necessary.
func (w *MuxWriter) writeHeaders(streamID uint32, headers []Header) error {
encodedHeaders, err := w.encodeHeaders(headers)
if err != nil || len(encodedHeaders) == 0 {
return err
}
blockSize := int(w.maxFrameSize)
// CONTINUATION is unnecessary; the headers fit within the blockSize
if len(encodedHeaders) < blockSize {
return w.f.WriteHeaders(http2.HeadersFrameParam{
StreamID: streamID,
EndHeaders: true,
BlockFragment: encodedHeaders,
})
}
choppedHeaders := chopEncodedHeaders(encodedHeaders, blockSize)
// len(choppedHeaders) is at least 2
if err := w.f.WriteHeaders(http2.HeadersFrameParam{StreamID: streamID, EndHeaders: false, BlockFragment: choppedHeaders[0]}); err != nil {
return err
}
for i := 1; i < len(choppedHeaders)-1; i++ {
if err := w.f.WriteContinuation(streamID, false, choppedHeaders[i]); err != nil {
return err
}
}
if err := w.f.WriteContinuation(streamID, true, choppedHeaders[len(choppedHeaders)-1]); err != nil {
return err
}
return nil
}
// Partition a slice of bytes into `len(slice) / blockSize` slices of length `blockSize`
func chopEncodedHeaders(headers []byte, chunkSize int) [][]byte {
var divided [][]byte
for i := 0; i < len(headers); i += chunkSize {
end := i + chunkSize
if end > len(headers) {
end = len(headers)
}
divided = append(divided, headers[i:end])
}
return divided
}
func (w *MuxWriter) writeUseDictionary(dictRequest useDictRequest) error {
err := w.f.WriteRawFrame(FrameUseDictionary, 0, dictRequest.streamID, []byte{byte(dictRequest.dictID)})
if err != nil {
return err
}
payload := make([]byte, 0, 64)
for _, set := range dictRequest.setDict {
payload = append(payload, byte(set.dictID))
payload = appendVarInt(payload, 7, uint64(set.dictSZ))
payload = append(payload, 0x80) // E = 1, D = 0, Truncate = 0
}
err = w.f.WriteRawFrame(FrameSetDictionary, FlagSetDictionaryAppend, dictRequest.streamID, payload)
return err
}

26
h2mux/muxwriter_test.go
View File

@ -1,26 +0,0 @@
package h2mux
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestChopEncodedHeaders(t *testing.T) {
mockEncodedHeaders := make([]byte, 5)
for i := range mockEncodedHeaders {
mockEncodedHeaders[i] = byte(i)
}
chopped := chopEncodedHeaders(mockEncodedHeaders, 4)
assert.Equal(t, 2, len(chopped))
assert.Equal(t, []byte{0, 1, 2, 3}, chopped[0])
assert.Equal(t, []byte{4}, chopped[1])
}
func TestChopEncodedEmptyHeaders(t *testing.T) {
mockEncodedHeaders := make([]byte, 0)
chopped := chopEncodedHeaders(mockEncodedHeaders, 3)
assert.Equal(t, 0, len(chopped))
}

151
h2mux/readylist.go
View File

@ -1,151 +0,0 @@
package h2mux
import "sync"
// ReadyList multiplexes several event signals onto a single channel.
type ReadyList struct {
// signalC is used to signal that a stream can be enqueued
signalC chan uint32
// waitC is used to signal the ID of the first ready descriptor
waitC chan uint32
// doneC is used to signal that run should terminate
doneC chan struct{}
closeOnce sync.Once
}
func NewReadyList() *ReadyList {
rl := &ReadyList{
signalC: make(chan uint32),
waitC: make(chan uint32),
doneC: make(chan struct{}),
}
go rl.run()
return rl
}
// ID is the stream ID
func (r *ReadyList) Signal(ID uint32) {
select {
case r.signalC <- ID:
// ReadyList already closed
case <-r.doneC:
}
}
func (r *ReadyList) ReadyChannel() <-chan uint32 {
return r.waitC
}
func (r *ReadyList) Close() {
r.closeOnce.Do(func() {
close(r.doneC)
})
}
func (r *ReadyList) run() {
defer close(r.waitC)
var queue readyDescriptorQueue
var firstReady *readyDescriptor
activeDescriptors := newReadyDescriptorMap()
for {
if firstReady == nil {
select {
case i := <-r.signalC:
firstReady = activeDescriptors.SetIfMissing(i)
case <-r.doneC:
return
}
}
select {
case r.waitC <- firstReady.ID:
activeDescriptors.Delete(firstReady.ID)
firstReady = queue.Dequeue()
case i := <-r.signalC:
newReady := activeDescriptors.SetIfMissing(i)
if newReady != nil {
// key doesn't exist
queue.Enqueue(newReady)
}
case <-r.doneC:
return
}
}
}
type readyDescriptor struct {
ID uint32
Next *readyDescriptor
}
// readyDescriptorQueue is a queue of readyDescriptors in the form of a singly-linked list.
// The nil readyDescriptorQueue is an empty queue ready for use.
type readyDescriptorQueue struct {
Head *readyDescriptor
Tail *readyDescriptor
}
func (q *readyDescriptorQueue) Empty() bool {
return q.Head == nil
}
func (q *readyDescriptorQueue) Enqueue(x *readyDescriptor) {
if x.Next != nil {
panic("enqueued already queued item")
}
if q.Empty() {
q.Head = x
q.Tail = x
} else {
q.Tail.Next = x
q.Tail = x
}
}
// Dequeue returns the first readyDescriptor in the queue, or nil if empty.
func (q *readyDescriptorQueue) Dequeue() *readyDescriptor {
if q.Empty() {
return nil
}
x := q.Head
q.Head = x.Next
x.Next = nil
return x
}
// readyDescriptorQueue is a map of readyDescriptors keyed by ID.
// It maintains a free list of deleted ready descriptors.
type readyDescriptorMap struct {
descriptors map[uint32]*readyDescriptor
free []*readyDescriptor
}
func newReadyDescriptorMap() *readyDescriptorMap {
return &readyDescriptorMap{descriptors: make(map[uint32]*readyDescriptor)}
}
// create or reuse a readyDescriptor if the stream is not in the queue.
// This avoid stream starvation caused by a single high-bandwidth stream monopolising the writer goroutine
func (m *readyDescriptorMap) SetIfMissing(key uint32) *readyDescriptor {
if _, ok := m.descriptors[key]; ok {
return nil
}
var newDescriptor *readyDescriptor
if len(m.free) > 0 {
// reuse deleted ready descriptors
newDescriptor = m.free[len(m.free)-1]
m.free = m.free[:len(m.free)-1]
} else {
newDescriptor = &readyDescriptor{}
}
newDescriptor.ID = key
m.descriptors[key] = newDescriptor
return newDescriptor
}
func (m *readyDescriptorMap) Delete(key uint32) {
if descriptor, ok := m.descriptors[key]; ok {
m.free = append(m.free, descriptor)
delete(m.descriptors, key)
}
}

171
h2mux/readylist_test.go
View File

@ -1,171 +0,0 @@
package h2mux
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func assertEmpty(t *testing.T, rl *ReadyList) {
select {
case <-rl.ReadyChannel():
t.Fatal("Spurious wakeup")
default:
}
}
func assertClosed(t *testing.T, rl *ReadyList) {
select {
case _, ok := <-rl.ReadyChannel():
assert.False(t, ok, "ReadyChannel was not closed")
case <-time.After(100 * time.Millisecond):
t.Fatalf("Timeout")
}
}
func receiveWithTimeout(t *testing.T, rl *ReadyList) uint32 {
select {
case i := <-rl.ReadyChannel():
return i
case <-time.After(100 * time.Millisecond):
t.Fatalf("Timeout")
return 0
}
}
func TestReadyListEmpty(t *testing.T) {
rl := NewReadyList()
// no signals, receive should fail
assertEmpty(t, rl)
}
func TestReadyListSignal(t *testing.T) {
rl := NewReadyList()
assertEmpty(t, rl)
rl.Signal(0)
if receiveWithTimeout(t, rl) != 0 {
t.Fatalf("Received wrong ID of signalled event")
}
assertEmpty(t, rl)
}
func TestReadyListMultipleSignals(t *testing.T) {
rl := NewReadyList()
assertEmpty(t, rl)
// Signals should not block;
// Duplicate unhandled signals should not cause multiple wakeups
signalled := [5]bool{}
for i := range signalled {
rl.Signal(uint32(i))
rl.Signal(uint32(i))
}
// All signals should be received once (in any order)
for range signalled {
i := receiveWithTimeout(t, rl)
if signalled[i] {
t.Fatalf("Received signal %d more than once", i)
}
signalled[i] = true
}
for i := range signalled {
if !signalled[i] {
t.Fatalf("Never received signal %d", i)
}
}
assertEmpty(t, rl)
}
func TestReadyListClose(t *testing.T) {
rl := NewReadyList()
rl.Close()
// readyList.run() occurs in a separate goroutine,
// so there's no way to directly check that run() has terminated.
// Perform an indirect check: is the ready channel closed?
assertClosed(t, rl)
// a second rl.Close() shouldn't cause a panic
rl.Close()
// Signal shouldn't block after Close()
done := make(chan struct{})
go func() {
for i := 0; i < 5; i++ {
rl.Signal(uint32(i))
}
close(done)
}()
select {
case <-done:
case <-time.After(100 * time.Millisecond):
t.Fatal("Test timed out")
}
}
func TestReadyDescriptorQueue(t *testing.T) {
var queue readyDescriptorQueue
items := [4]readyDescriptor{}
for i := range items {
items[i].ID = uint32(i)
}
if !queue.Empty() {
t.Fatalf("nil queue should be empty")
}
queue.Enqueue(&items[3])
queue.Enqueue(&items[1])
queue.Enqueue(&items[0])
queue.Enqueue(&items[2])
if queue.Empty() {
t.Fatalf("Empty should be false after enqueue")
}
i := queue.Dequeue().ID
if i != 3 {
t.Fatalf("item 3 should have been dequeued, got %d instead", i)
}
i = queue.Dequeue().ID
if i != 1 {
t.Fatalf("item 1 should have been dequeued, got %d instead", i)
}
i = queue.Dequeue().ID
if i != 0 {
t.Fatalf("item 0 should have been dequeued, got %d instead", i)
}
i = queue.Dequeue().ID
if i != 2 {
t.Fatalf("item 2 should have been dequeued, got %d instead", i)
}
if !queue.Empty() {
t.Fatal("queue should be empty after dequeuing all items")
}
if queue.Dequeue() != nil {
t.Fatal("dequeue on empty queue should return nil")
}
}
func TestReadyDescriptorMap(t *testing.T) {
m := newReadyDescriptorMap()
m.Delete(42)
// (delete of missing key should be a noop)
x := m.SetIfMissing(42)
if x == nil {
t.Fatal("SetIfMissing for new key returned nil")
}
if m.SetIfMissing(42) != nil {
t.Fatal("SetIfMissing for existing key returned non-nil")
}
// this delete has effect
m.Delete(42)
// the next set should reuse the old object
y := m.SetIfMissing(666)
if y == nil {
t.Fatal("SetIfMissing for new key returned nil")
}
if x != y {
t.Fatal("SetIfMissing didn't reuse freed object")
}
}

29
h2mux/rtt.go
View File

@ -1,29 +0,0 @@
package h2mux
import (
"sync/atomic"
)
// PingTimestamp is an atomic interface around ping timestamping and signalling.
type PingTimestamp struct {
ts int64
signal Signal
}
func NewPingTimestamp() *PingTimestamp {
return &PingTimestamp{signal: NewSignal()}
}
func (pt *PingTimestamp) Set(v int64) {
if atomic.SwapInt64(&pt.ts, v) != 0 {
pt.signal.Signal()
}
}
func (pt *PingTimestamp) Get() int64 {
return atomic.SwapInt64(&pt.ts, 0)
}
func (pt *PingTimestamp) GetUpdateChan() <-chan struct{} {
return pt.signal.WaitChannel()
}

1
h2mux/sample/ghost-url.min.js vendored
View File

@ -1 +0,0 @@
!function(){"use strict";function a(a){var b,c=[];if(!a)return"";for(b in a)a.hasOwnProperty(b)&&(a[b]||a[b]===!1)&&c.push(b+"="+encodeURIComponent(a[b]));return c.length?"?"+c.join("&"):""}var b,c,d,e,f="https://cloudflare.ghost.io/ghost/api/v0.1/";d={api:function(){var d,e=Array.prototype.slice.call(arguments),g=f;return d=e.pop(),d&&"object"!=typeof d&&(e.push(d),d={}),d=d||{},d.client_id=b,d.client_secret=c,e.length&&e.forEach(function(a){g+=a.replace(/^\/|\/$/g,"")+"/"}),g+a(d)}},e=function(a){b=a.clientId?a.clientId:"",c=a.clientSecret?a.clientSecret:"",f=a.url?a.url:f.match(/{\{api-url}}/)?"":f},"undefined"!=typeof window&&(window.ghost=window.ghost||{},window.ghost.url=d,window.ghost.init=e),"undefined"!=typeof module&&(module.exports={url:d,init:e})}();

537
h2mux/sample/index.html
View File

@ -1,537 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<title>Cloudflare Blog</title>
<meta name="description" content="" />
<meta name="HandheldFriendly" content="True">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
<link rel="shortcut icon" href="/assets/images/favicon.ico?v=b6cf3f99a6">
<link rel="apple-touch-icon-precomposed" sizes="57x57" href="/assets/images/apple-touch-icon-57x57-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/assets/images/apple-touch-icon-72x72-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/assets/images/apple-touch-icon-114x114-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/assets/images/apple-touch-icon-144x144-precomposed.png?v=b6cf3f99a6" />
<link rel="stylesheet" type="text/css" href="/assets/css/screen.css?v=b6cf3f99a6" />
<!--[if lt IE 9]><link rel="stylesheet" type="text/css" href="/assets/css/ie.css?v=b6cf3f99a6" /><![endif]-->
<!--<link href="http://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,400,700,300,600" rel="stylesheet" type="text/css">-->
<script>(function(G,o,O,g,l){G.GoogleAnalyticsObject=O;G[O]||(G[O]=function(){(G[O].q=G[O].q||[]).push(arguments)});G[O].l=+new Date;g=o.createElement('script'),l=o.scripts[0];g.src='//www.google-analytics.com/analytics.js';l.parentNode.insertBefore(g,l)}(this,document,'ga'));ga('create','UA-10218544-12', 'auto');ga('send','pageview')</script>
<link rel="canonical" href="http://blog.cloudflare.com/" />
<meta name="referrer" content="no-referrer-when-downgrade" />
<link rel="next" href="https://blog.cloudflare.com/page/2/" />
<meta property="og:site_name" content="Cloudflare Blog" />
<meta property="og:type" content="website" />
<meta property="og:title" content="Cloudflare Blog" />
<meta property="og:url" content="http://blog.cloudflare.com/" />
<meta property="og:image" content="http://blog.cloudflare.com/content/images/2016/09/logo-for-blog_thumb-1.png" />
<meta property="article:publisher" content="https://www.facebook.com/Cloudflare" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="Cloudflare Blog" />
<meta name="twitter:url" content="http://blog.cloudflare.com/" />
<meta name="twitter:image" content="http://blog.cloudflare.com/content/images/2016/09/logo-for-blog_thumb-1.png" />
<meta name="twitter:site" content="@cloudflare" />
<meta property="og:image:width" content="189" />
<meta property="og:image:height" content="47" />
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Website",
"publisher": {
"@type": "Organization",
"name": "Cloudflare Blog",
"logo": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2016/09/logo-for-blog_thumb.png",
"width": 189,
"height": 47
}
},
"url": "https://blog.cloudflare.com/",
"image": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2016/09/logo-for-blog_thumb-1.png",
"width": 189,
"height": 47
},
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "http://blog.cloudflare.com"
}
}
</script>
<script type="text/javascript" src="/shared/ghost-url.min.js?v=b6cf3f99a6"></script>
<script type="text/javascript">
ghost.init({
clientId: "ghost-frontend",
clientSecret: "cf0df60d1ab4"
});
</script>
<meta name="generator" content="Ghost 0.11" />
<link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
<meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
<meta class="swiftype" name="language" data-type="string" content="en" />
<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
<script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
<script>
var trackRecruitingLink = function(role, url) {
ga('send', 'event', 'recruiting', 'jobscore-click', role, {
'transport': 'beacon',
'hitCallback': function(){document.location = url;}
});
}
</script>
<script type="text/javascript">
(function() {
var didInit = false;
function initMunchkin() {
if(didInit === false) {
didInit = true;
Munchkin.init('713-XSC-918');
}
}
var s = document.createElement('script');
s.type = 'text/javascript';
s.async = true;
s.src = '//munchkin.marketo.net/munchkin.js';
s.onreadystatechange = function() {
if (this.readyState == 'complete' || this.readyState == 'loaded') {
initMunchkin();
}
};
s.onload = initMunchkin;
document.getElementsByTagName('head')[0].appendChild(s);
})();
</script>
<script>
var HTMLAttrToAdd = document.querySelector("html");
HTMLAttrToAdd.setAttribute("lang", "en");
</script>
<style>
table {
background-color: transparent;
}
td {
padding: 5px 1em;
}
pre {
max-height: 500px;
overflow-y: scroll;
}
</style>
<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
<link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
<style>
.st-default-search-input {
font-family: Helvetica, Arial, "Lucida Grande", sans-serif;
font-size: 14px;
line-height: 16px;
font-weight: 400;
-moz-transition: opacity 0.2s;
-o-transition: opacity 0.2s;
-webkit-transition: opacity 0.2s;
transition: opacity 0.2s;
display: inline-block;
width: 190px;
height: 16px;
padding: 7px 11px 7px 28px;
border: 1px solid rgba(0, 0, 0, 0.25);
color: #444;
-moz-box-sizing: content-box;
box-sizing: content-box;
-moz-border-radius: 5px;
-webkit-border-radius: 5px;
border-radius: 5px;
background: #fff 8px 8px no-repeat url("data:image/png;base64, iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAYAAABy6%2BR8AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAIGNIUk0AAG11AABzoAAA%2FN0AAINkAABw6AAA7GgAADA%2BAAAQkOTsmeoAAAESSURBVHjajNCxS9VRGMbxz71E4OwgoXPQxVEpXCI47%2BZqGP0LCoJO7UVD3QZzb3SwcHB7F3Uw3Zpd%2FAPCcJKG7Dj4u%2FK7Pwp94HDg5Xyf5z1Pr9YKImKANTzFXxzjU2ae6qhXaxURr%2FAFl9hHDy%2FwEK8z89sYVEp5gh84wMvMvGiSJ%2FEV85jNzLMR1McqfmN5BEBmnmMJFSvtpH7jdJiZv7q7Z%2BZPfMdcF6rN%2FT%2F1m2LGBkd4HhFT3dcRMY2FpskxaLNpayciHrWAGeziD7b%2BVfkithuTk8bkGa4wgWFmbrSTZOYeBvjc%2BucQj%2FEe6xHx4Taq1nrnKaW8K6XUUsrHWuvNevdRRLzFGwzvDbXAB9cDAHvhedDruuxSAAAAAElFTkSuQmCC")
}
.st-ui-close-button {
-moz-transition: none;
-o-transition: none;
-webkit-transition: none;
transition: none
}
</style>
</head>
<body class="home-template">
<div id="fb-root"></div>
<header id="header" class="header">
<div class="wrapper">
<a href="https://www.cloudflare.com" class="logo logo-header">Cloudflare</a>
<nav id="main-menu" class="header-navigation navigation" role="navigation">
<ul class="menu menu-header">
<li><a href="https://blog.cloudflare.com/">Blog home</a></li>
<li><a href="https://www.cloudflare.com/overview" tabindex="1">What we do</a></li>
<li><a href="https://www.cloudflare.com/support" tabindex="9">Support</a></li>
<li><a href="https://www.cloudflare.com/community" tabindex="9">Community</a></li>
<li><a href="https://www.cloudflare.com/login" tabindex="10">Login</a></li>
<li><a href="https://www.cloudflare.com/sign-up" class="btn btn-success" tabindex="11">Sign up</a></li>
</ul>
</nav>
</div>
</header>
<div class="wrapper reverse-sidebar">
<section class="primary-content" role="main">
<article class="post tag-google-cloud tag-cloud-computing tag-internet-summit">
<header class="post-header">
<h2 class="title"><a href="/living-in-a-multi-cloud-world/">Living In A Multi-Cloud World</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 21st, 2017 4:30PM">November 21st, 2017 4:30PM</time>
by <a href="/author/sergi/">Sergi Isasi</a>.
</div>
</header>
<div class="post-excerpt">
<p>A few months ago at Cloudflare’s Internet Summit, we hosted a discussion on A Cloud Without Handcuffs with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS. The conversation touched on multiple areas, but it’s clear that more and more companies are recognizing the need to have some strategy around hosting their applications on multiple cloud providers. Earlier this year,&hellip;</p>
</div>
<footer>
<a href="/living-in-a-multi-cloud-world/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/living-in-a-multi-cloud-world/#disqus_thread">Comments</a> | tagged with <a href="/tag/google-cloud/">Google Cloud</a>, <a href="/tag/cloud-computing/">Cloud Computing</a>, <a href="/tag/internet-summit/">Internet Summit</a>
</span>
</small>
</footer>
</article>
<article class="post tag-legal tag-jengo tag-patents">
<header class="post-header">
<h2 class="title"><a href="/supreme-court-wanders-into-patent-troll-fight/">The Supreme Court Wanders into the Patent Troll Fight</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 20th, 2017 6:18PM">November 20th, 2017 6:18PM</time>
by <a href="/author/edo-royker/">Edo Royker</a>.
</div>
</header>
<div class="post-excerpt">
<p>Next Monday, the US Supreme Court will hear oral arguments in Oil States Energy Services, LLC vs. Greene’s Energy Group, LLC, which is a case to determine whether the Inter Partes Review (IPR) administrative process at the US Patent and Trademark Office (USPTO) used to determine the validity of patents is constitutional. The constitutionality of the IPR process is one of the biggest legal issues facing innovative&hellip;</p>
</div>
<footer>
<a href="/supreme-court-wanders-into-patent-troll-fight/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/supreme-court-wanders-into-patent-troll-fight/#disqus_thread">Comments</a> | tagged with <a href="/tag/legal/">Legal</a>, <a href="/tag/jengo/">Jengo</a>, <a href="/tag/patents/">Patents</a>
</span>
</small>
</footer>
</article>
<article class="post tag-cloudflare-apps tag-developers tag-user-engagement">
<header class="post-header">
<h2 class="title"><a href="/7cloudflareappsengagement/">7 Cloudflare Apps Which Increase User Engagement on Your Site</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 14th, 2017 8:21PM">November 14th, 2017 8:21PM</time>
by <a href="/author/andrew/">Andrew Fitch</a>.
</div>
</header>
<div class="post-excerpt">
<p>Cloudflare Apps now lists 95 apps from apps which grow email lists to apps which acquire new customers to apps which help site owners make more money. The great thing about these apps is that users don't have to have any coding or development skills. They can just sign up for the app and start using it on their sites. Let’s take a moment to highlight some&hellip;</p>
</div>
<footer>
<a href="/7cloudflareappsengagement/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/7cloudflareappsengagement/#disqus_thread">Comments</a> | tagged with <a href="/tag/cloudflare-apps/">Cloudflare Apps</a>, <a href="/tag/developers/">Developers</a>, <a href="/tag/user-engagement/">User Engagement</a>
</span>
</small>
</footer>
</article>
<article class="post tag-acquisitions tag-cloudflare-team tag-mobile tag-neumob">
<header class="post-header">
<h2 class="title"><a href="/neumob-optimizing-mobile/">The Super Secret Cloudflare Master Plan, or why we acquired Neumob</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 14th, 2017 2:00PM">November 14th, 2017 2:00PM</time>
by <a href="/author/john-graham-cumming/">John Graham-Cumming</a>.
</div>
</header>
<div class="post-excerpt">
<p>We announced today that Cloudflare has acquired Neumob. Neumob’s team built exceptional technology to speed up mobile apps, reduce errors on challenging mobile networks, and increase conversions. Cloudflare will integrate the Neumob technology with our global network to give Neumob truly global reach. It’s tempting to think of the Neumob acquisition as a point product added to the Cloudflare portfolio. But it actually represents a key&hellip;</p>
</div>
<footer>
<a href="/neumob-optimizing-mobile/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/neumob-optimizing-mobile/#disqus_thread">Comments</a> | tagged with <a href="/tag/acquisitions/">Acquisitions</a>, <a href="/tag/cloudflare-team/">Cloudflare Team</a>, <a href="/tag/mobile/">Mobile</a>, <a href="/tag/neumob/">Neumob</a>
</span>
</small>
</footer>
</article>
<article class="post tag-security tag-legal tag-privacy tag-attacks">
<header class="post-header">
<h2 class="title"><a href="/thwarting-the-tactics-of-the-equifax-attackers/">Thwarting the Tactics of the Equifax Attackers</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 13th, 2017 4:09PM">November 13th, 2017 4:09PM</time>
by <a href="/author/alex-cruz-farmer/">Alex Cruz Farmer</a>.
</div>
</header>
<div class="post-excerpt">
<p>We are now 3 months on from one of the biggest, most significant data breaches in history, but has it redefined people's awareness on security? The answer to that is absolutely yes, awareness is at an all-time high. Awareness, however, does not always result in positive action. The fallacy which is often assumed is "surely, if I keep my software up to date with all the patches, that's&hellip;</p>
</div>
<footer>
<a href="/thwarting-the-tactics-of-the-equifax-attackers/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/thwarting-the-tactics-of-the-equifax-attackers/#disqus_thread">Comments</a> | tagged with <a href="/tag/security/">Security</a>, <a href="/tag/legal/">Legal</a>, <a href="/tag/privacy/">Privacy</a>, <a href="/tag/attacks/">Attacks</a>
</span>
</small>
</footer>
</article>
<article class="post tag-go tag-performance tag-golang tag-developers">
<header class="post-header">
<h2 class="title"><a href="/go-dont-collect-my-garbage/">Go, don&#x27;t collect my garbage</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 13th, 2017 10:31AM">November 13th, 2017 10:31AM</time>
by <a href="/author/vlad-krasnov/">Vlad Krasnov</a>.
</div>
</header>
<div class="post-excerpt">
<p>Not long ago I needed to benchmark the performance of Golang on a many-core machine. I took several of the benchmarks that are bundled with the Go source code, copied them, and modified them to run on all available threads. In that case the machine has 24 cores and 48 threads. CC BY-SA 2.0 image by sponki25 I started with ECDSA P256 Sign, probably because I have&hellip;</p>
</div>
<footer>
<a href="/go-dont-collect-my-garbage/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/go-dont-collect-my-garbage/#disqus_thread">Comments</a> | tagged with <a href="/tag/go/">Go</a>, <a href="/tag/performance/">Performance</a>, <a href="/tag/golang/">golang</a>, <a href="/tag/developers/">Developers</a>
</span>
</small>
</footer>
</article>
<article class="post tag-developers tag-javascript tag-php tag-lua tag-go tag-meetup tag-cloudflare-meetups tag-community tag-pizza">
<header class="post-header">
<h2 class="title"><a href="/cloudflare-wants-to-buy-your-meetup-group-pizza/">Cloudflare Wants to Buy Your Meetup Group Pizza</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 10th, 2017 3:00PM">November 10th, 2017 3:00PM</time>
by <a href="/author/andrew/">Andrew Fitch</a>.
</div>
</header>
<div class="post-excerpt">
<p>If you’re a web dev / devops / etc. meetup group that also works toward building a faster, safer Internet, I want to support your awesome group by buying you pizza. If your group’s focus falls within one of the subject categories below and you’re willing to give us a 30 second shout out and tweet a photo of your group and @Cloudflare, your meetup’s pizza&hellip;</p>
</div>
<footer>
<a href="/cloudflare-wants-to-buy-your-meetup-group-pizza/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/cloudflare-wants-to-buy-your-meetup-group-pizza/#disqus_thread">Comments</a> | tagged with <a href="/tag/developers/">Developers</a>, <a href="/tag/javascript/">javascript</a>, <a href="/tag/php/">php</a>, <a href="/tag/lua/">lua</a>, <a href="/tag/go/">Go</a>, <a href="/tag/meetup/">MeetUp</a>, <a href="/tag/cloudflare-meetups/">Cloudflare Meetups</a>, <a href="/tag/community/">Community</a>, <a href="/tag/pizza/">Pizza</a>
</span>
</small>
</footer>
</article>
<article class="post">
<header class="post-header">
<h2 class="title"><a href="/on-the-dangers-of-intels-frequency-scaling/">On the dangers of Intel&#x27;s frequency scaling</a></h2>
<div class="meta">
Published on <time class="meta-date" datetime="November 10th, 2017 11:06AM">November 10th, 2017 11:06AM</time>
by <a href="/author/vlad-krasnov/">Vlad Krasnov</a>.
</div>
</header>
<div class="post-excerpt">
<p>While I was writing the post comparing the new Qualcomm server chip, Centriq, to our current stock of Intel Skylake-based Xeons, I noticed a disturbing phenomena. When benchmarking OpenSSL 1.1.1dev, I discovered that the performance of the cipher ChaCha20-Poly1305 does not scale very well. On a single thread, it performed at the speed of approximately 2.89GB/s, whereas on 24 cores, and 48 threads it&hellip;</p>
</div>
<footer>
<a href="/on-the-dangers-of-intels-frequency-scaling/" class="more">Read more &raquo; </a><br>
<small>
<span class="post-meta">
<a href="/on-the-dangers-of-intels-frequency-scaling/#disqus_thread">Comments</a>
</span>
</small>
</footer>
</article>
<section class="clearfix" role="navigation">
<a class="newer-posts btn" href="/page/2/">Older &raquo;</a>
</section>
</section>
<aside class="sidebar">
<div class="widget">
<input type="text" placeholder="Search the blog" class="st-default-search-input"></input>
<script type="text/javascript">
(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t);
e=d.getElementsByTagName(t)[0];s.async=0;s.src=u;e.parentNode.insertBefore(s,e);
})(window,document,'script','//s.swiftypecdn.com/install/v2/st.js','_st');
_st('install','_KobMC_zsd_tDx_7NWiX','2.0.0');
</script>
</div>
<div class="widget">
<h4 class="widget-title">Cloudflare blog</h4>
<p style="margin-top: 20px">
<a href="https://www.cloudflare.com/enterprise-service-request" class="btn btn-success" tabindex="11" target="_blank">Contact our team</a>
</p>
<p>
<strong>US callers</strong><br/>
1 (888) 99-FLARE <br/>
<strong>UK callers</strong><br/>
+44 (0)20 3514 6970<br/>
<strong>International callers</strong><br/>
+1 (650) 319-8930 <BR/><BR/>
<a href="https://www.cloudflare.com/plans" target="_blank">Full feature list and plan types</a>
</p>
<p>Cloudflare provides performance and security for any website. More than 6 million websites use Cloudflare.</p>
<p>There is no hardware or software. Cloudflare works at the DNS level. It takes only 5 minutes to sign up. To learn more, please visit our website</p>
</div>
<div class="widget">
<h4 class="widget-title">Cloudflare features</h4>
<ul class="menu menu-sidebar">
<li><a href="https://www.cloudflare.com/">Overview</a></li>
<li><a href="https://www.cloudflare.com/cdn/">CDN</a></li>
<li><a href="https://www.cloudflare.com/website-optimization/">Optimizer</a></li>
<li><a href="https://www.cloudflare.com/security/">Security</a></li>
<li><a href="https://www.cloudflare.com/analytics/">Analytics</a></li>
<li><a href="https://www.cloudflare.com/apps">Apps</a></li>
<li><a href="https://www.cloudflare.com/network/">Network map</a></li>
<li><a href="https://www.cloudflarestatus.com">System status</a></li>
</ul>
</div>
<div id="mc_embed_signup" class="widget">
<form action="https://cloudflare.us5.list-manage.com/subscribe/post?u=d80d4d74266c0c044b0bcd7ca&amp;id=8dc0bf9dea" method="post" id="mc-embedded-subscribe-form" name="mc-embedded-subscribe-form" class="validate" target="_blank" novalidate>
<input type="email" value="" name="EMAIL" class="width-full required email" id="mce-EMAIL" placeholder="Enter your email address"/>
<div id="mce-responses" class="clearfix">
<div class="response" id="mce-error-response" style="display:none"></div>
<div class="response" id="mce-success-response" style="display:none"></div>
</div>
<div class="clearfix">
<button type="submit" name="subscribe" id="mc-embedded-subscribe" class="btn btn-primary width-full">Sign up for email updates</button>
</div>
</form>
</div>
</aside>
</div>
<footer id="footer" class="footer">
<div class="wrapper">
<nav class="navigation footer-nav">
<ul role="navigation">
<li id="cf_nav_menu-2" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">What We Do</h6>
<div class="menu-what-we-do-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/plans">Plans</a></li>
<li><a href="https://www.cloudflare.com/performance/">Performance</a></li>
<li><a href="https://www.cloudflare.com/security/">Security</a></li>
<li><a href="https://www.cloudflare.com/reliability/">Reliability</a></li>
<li><a href="https://www.cloudflare.com/apps">Apps</a></li>
<li><a href="https://www.cloudflare.com/network-map">Network</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-3" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Resources</h6>
<div class="menu-support-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/support">Help Center</a></li>
<li><a href="https://www.cloudflare.com/community">Community</a></li>
<li><a href="https://www.cloudflare.com/video">Video Guides</a></li>
<li><a href="https://www.cloudflarestatus.com">System Status</a></li>
<li><a href="https://www.cloudflare.com/contact">Contact Us</a></li>
<li class="active"><a href="/">Blog</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-4" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Not a Developer?</h6>
<div class="menu-resources-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/case-studies">Case Studies</a></li>
<li><a href="https://www.cloudflare.com/resources/">White Papers</a></li>
<li><a href="https://www.cloudflare.com/internet-summit/">Internet Summit</a></li>
<li><a href="https://www.cloudflare.com/hosting-partners">Partners</a></li>
<li><a href="https://www.cloudflare.com/hosting-partners">Integrations</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-5" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">About Us</h6>
<div class="menu-about-us-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/people">Our Team</a></li>
<li><a href="https://www.cloudflare.com/join-our-team">Careers</a></li>
<li><a href="https://www.cloudflare.com/press-center">Press</a></li>
<li><a href="https://www.cloudflare.com/terms">Terms of Service</a></li>
<li><a href="https://www.cloudflare.com/security-policy/">Privacy &amp; Security</a></li>
<li><a href="https://www.cloudflare.com/abuse/">Trust &amp; Safety</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-6" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Connect</h6>
<div class="menu-connect-container">
<ul class="menu menu-footer">
<li><a href="http://twitter.com/cloudflare">Twitter</a></li>
<li><a href="https://www.facebook.com/Cloudflare">Facebook</a></li>
<li><a href="https://www.linkedin.com/company/cloudflare-inc-">LinkedIn</a></li>
<li><a href="https://www.youtube.com/cloudflare-">YouTube</a></li>
<li><a href="https://plus.google.com/+cloudflare/posts">Google+</a></li>
<li><a href="/rss/">RSS</a></li>
</ul>
</div>
</li>
</ul>
<div class="credits">All content &copy; 2017 <a href="https://cloudflare.com">Cloudflare</a>. Proudly published with <a href="https://ghost.org">Ghost</a>.</div>
</nav>
</div>
</footer>
<script>
var links = document.links;
for (var i = 0, linksLength = links.length; i < linksLength; i++) {
if (links[i].hostname != window.location.hostname) {
links[i].target = '_blank';
}
}
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/prism.min.js"></script>
<script type="text/javascript" src="/assets/js/jquery.fitvids.js?v=b6cf3f99a6"></script>
<script type="text/javascript">
$(document).ready(function(){ $(".post-content").fitVids(); });
</script>
<script type="text/javascript">
var disqus_shortname = 'cloudflare';
(function () {
var s = document.createElement('script'); s.async = true;
s.type = 'text/javascript';
s.src = '//' + disqus_shortname + '.disqus.com/count.js';
(document.getElementsByTagName('HEAD')[0] || document.getElementsByTagName('BODY')[0]).appendChild(s);
}());
</script>
</body>
</html>

515
h2mux/sample/index1.html
View File

@ -1,515 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<title>Living In A Multi-Cloud World</title>
<meta name="description" content="At our recent Internet Summit, we hosted a discussion on A Cloud Without Handcuffs with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS." />
<meta name="HandheldFriendly" content="True">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
<link rel="shortcut icon" href="/assets/images/favicon.ico?v=b6cf3f99a6">
<link rel="apple-touch-icon-precomposed" sizes="57x57" href="/assets/images/apple-touch-icon-57x57-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/assets/images/apple-touch-icon-72x72-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/assets/images/apple-touch-icon-114x114-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/assets/images/apple-touch-icon-144x144-precomposed.png?v=b6cf3f99a6" />
<link rel="stylesheet" type="text/css" href="/assets/css/screen.css?v=b6cf3f99a6" />
<!--[if lt IE 9]><link rel="stylesheet" type="text/css" href="/assets/css/ie.css?v=b6cf3f99a6" /><![endif]-->
<!--<link href="http://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,400,700,300,600" rel="stylesheet" type="text/css">-->
<script>(function(G,o,O,g,l){G.GoogleAnalyticsObject=O;G[O]||(G[O]=function(){(G[O].q=G[O].q||[]).push(arguments)});G[O].l=+new Date;g=o.createElement('script'),l=o.scripts[0];g.src='//www.google-analytics.com/analytics.js';l.parentNode.insertBefore(g,l)}(this,document,'ga'));ga('create','UA-10218544-12', 'auto');ga('send','pageview')</script>
<link rel="canonical" href="http://blog.cloudflare.com/living-in-a-multi-cloud-world/" />
<meta name="referrer" content="no-referrer-when-downgrade" />
<link rel="amphtml" href="http://blog.cloudflare.com/living-in-a-multi-cloud-world/amp/" />
<meta property="og:site_name" content="Cloudflare Blog" />
<meta property="og:type" content="article" />
<meta property="og:title" content="Living In A Multi-Cloud World" />
<meta property="og:description" content="At our recent Internet Summit, we hosted a discussion on A Cloud Without Handcuffs with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS." />
<meta property="og:url" content="http://blog.cloudflare.com/living-in-a-multi-cloud-world/" />
<meta property="og:image" content="http://blog.cloudflare.com/content/images/2017/11/Cloudflare_Multi_Cloud-1.png" />
<meta property="article:published_time" content="2017-11-21T16:30:00.000Z" />
<meta property="article:modified_time" content="2017-11-21T16:35:36.000Z" />
<meta property="article:tag" content="Google Cloud" />
<meta property="article:tag" content="Cloud Computing" />
<meta property="article:tag" content="Internet Summit" />
<meta property="article:publisher" content="https://www.facebook.com/Cloudflare" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="Living In A Multi-Cloud World" />
<meta name="twitter:description" content="At our recent Internet Summit, we hosted a discussion on A Cloud Without Handcuffs with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS." />
<meta name="twitter:url" content="http://blog.cloudflare.com/living-in-a-multi-cloud-world/" />
<meta name="twitter:image" content="http://blog.cloudflare.com/content/images/2017/11/Cloudflare_Multi_Cloud-1.png" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Sergi Isasi" />
<meta name="twitter:label2" content="Filed under" />
<meta name="twitter:data2" content="Google Cloud, Cloud Computing, Internet Summit" />
<meta name="twitter:site" content="@cloudflare" />
<meta name="twitter:creator" content="@sgisasi" />
<meta property="og:image:width" content="2002" />
<meta property="og:image:height" content="934" />
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"publisher": {
"@type": "Organization",
"name": "Cloudflare Blog",
"logo": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2016/09/logo-for-blog_thumb.png",
"width": 189,
"height": 47
}
},
"author": {
"@type": "Person",
"name": "Sergi Isasi",
"image": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2017/11/FullSizeRender_jpeg.png",
"width": 487,
"height": 487
},
"url": "http://blog.cloudflare.com/author/sergi/",
"sameAs": [
"https://twitter.com/sgisasi"
],
"description": "Product Management @ Cloudflare. "
},
"headline": "Living In A Multi-Cloud World",
"url": "https://blog.cloudflare.com/living-in-a-multi-cloud-world/",
"datePublished": "2017-11-21T16:30:00.000Z",
"dateModified": "2017-11-21T16:35:36.000Z",
"image": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2017/11/Cloudflare_Multi_Cloud-1.png",
"width": 2002,
"height": 934
},
"keywords": "Google Cloud, Cloud Computing, Internet Summit",
"description": "At our recent Internet Summit, we hosted a discussion on A Cloud Without Handcuffs with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS.",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "http://blog.cloudflare.com"
}
}
</script>
<script type="text/javascript" src="/shared/ghost-url.min.js?v=b6cf3f99a6"></script>
<script type="text/javascript">
ghost.init({
clientId: "ghost-frontend",
clientSecret: "cf0df60d1ab4"
});
</script>
<meta name="generator" content="Ghost 0.11" />
<link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
<meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
<meta class="swiftype" name="language" data-type="string" content="en" />
<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
<script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
<script>
var trackRecruitingLink = function(role, url) {
ga('send', 'event', 'recruiting', 'jobscore-click', role, {
'transport': 'beacon',
'hitCallback': function(){document.location = url;}
});
}
</script>
<script type="text/javascript">
(function() {
var didInit = false;
function initMunchkin() {
if(didInit === false) {
didInit = true;
Munchkin.init('713-XSC-918');
}
}
var s = document.createElement('script');
s.type = 'text/javascript';
s.async = true;
s.src = '//munchkin.marketo.net/munchkin.js';
s.onreadystatechange = function() {
if (this.readyState == 'complete' || this.readyState == 'loaded') {
initMunchkin();
}
};
s.onload = initMunchkin;
document.getElementsByTagName('head')[0].appendChild(s);
})();
</script>
<script>
var HTMLAttrToAdd = document.querySelector("html");
HTMLAttrToAdd.setAttribute("lang", "en");
</script>
<style>
table {
background-color: transparent;
}
td {
padding: 5px 1em;
}
pre {
max-height: 500px;
overflow-y: scroll;
}
</style>
<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
<link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
<style>
.st-default-search-input {
font-family: Helvetica, Arial, "Lucida Grande", sans-serif;
font-size: 14px;
line-height: 16px;
font-weight: 400;
-moz-transition: opacity 0.2s;
-o-transition: opacity 0.2s;
-webkit-transition: opacity 0.2s;
transition: opacity 0.2s;
display: inline-block;
width: 190px;
height: 16px;
padding: 7px 11px 7px 28px;
border: 1px solid rgba(0, 0, 0, 0.25);
color: #444;
-moz-box-sizing: content-box;
box-sizing: content-box;
-moz-border-radius: 5px;
-webkit-border-radius: 5px;
border-radius: 5px;
background: #fff 8px 8px no-repeat url("data:image/png;base64, iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAYAAABy6%2BR8AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAIGNIUk0AAG11AABzoAAA%2FN0AAINkAABw6AAA7GgAADA%2BAAAQkOTsmeoAAAESSURBVHjajNCxS9VRGMbxz71E4OwgoXPQxVEpXCI47%2BZqGP0LCoJO7UVD3QZzb3SwcHB7F3Uw3Zpd%2FAPCcJKG7Dj4u%2FK7Pwp94HDg5Xyf5z1Pr9YKImKANTzFXxzjU2ae6qhXaxURr%2FAFl9hHDy%2FwEK8z89sYVEp5gh84wMvMvGiSJ%2FEV85jNzLMR1McqfmN5BEBmnmMJFSvtpH7jdJiZv7q7Z%2BZPfMdcF6rN%2FT%2F1m2LGBkd4HhFT3dcRMY2FpskxaLNpayciHrWAGeziD7b%2BVfkithuTk8bkGa4wgWFmbrSTZOYeBvjc%2BucQj%2FEe6xHx4Taq1nrnKaW8K6XUUsrHWuvNevdRRLzFGwzvDbXAB9cDAHvhedDruuxSAAAAAElFTkSuQmCC")
}
.st-ui-close-button {
-moz-transition: none;
-o-transition: none;
-webkit-transition: none;
transition: none
}
</style>
</head>
<body class="post-template tag-google-cloud tag-cloud-computing tag-internet-summit">
<div id="fb-root"></div>
<header id="header" class="header">
<div class="wrapper">
<a href="https://www.cloudflare.com" class="logo logo-header">Cloudflare</a>
<nav id="main-menu" class="header-navigation navigation" role="navigation">
<ul class="menu menu-header">
<li><a href="https://blog.cloudflare.com/">Blog home</a></li>
<li><a href="https://www.cloudflare.com/overview" tabindex="1">What we do</a></li>
<li><a href="https://www.cloudflare.com/support" tabindex="9">Support</a></li>
<li><a href="https://www.cloudflare.com/community" tabindex="9">Community</a></li>
<li><a href="https://www.cloudflare.com/login" tabindex="10">Login</a></li>
<li><a href="https://www.cloudflare.com/sign-up" class="btn btn-success" tabindex="11">Sign up</a></li>
</ul>
</nav>
</div>
</header>
<div class="wrapper reverse-sidebar">
<section class="primary-content" role="main">
<article class="post tag-google-cloud tag-cloud-computing tag-internet-summit">
<header class="post-header">
<h1 class="title">Living In A Multi-Cloud World</h1>
<div class="meta">
<time class="meta-date" datetime="2017-11-21">21 Nov 2017</time>
by <a href="/author/sergi/">Sergi Isasi</a>.
</div>
<div class="social">
<div class="g-plusone" data-size="medium" data-href="https://blog.cloudflare.com/living-in-a-multi-cloud-world/"></div>
<script type="IN/Share" data-url="https://blog.cloudflare.com/living-in-a-multi-cloud-world/" data-counter="right"></script>
<div class="fb-like" data-href="https://blog.cloudflare.com/living-in-a-multi-cloud-world/" data-layout="button_count" data-action="like" data-show-faces="false" data-share="false"></div>
<a href="https://twitter.com/share" class="twitter-share-button" data-url="https://blog.cloudflare.com/living-in-a-multi-cloud-world/" data-text="Living In A Multi-Cloud World" data-via="cloudflare" data-related="cloudflare">Tweet</a>
</div>
</header>
<div class="post-content">
<p>A few months ago at Cloudflare’s <a href="https://www.cloudflare.com/internet-summit/">Internet Summit</a>, we hosted a discussion on <a href="https://blog.cloudflare.com/a-cloud-without-handcuffs/">A Cloud Without Handcuffs</a> with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS. The conversation touched on multiple areas, but it’s clear that more and more companies are recognizing the need to have some strategy around hosting their applications on multiple cloud providers.</p>
<p>Earlier this year, Mary Meeker published her annual <a href="http://www.kpcb.com/internet-trends">Internet Trends</a> report which revealed that 22% of respondents viewed Cloud Vendor Lock-In as a top 3 concern, up from just 7% in 2012. This is in contrast to previous top concerns, Data Security and Cost &amp; Savings, both of which dropped amongst those surveyed.</p>
<p><img src="/content/images/2017/11/Mary-Meeker-Internet-Trends-2017.png" alt="Internet Trends" /></p>
<p>At Cloudflare, our mission is to help build a better internet. To fulfill this mission, our customers need to have consistent access to the best technology and services, over time. This is especially the case with respect to storage and compute providers. This means not becoming locked-in to any single provider and taking advantage of multiple cloud computing vendors (such as Amazon Web Services or Google Cloud Platform) for the same end user services. </p>
<h3 id="thebenefitsofhavingmultiplecloudvendors">The Benefits of Having Multiple Cloud Vendors</h3>
<p>There are a number of potential challenges when selecting a single cloud provider. Though there may be scenarios where it makes sense to consolidate on a single vendor, our belief is that it is important that customers are aware of their choice and downsides of being potentially locked-in to that particular vendor. In short, know what trade offs you are making should you decide to continue to consolidate parts of your network, compute, and storage with a single cloud provider. While not comprehensive, here are a few trade-offs you may be making if you are locked-in to one cloud.</p>
<h4 id="costefficiences">Cost Efficiences</h4>
<p>For some companies, there may be a cost savings involved in spreading traffic across multiple vendors. Some can take advantage of free or reduced cost tiers at lower volumes. Vendors may provide reduced costs for certain times of day that are lower utilized on their infrastructure. Applications can have varying compute requirements amongst layers of the application: some may require faster, immediate processing while others may benefit from delayed processing at a lower cost. </p>
<h4 id="negotiationstrength">Negotiation Strength</h4>
<p>One of the most important reasons to consider deploying in multiple cloud providers is to minimize your reliance on a single vendor’s technology for your critical business processes. As you become more vertically integrated with any vendor, your negotiation posture for pricing or favorable contract terms becomes diminished. Having production ready code available on multiple providers allows you to have less technical debt should you need to change. If you go a step further and are already sending traffic to multiple providers, you have minimized the technical debt required to switch and can negotiate from a position of strength.</p>
<h4 id="businesscontinuityorhighavailability">Business Continuity or High Availability</h4>
<p>While the major cloud providers are generally reliable, there have been a few notable outages in recent years. The most significant in recent memory being Amazon’s <a href="https://aws.amazon.com/message/41926/">US-EAST S3</a> outage in February. Some organizations may have a policy specifying multiple providers for high availability while others should consider it where necessary and feasible as a best practice. A multi-cloud strategy can lower operational risk from a single vendor’s mistakes causing a significant outage for a mission critical application.</p>
<h4 id="experimentation">Experimentation</h4>
<p>One of the exciting things about having competition in the space is the level of innovation and feature velocity of each provider. Every year there are major announcements of new products or features that may have a significant impact on improving your organization's competitive advantage. Having test and production environments in multiple providers gives your engineers the ability to understand and experiment with a new capability in the context of your technology stack and data. You may even try these features for a portion of your traffic and get real world data on any benefits realized.</p>
<h3 id="cloudflaresrole">Cloudflare’s Role</h3>
<p>Cloudflare is an independent third party in your multi-cloud strategy. Our goal is to minimize the layers of lock-in between you and a provider and lower the effort of change. In particular, one area where we can help right away is to minimize the operational changes necessary at the network, similar to what Kubernetes can do at the storage and compute level. As a benefit of our network, you can also have a centralized point for security and operational control.</p>
<p><img src="/content/images/2017/11/Cloudflare_Multi_Cloud.png" alt="Cloudflare Multi Cloud" /></p>
<p>Cloudflare’s Load Balancing can easily be configured to act as your global application traffic aggregator and distribute your traffic amongst origins at as many clouds as you choose to utilize. Active layer 7 health checks continually probe your origins and can automatically move traffic in the case of network or application failure. All consolidated web traffic can be inspected and acted upon by Cloudflare’s best of breed <a href="https://www.cloudflare.com/security/">Security</a> services, providing a single control point and visibility across all application traffic, regardless of which cloud the origin may be on. You also have the benefit of Cloudflare’s <a href="https://www.cloudflare.com/network/">Global Anycast Network</a>, providing for better speed and higher availability regardless of which clouds your origins are hosted on.</p>
<h3 id="billforwardusingcloudflaretoimplementmulticloud">Billforward: Using Cloudflare to Implement Multi-Cloud</h3>
<p>Billforward is a San Francisco and London based startup that is focused and mission driven on changing the way people bill and charge their customers, providing a solution to the complexities of Quote-to-Cash. Their platform is built on a number of Rest APIs that other developers call to bill and generate revenue for their own companies. </p>
<p>Billforward is using Cloudflare for its core customer facing application to failover traffic between Google Compute Engine and Amazon Web Services. Acting as a reverse proxy, Cloudflare receives all requests for and decides which of Billforward’s two configured cloud origins to use based upon the availability of that origin in near real-time. This allows Billforward to completely manage the connections to and from two disparate cloud providers using Cloudflare’s UI or API. Billforward is in the process of migrating all of their customer facing domains to a similar setup.</p>
<h4 id="configuration">Configuration</h4>
<p>Billforward has a single load balanced hostname with two available Pools. They’ve named the two Pools with “gce” and “aws” labels and each Pool has one Origin associated with it. All of the Pools are enabled and the entire LB/hostname is proxied through Cloudflare (as indicated by the orange cloud).</p>
<p><img src="/content/images/2017/11/Billforward_Config_UI.png" alt="Billforward Configuration UI" /></p>
<p>Cloudflare probes Billforward’s Origins once every minute from all of Cloudflare’s data centers around the world (a feature available to all Load Balancing Enterprise customers). If Billforward’s GCE Origin goes down, Cloudflare will quickly and automatically failover to the AWS Origin with no actions required from Billforward’s team.</p>
<p>Google Compute Engine was chosen as the primary provider for this application by virtue of cost. Martin Lee, Site Reliability Engineer at Billforward says, “Essentially, GCE is cheaper for our general purpose computing needs but we're more experienced with deployments in AWS. This strategy allows us to switch back and forth at will and avoid being tied in to either platform.” It is likely that Billforward will change the priority as pricing models evolve. <br />
<br> </p>
<blockquote>
<p>“It's a fairly fast moving world and features released by cloud providers can have a meaningful impact on performance and cost on a week by week basis - it helps to stay flexible,” says Martin. “We may also change priority based on features.”</p>
</blockquote>
<p><br>For orchestration of the compute and storage layers, Billforward uses <a href="https://www.docker.com/">Docker</a> containers managed through <a href="http://www.rancher.com/">Rancher</a>. They use distinct environments between cloud providers but are considering bridging an environment across cloud providers and using VPNs between them, which will enable them to move load between providers even more easily. “Our system is loosely coupled through a message queue,” adds Martin. “Having a container system across clouds means we can really take advantage of this - we can very easily move workloads across clouds without any danger of dropping tasks or ending up in an inconsistent state.”</p>
<h4 id="benefits">Benefits</h4>
<p>Billforward manages these connections at Cloudflare’s edge. Through this interface (or via the Cloudflare APIs), they can also manually move traffic from GCE to AWS by just disabling the GCE pool or by rearranging the Pool priority and make AWS the primary. These changes are near instant on the Cloudflare network and require no downtime to Billforward’s customer facing application. This allows them to act on potential advantageous pricing changes between the two cloud providers or move traffic to hit pricing tiers. </p>
<p>In addition, Billforward is now not “locked-in” to either provider’s network; being able to move traffic and without any downtime means they can make traffic changes independent of Amazon or Google. They can also integrate additional cloud providers any time they deem fit: adding Microsoft Azure, for example, as a third Origin would be as simple as creating a new Pool and adding it to the Load Balancer. </p>
<p>Billforward is a good example of a forward thinking company that is taking advantage of technologies from multiple providers to best serve their business and customers, while not being reliant on a single vendor. For further detail on their setup using Cloudflare, please check their <a href="https://www.billforward.net/blog/being-multi-cloud-with-cloudflare/">blog</a>.</p>
</div>
<footer>
<small>
Tagged with <a href="/tag/google-cloud/">Google Cloud</a>, <a href="/tag/cloud-computing/">Cloud Computing</a>, <a href="/tag/internet-summit/">Internet Summit</a>
</small>
</footer>
<aside class="section learn-more">
<h5>Want to learn more about Cloudflare?</h5>
<p><a href="https://www.cloudflare.com" class="btn btn-success">Learn more</a></p>
</aside>
<aside class="section comments">
<h3>Comments</h3>
</aside>
<div id="disqus_thread"></div>
<script type="text/javascript">
var disqus_shortname = 'cloudflare';
(function() {
var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
})();
</script>
<noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
<a href="http://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>
</article>
<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');
</script>
<script>(function(d, s, id) {
var js, fjs = d.getElementsByTagName(s)[0];
if (d.getElementById(id)) return;
js = d.createElement(s); js.id = id;
js.src = "//connect.facebook.net/en_US/all.js#xfbml=1&appId=596756540369391";
fjs.parentNode.insertBefore(js, fjs);
}(document, 'script', 'facebook-jssdk'));
</script>
<script src="//platform.linkedin.com/in.js" type="text/javascript">lang: en_US</script>
<script type="text/javascript">
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/platform.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
</section>
<aside class="sidebar">
<div class="widget">
<input type="text" placeholder="Search the blog" class="st-default-search-input"></input>
<script type="text/javascript">
(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t);
e=d.getElementsByTagName(t)[0];s.async=0;s.src=u;e.parentNode.insertBefore(s,e);
})(window,document,'script','//s.swiftypecdn.com/install/v2/st.js','_st');
_st('install','_KobMC_zsd_tDx_7NWiX','2.0.0');
</script>
</div>
<div class="widget">
<h4 class="widget-title">Cloudflare blog</h4>
<p style="margin-top: 20px">
<a href="https://www.cloudflare.com/enterprise-service-request" class="btn btn-success" tabindex="11" target="_blank">Contact our team</a>
</p>
<p>
<strong>US callers</strong><br/>
1 (888) 99-FLARE <br/>
<strong>UK callers</strong><br/>
+44 (0)20 3514 6970<br/>
<strong>International callers</strong><br/>
+1 (650) 319-8930 <BR/><BR/>
<a href="https://www.cloudflare.com/plans" target="_blank">Full feature list and plan types</a>
</p>
<p>Cloudflare provides performance and security for any website. More than 6 million websites use Cloudflare.</p>
<p>There is no hardware or software. Cloudflare works at the DNS level. It takes only 5 minutes to sign up. To learn more, please visit our website</p>
</div>
<div class="widget">
<h4 class="widget-title">Cloudflare features</h4>
<ul class="menu menu-sidebar">
<li><a href="https://www.cloudflare.com/">Overview</a></li>
<li><a href="https://www.cloudflare.com/cdn/">CDN</a></li>
<li><a href="https://www.cloudflare.com/website-optimization/">Optimizer</a></li>
<li><a href="https://www.cloudflare.com/security/">Security</a></li>
<li><a href="https://www.cloudflare.com/analytics/">Analytics</a></li>
<li><a href="https://www.cloudflare.com/apps">Apps</a></li>
<li><a href="https://www.cloudflare.com/network/">Network map</a></li>
<li><a href="https://www.cloudflarestatus.com">System status</a></li>
</ul>
</div>
<div id="mc_embed_signup" class="widget">
<form action="https://cloudflare.us5.list-manage.com/subscribe/post?u=d80d4d74266c0c044b0bcd7ca&amp;id=8dc0bf9dea" method="post" id="mc-embedded-subscribe-form" name="mc-embedded-subscribe-form" class="validate" target="_blank" novalidate>
<input type="email" value="" name="EMAIL" class="width-full required email" id="mce-EMAIL" placeholder="Enter your email address"/>
<div id="mce-responses" class="clearfix">
<div class="response" id="mce-error-response" style="display:none"></div>
<div class="response" id="mce-success-response" style="display:none"></div>
</div>
<div class="clearfix">
<button type="submit" name="subscribe" id="mc-embedded-subscribe" class="btn btn-primary width-full">Sign up for email updates</button>
</div>
</form>
</div>
</aside>
</div>
<footer id="footer" class="footer">
<div class="wrapper">
<nav class="navigation footer-nav">
<ul role="navigation">
<li id="cf_nav_menu-2" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">What We Do</h6>
<div class="menu-what-we-do-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/plans">Plans</a></li>
<li><a href="https://www.cloudflare.com/performance/">Performance</a></li>
<li><a href="https://www.cloudflare.com/security/">Security</a></li>
<li><a href="https://www.cloudflare.com/reliability/">Reliability</a></li>
<li><a href="https://www.cloudflare.com/apps">Apps</a></li>
<li><a href="https://www.cloudflare.com/network-map">Network</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-3" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Resources</h6>
<div class="menu-support-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/support">Help Center</a></li>
<li><a href="https://www.cloudflare.com/community">Community</a></li>
<li><a href="https://www.cloudflare.com/video">Video Guides</a></li>
<li><a href="https://www.cloudflarestatus.com">System Status</a></li>
<li><a href="https://www.cloudflare.com/contact">Contact Us</a></li>
<li class="active"><a href="/">Blog</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-4" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Not a Developer?</h6>
<div class="menu-resources-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/case-studies">Case Studies</a></li>
<li><a href="https://www.cloudflare.com/resources/">White Papers</a></li>
<li><a href="https://www.cloudflare.com/internet-summit/">Internet Summit</a></li>
<li><a href="https://www.cloudflare.com/hosting-partners">Partners</a></li>
<li><a href="https://www.cloudflare.com/hosting-partners">Integrations</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-5" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">About Us</h6>
<div class="menu-about-us-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/people">Our Team</a></li>
<li><a href="https://www.cloudflare.com/join-our-team">Careers</a></li>
<li><a href="https://www.cloudflare.com/press-center">Press</a></li>
<li><a href="https://www.cloudflare.com/terms">Terms of Service</a></li>
<li><a href="https://www.cloudflare.com/security-policy/">Privacy &amp; Security</a></li>
<li><a href="https://www.cloudflare.com/abuse/">Trust &amp; Safety</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-6" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Connect</h6>
<div class="menu-connect-container">
<ul class="menu menu-footer">
<li><a href="http://twitter.com/cloudflare">Twitter</a></li>
<li><a href="https://www.facebook.com/Cloudflare">Facebook</a></li>
<li><a href="https://www.linkedin.com/company/cloudflare-inc-">LinkedIn</a></li>
<li><a href="https://www.youtube.com/cloudflare-">YouTube</a></li>
<li><a href="https://plus.google.com/+cloudflare/posts">Google+</a></li>
<li><a href="/rss/">RSS</a></li>
</ul>
</div>
</li>
</ul>
<div class="credits">All content &copy; 2017 <a href="https://cloudflare.com">Cloudflare</a>. Proudly published with <a href="https://ghost.org">Ghost</a>.</div>
</nav>
</div>
</footer>
<script>
var links = document.links;
for (var i = 0, linksLength = links.length; i < linksLength; i++) {
if (links[i].hostname != window.location.hostname) {
links[i].target = '_blank';
}
}
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/prism.min.js"></script>
<script type="text/javascript" src="/assets/js/jquery.fitvids.js?v=b6cf3f99a6"></script>
<script type="text/javascript">
$(document).ready(function(){ $(".post-content").fitVids(); });
</script>
<script type="text/javascript">
var disqus_shortname = 'cloudflare';
(function () {
var s = document.createElement('script'); s.async = true;
s.type = 'text/javascript';
s.src = '//' + disqus_shortname + '.disqus.com/count.js';
(document.getElementsByTagName('HEAD')[0] || document.getElementsByTagName('BODY')[0]).appendChild(s);
}());
</script>
</body>
</html>

502
h2mux/sample/index2.html
View File

@ -1,502 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<title>SCOTUS Wanders into Patent Troll Fight</title>
<meta name="description" content="" />
<meta name="HandheldFriendly" content="True">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
<link rel="shortcut icon" href="/assets/images/favicon.ico?v=b6cf3f99a6">
<link rel="apple-touch-icon-precomposed" sizes="57x57" href="/assets/images/apple-touch-icon-57x57-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/assets/images/apple-touch-icon-72x72-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/assets/images/apple-touch-icon-114x114-precomposed.png?v=b6cf3f99a6" />
<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/assets/images/apple-touch-icon-144x144-precomposed.png?v=b6cf3f99a6" />
<link rel="stylesheet" type="text/css" href="/assets/css/screen.css?v=b6cf3f99a6" />
<!--[if lt IE 9]><link rel="stylesheet" type="text/css" href="/assets/css/ie.css?v=b6cf3f99a6" /><![endif]-->
<!--<link href="http://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,400,700,300,600" rel="stylesheet" type="text/css">-->
<script>(function(G,o,O,g,l){G.GoogleAnalyticsObject=O;G[O]||(G[O]=function(){(G[O].q=G[O].q||[]).push(arguments)});G[O].l=+new Date;g=o.createElement('script'),l=o.scripts[0];g.src='//www.google-analytics.com/analytics.js';l.parentNode.insertBefore(g,l)}(this,document,'ga'));ga('create','UA-10218544-12', 'auto');ga('send','pageview')</script>
<link rel="canonical" href="http://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/" />
<meta name="referrer" content="no-referrer-when-downgrade" />
<link rel="amphtml" href="http://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/amp/" />
<meta property="og:site_name" content="Cloudflare Blog" />
<meta property="og:type" content="article" />
<meta property="og:title" content="SCOTUS Wanders into Patent Troll Fight" />
<meta property="og:description" content="Next Monday, the US Supreme Court will hear oral arguments in Oil States Energy Services, LLC vs. Greene’s Energy Group, LLC, which is a case to determine whether the Inter Partes Review (IPR) administrative process at the US Patent and Trademark Office (USPTO) used to determine the validity of" />
<meta property="og:url" content="http://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/" />
<meta property="og:image" content="http://blog.cloudflare.com/content/images/2017/11/Thomas_Rowlandson_-_The_Privy_Council_of_a_King_-_Google_Art_Project--1-.jpg" />
<meta property="article:published_time" content="2017-11-20T18:18:00.000Z" />
<meta property="article:modified_time" content="2017-11-20T22:51:13.000Z" />
<meta property="article:tag" content="Legal" />
<meta property="article:tag" content="Jengo" />
<meta property="article:tag" content="Patents" />
<meta property="article:publisher" content="https://www.facebook.com/Cloudflare" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="SCOTUS Wanders into Patent Troll Fight" />
<meta name="twitter:description" content="Next Monday, the US Supreme Court will hear oral arguments in Oil States Energy Services, LLC vs. Greene’s Energy Group, LLC, which is a case to determine whether the Inter Partes Review (IPR) administrative process at the US Patent and Trademark Office (USPTO) used to determine the validity of" />
<meta name="twitter:url" content="http://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/" />
<meta name="twitter:image" content="http://blog.cloudflare.com/content/images/2017/11/Thomas_Rowlandson_-_The_Privy_Council_of_a_King_-_Google_Art_Project--1-.jpg" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Edo Royker" />
<meta name="twitter:label2" content="Filed under" />
<meta name="twitter:data2" content="Legal, Jengo, Patents" />
<meta name="twitter:site" content="@cloudflare" />
<meta property="og:image:width" content="4468" />
<meta property="og:image:height" content="3183" />
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"publisher": {
"@type": "Organization",
"name": "Cloudflare Blog",
"logo": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2016/09/logo-for-blog_thumb.png",
"width": 189,
"height": 47
}
},
"author": {
"@type": "Person",
"name": "Edo Royker",
"image": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2017/11/AAEAAQAAAAAAAAdiAAAAJDdiMzU0OWYxLTBmOTMtNGZhZi1hNDQ1LTBhNjJhZDdmMGRlZA.jpg",
"width": 200,
"height": 200
},
"url": "http://blog.cloudflare.com/author/edo-royker/",
"sameAs": []
},
"headline": "SCOTUS Wanders into Patent Troll Fight",
"url": "https://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/",
"datePublished": "2017-11-20T18:18:00.000Z",
"dateModified": "2017-11-20T22:51:13.000Z",
"image": {
"@type": "ImageObject",
"url": "http://blog.cloudflare.com/content/images/2017/11/Thomas_Rowlandson_-_The_Privy_Council_of_a_King_-_Google_Art_Project--1-.jpg",
"width": 4468,
"height": 3183
},
"keywords": "Legal, Jengo, Patents",
"description": "Next Monday, the US Supreme Court will hear oral arguments in Oil States Energy Services, LLC vs. Greene’s Energy Group, LLC, which is a case to determine whether the Inter Partes Review (IPR) administrative process at the US Patent and Trademark Office (USPTO) used to determine the validity of",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "http://blog.cloudflare.com"
}
}
</script>
<script type="text/javascript" src="/shared/ghost-url.min.js?v=b6cf3f99a6"></script>
<script type="text/javascript">
ghost.init({
clientId: "ghost-frontend",
clientSecret: "cf0df60d1ab4"
});
</script>
<meta name="generator" content="Ghost 0.11" />
<link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
<meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
<meta class="swiftype" name="language" data-type="string" content="en" />
<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
<script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
<script>
var trackRecruitingLink = function(role, url) {
ga('send', 'event', 'recruiting', 'jobscore-click', role, {
'transport': 'beacon',
'hitCallback': function(){document.location = url;}
});
}
</script>
<script type="text/javascript">
(function() {
var didInit = false;
function initMunchkin() {
if(didInit === false) {
didInit = true;
Munchkin.init('713-XSC-918');
}
}
var s = document.createElement('script');
s.type = 'text/javascript';
s.async = true;
s.src = '//munchkin.marketo.net/munchkin.js';
s.onreadystatechange = function() {
if (this.readyState == 'complete' || this.readyState == 'loaded') {
initMunchkin();
}
};
s.onload = initMunchkin;
document.getElementsByTagName('head')[0].appendChild(s);
})();
</script>
<script>
var HTMLAttrToAdd = document.querySelector("html");
HTMLAttrToAdd.setAttribute("lang", "en");
</script>
<style>
table {
background-color: transparent;
}
td {
padding: 5px 1em;
}
pre {
max-height: 500px;
overflow-y: scroll;
}
</style>
<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
<link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
<style>
.st-default-search-input {
font-family: Helvetica, Arial, "Lucida Grande", sans-serif;
font-size: 14px;
line-height: 16px;
font-weight: 400;
-moz-transition: opacity 0.2s;
-o-transition: opacity 0.2s;
-webkit-transition: opacity 0.2s;
transition: opacity 0.2s;
display: inline-block;
width: 190px;
height: 16px;
padding: 7px 11px 7px 28px;
border: 1px solid rgba(0, 0, 0, 0.25);
color: #444;
-moz-box-sizing: content-box;
box-sizing: content-box;
-moz-border-radius: 5px;
-webkit-border-radius: 5px;
border-radius: 5px;
background: #fff 8px 8px no-repeat url("data:image/png;base64, iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAYAAABy6%2BR8AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAIGNIUk0AAG11AABzoAAA%2FN0AAINkAABw6AAA7GgAADA%2BAAAQkOTsmeoAAAESSURBVHjajNCxS9VRGMbxz71E4OwgoXPQxVEpXCI47%2BZqGP0LCoJO7UVD3QZzb3SwcHB7F3Uw3Zpd%2FAPCcJKG7Dj4u%2FK7Pwp94HDg5Xyf5z1Pr9YKImKANTzFXxzjU2ae6qhXaxURr%2FAFl9hHDy%2FwEK8z89sYVEp5gh84wMvMvGiSJ%2FEV85jNzLMR1McqfmN5BEBmnmMJFSvtpH7jdJiZv7q7Z%2BZPfMdcF6rN%2FT%2F1m2LGBkd4HhFT3dcRMY2FpskxaLNpayciHrWAGeziD7b%2BVfkithuTk8bkGa4wgWFmbrSTZOYeBvjc%2BucQj%2FEe6xHx4Taq1nrnKaW8K6XUUsrHWuvNevdRRLzFGwzvDbXAB9cDAHvhedDruuxSAAAAAElFTkSuQmCC")
}
.st-ui-close-button {
-moz-transition: none;
-o-transition: none;
-webkit-transition: none;
transition: none
}
</style>
</head>
<body class="post-template tag-legal tag-jengo tag-patents">
<div id="fb-root"></div>
<header id="header" class="header">
<div class="wrapper">
<a href="https://www.cloudflare.com" class="logo logo-header">Cloudflare</a>
<nav id="main-menu" class="header-navigation navigation" role="navigation">
<ul class="menu menu-header">
<li><a href="https://blog.cloudflare.com/">Blog home</a></li>
<li><a href="https://www.cloudflare.com/overview" tabindex="1">What we do</a></li>
<li><a href="https://www.cloudflare.com/support" tabindex="9">Support</a></li>
<li><a href="https://www.cloudflare.com/community" tabindex="9">Community</a></li>
<li><a href="https://www.cloudflare.com/login" tabindex="10">Login</a></li>
<li><a href="https://www.cloudflare.com/sign-up" class="btn btn-success" tabindex="11">Sign up</a></li>
</ul>
</nav>
</div>
</header>
<div class="wrapper reverse-sidebar">
<section class="primary-content" role="main">
<article class="post tag-legal tag-jengo tag-patents">
<header class="post-header">
<h1 class="title">The Supreme Court Wanders into the Patent Troll Fight</h1>
<div class="meta">
<time class="meta-date" datetime="2017-11-20">20 Nov 2017</time>
by <a href="/author/edo-royker/">Edo Royker</a>.
</div>
<div class="social">
<div class="g-plusone" data-size="medium" data-href="https://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/"></div>
<script type="IN/Share" data-url="https://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/" data-counter="right"></script>
<div class="fb-like" data-href="https://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/" data-layout="button_count" data-action="like" data-show-faces="false" data-share="false"></div>
<a href="https://twitter.com/share" class="twitter-share-button" data-url="https://blog.cloudflare.com/supreme-court-wanders-into-patent-troll-fight/" data-text="The Supreme Court Wanders into the Patent Troll Fight" data-via="cloudflare" data-related="cloudflare">Tweet</a>
</div>
</header>
<div class="post-content">
<p>Next Monday, the US Supreme Court will hear oral arguments in <em>Oil States Energy Services, LLC vs. Greene’s Energy Group, LLC</em>, which is a case to determine whether the Inter Partes Review (IPR) administrative process at the US Patent and Trademark Office (USPTO) used to determine the validity of patents is constitutional. </p>
<p>The constitutionality of the IPR process is one of the biggest legal issues facing innovative technology companies, as the availability of this process has greatly reduced the anticipated costs, and thereby lessened the threat, of patent troll litigation. As we discuss in this blog post, it is ironic that the outcome of a case that is of such great importance to the technology community today may hinge on what courts in Britain were and were not doing more than 200 years ago.</p>
<p><img src="/content/images/2017/11/Thomas_Rowlandson_-_The_Privy_Council_of_a_King_-_Google_Art_Project.jpg" alt="" title="" /><small>Thomas Rowlandson [Public domain], via <a href="https://commons.wikimedia.org/wiki/File%3AThomas_Rowlandson_-_The_Privy_Council_of_a_King_-_Google_Art_Project.jpg">Wikimedia Commons</a></small></p>
<p>As we have discussed in prior <a href="https://blog.cloudflare.com/project-jengo-challenges/">blog posts</a>, the stakes are high: if the Supreme Court finds IPR unconstitutional, then the entire system of administrative review by the USPTO — including IPR and ex parte processes — will be shuttered. This would be a mistake, as administrative recourse at the USPTO is one of the few ways to avoid the considerable costs and delays of federal court litigation, which can take years and run into the millions of dollars. Those heavy costs are often leveraged by patent trolls when they threaten litigation in the effort to procure easy and lucrative settlements from their targets. </p>
<h3 id="cloudflareispursuingourfightagainstpatenttrollsallthewaytothestepsofthesupremecourt">Cloudflare is Pursuing Our Fight Against Patent Trolls All the Way to the Steps of the Supreme Court</h3>
<p>Cloudflare joined Dell, Facebook, and a number of other companies, all practicing entities with large patent portfolios, in a <em>brief amici curiae</em> (or ‘friend of the court’ brief) in support of the IPR process, because it has a substantial positive impact on technological innovation in the United States. Amicus briefs allow parties who are interested in the outcome of a case, but are not parties to the immediate dispute before the court, to have input into the court’s deliberations. </p>
<p>As many of you are aware, we were sued by Blackbird Technologies, a notorious patent troll, earlier this year for patent infringement, and initiated <a href="https://blog.cloudflare.com/project-jengo/">Project Jengo</a> to crowd source prior art searches and invalidate Blackbird’s patents. One of our strategies for quickly and efficiently invalidating Blackbird’s patents is to take advantage of the IPR process at the USPTO, which can be completed in about half the time and at one tenth of the cost of a federal court case, and to initiate ex parte proceedings against Blackbird’s other patents that are overly broad and invalid. </p>
<p>A full copy of the Amicus Brief we joined in the Oil States case is <a href="http://www.scotusblog.com/wp-content/uploads/2017/11/16-712-bsac-Dell.pdf">available here</a>, and a summary of the argument follows. </p>
<h3 id="oilstatesmakesitscase">Oil States Makes its Case</h3>
<p>Oil States is an oilfield services and drilling equipment manufacturing company. The USPTO invalidated one of its patents related to oil drilling technology in an IPR proceeding while Oil States had a lawsuit pending against one of its competitors claiming infringement of its patent. After it lost the IPR, Oil States lost an appeal in a lower federal court based on the findings of the IPR proceeding. The Supreme Court agreed to hear the case to determine whether once the USPTO issues a patent, an inventor has a constitutionally protected property right that — under <a href="http://www.heritage.org/constitution/#!/articles/3">Article III</a> of the U.S. Constitution (which outlines the powers of the judicial branch of the government), and the <a href="https://constitutioncenter.org/interactive-constitution/amendments/amendment-vii">7th Amendment</a> (which addresses the right to a jury trial in certain types of cases) — cannot be revoked without intervention by the court system. </p>
<p><img src="/content/images/2017/11/2770193028_68edc662a9_b.jpg" alt="" title="" /><small><a href="https://www.flickr.com/photos/paul_lowry/2770193028">Image</a> by <a href="https://creativecommons.org/licenses/by/2.0/">Paul Lowry</a></small></p>
<p>As the patent owner, Oil States argues that the IPR process violates the relevant provisions of the constitution by allowing an administrative body, the Patent Trial and Appeal Board (PTAB)--a non-judicial forum, to decide a matter which was historically handled by the judiciary. This argument rests upon the premise that there was a historical analogue to cancellation of patent claims available in the judiciary. Since cancellation of patent claims was historically available in the judiciary, the cancellation of patent claims today must be consistent with that history and done exclusively by courts. </p>
<p>This argument is flawed on multiple counts, which are set forth in the “friend of the court” brief we joined.</p>
<h4 id="firstflawanadministrativeprocessevenanoriginalistcanlove">First Flaw: An Administrative Process Even an Originalist Can Love</h4>
<p>As the amicus brief we joined points out, patent revocation did not historically rest within the <em>exclusive</em> province of the common law and chancery courts, the historical equivalents in Britain to the judiciary in the United States. Rather, prior to the Founding of the United States, patent revocation rested entirely with the Crown of England’s Privy Council, a non-judicial body comprising of advisors to the king or queen of England. It wasn’t until later that the Privy Council granted the chancery court (the judiciary branch) concurrent authority to revoke patents. Because a non-judicial body had the authority to revoke patents when the US Constitution was framed, the general principles of separation of powers and the right to trial in the Constitution do not require that patentability challenges be decided solely by courts. </p>
<h4 id="secondflawthejudicialrolewaslimited">Second Flaw: The Judicial Role was Limited</h4>
<p>Not only did British courts share the power to address patent rights historically, the part shared by the the courts was significantly limited. Historically, the common-law and chancery courts only received a partial delegation of the Privy Council’s authority to invalidate patents. Courts only had the authority to invalidate patents for issues related to things like inequitable conduct (e.g., making false statements in the original patent application). The limited authority delegated to the England Courts did not include the authority to seek claim <em>cancellation</em> based on elements intrinsic to the patent or patent application, like lack of novelty or obviousness as done under an IPR proceeding. Rather, such authority remained with the Privy Council, a non-court authority, which decided questions like whether the invention was really new. Thus, like the PTAB, the Privy Council was a non-judicial body charged with responsibility to assess patent validity based on criteria that included the novelty of the invention.</p>
<p>We think these arguments are compelling and provide very strong reasons why the Supreme Court should resist the request that such matters be resolved exclusively in federal courts. We hope that’s the position they do take because the real world implications are significant. </p>
<h3 id="dontmesswithagoodthing">Don’t Mess with a Good Thing</h3>
<p>The IPR process is not only consistent with the US Constitution, but it also advances the Patent Clause’s objective of promoting the progress of science and useful arts. That is, the “quid pro quo of the patent system; the public must receive meaningful disclosure in exchange for being excluded from practicing the invention for a limited period of time” by patent rights. (<a href="http://caselaw.findlaw.com/us-federal-circuit/1330083.html">Enzo Biochem, Inc. v. Gen-probe Inc.</a>) Congress created the IPR process in the America Invents Act in 2011 to use administrative review to weed out poor-quality patents that did not satisfy this quid pro quo because they had not actually disclosed very much. Congress sought to provide quick and cost effective administrative procedures for challenging the validity of patent claims that did not disclose novel inventions, or that claimed to disclose substantially more innovation than they actually did, to improve patent quality and restore confidence in the presumption of validity. In other words, Congress created a system to specifically permit the efficient challenge of the zealous assertion of vague and overly broad patents. </p>
<p>As a recent study by the Congressional Research Service found, non-practicing entity (i.e., patent troll) patent litigation “activity cost defendants and licensees $29 billion in 2011, a 400 percent increase over $7 billion in 2005” and “the losses are mostly deadweight, with less than 25 percent flowing to innovation and at least that much going towards legal fees.” (<em>see</em> <a href="https://fas.org/sgp/crs/misc/R42668.pdf">Brian T. Yeh, Cong. Research sERV., R42668</a>) The IPR process enables innovative companies to navigate patent troll activity in an efficient manner and devote a greater proportion of their resources to research and development, rather than litigation or cost-of-litigation settlement fees for invalid patents. </p>
<p><img src="/content/images/2017/11/Troll-slip.jpg" alt="" title="" /><small>By EFF-Graphics (<a href="http://creativecommons.org/licenses/by/3.0/us/deed.en">Own work</a>), via <a href="https://commons.wikimedia.org/wiki/File%3ATroll-slip.jpg">Wikimedia Commons</a></small></p>
<p>Additionally, the IPR process reduces the total number and associated costs of patent disputes in a number of ways.</p>
<ul>
<li><p>Patent owners, especially patent trolls, are less likely to threaten litigation or file an infringement suit based on patent claims that they know or suspect to be invalid. In fact, patent owners who threaten or file suit merely to seek cost-of-litigation settlements have become far less prevalent because of the availability of the IPR process to reduce the cost of litigation.</p></li>
<li><p>Patent owners are less likely to initiate litigation out of concerns that the IPR proceedings may culminate in PTAB’s cancellation of all patent claims asserted in the infringement suit.</p></li>
<li><p>Where the PTAB does not cancel all asserted claims, statutory estoppel and the PTAB’s claim construction may serve to narrow the infringement issues to be resolved by the district court.</p></li>
</ul>
<p>Our hope is that the US Supreme Court justices take into full consideration the larger community of innovative companies that are helped by the IPR system in battling patent trolls, and do not limit their consideration to the implications on the parties to <em>Oil States</em> (neither of which is a non-practicing entity). As we have explained, not only does the IPR process enable innovative companies to focus their resources on technological innovation, instead of legal fees, but allowing the USPTO to administer IPR and ex parte proceedings is entirely consistent with the US Constitution.</p>
<p>While we await a decision in <em>Oil States</em>, expect to see Cloudflare initiate IPR and ex parte proceedings against Blackbird Technologies patents in the coming months. </p>
<p>We will make sure to keep you updated. </p>
</div>
<footer>
<small>
Tagged with <a href="/tag/legal/">Legal</a>, <a href="/tag/jengo/">Jengo</a>, <a href="/tag/patents/">Patents</a>
</small>
</footer>
<aside class="section learn-more">
<h5>Want to learn more about Cloudflare?</h5>
<p><a href="https://www.cloudflare.com" class="btn btn-success">Learn more</a></p>
</aside>
<aside class="section comments">
<h3>Comments</h3>
</aside>
<div id="disqus_thread"></div>
<script type="text/javascript">
var disqus_shortname = 'cloudflare';
(function() {
var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
})();
</script>
<noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
<a href="http://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>
</article>
<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');
</script>
<script>(function(d, s, id) {
var js, fjs = d.getElementsByTagName(s)[0];
if (d.getElementById(id)) return;
js = d.createElement(s); js.id = id;
js.src = "//connect.facebook.net/en_US/all.js#xfbml=1&appId=596756540369391";
fjs.parentNode.insertBefore(js, fjs);
}(document, 'script', 'facebook-jssdk'));
</script>
<script src="//platform.linkedin.com/in.js" type="text/javascript">lang: en_US</script>
<script type="text/javascript">
(function() {
var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
po.src = 'https://apis.google.com/js/platform.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
})();
</script>
</section>
<aside class="sidebar">
<div class="widget">
<input type="text" placeholder="Search the blog" class="st-default-search-input"></input>
<script type="text/javascript">
(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t);
e=d.getElementsByTagName(t)[0];s.async=0;s.src=u;e.parentNode.insertBefore(s,e);
})(window,document,'script','//s.swiftypecdn.com/install/v2/st.js','_st');
_st('install','_KobMC_zsd_tDx_7NWiX','2.0.0');
</script>
</div>
<div class="widget">
<h4 class="widget-title">Cloudflare blog</h4>
<p style="margin-top: 20px">
<a href="https://www.cloudflare.com/enterprise-service-request" class="btn btn-success" tabindex="11" target="_blank">Contact our team</a>
</p>
<p>
<strong>US callers</strong><br/>
1 (888) 99-FLARE <br/>
<strong>UK callers</strong><br/>
+44 (0)20 3514 6970<br/>
<strong>International callers</strong><br/>
+1 (650) 319-8930 <BR/><BR/>
<a href="https://www.cloudflare.com/plans" target="_blank">Full feature list and plan types</a>
</p>
<p>Cloudflare provides performance and security for any website. More than 6 million websites use Cloudflare.</p>
<p>There is no hardware or software. Cloudflare works at the DNS level. It takes only 5 minutes to sign up. To learn more, please visit our website</p>
</div>
<div class="widget">
<h4 class="widget-title">Cloudflare features</h4>
<ul class="menu menu-sidebar">
<li><a href="https://www.cloudflare.com/">Overview</a></li>
<li><a href="https://www.cloudflare.com/cdn/">CDN</a></li>
<li><a href="https://www.cloudflare.com/website-optimization/">Optimizer</a></li>
<li><a href="https://www.cloudflare.com/security/">Security</a></li>
<li><a href="https://www.cloudflare.com/analytics/">Analytics</a></li>
<li><a href="https://www.cloudflare.com/apps">Apps</a></li>
<li><a href="https://www.cloudflare.com/network/">Network map</a></li>
<li><a href="https://www.cloudflarestatus.com">System status</a></li>
</ul>
</div>
<div id="mc_embed_signup" class="widget">
<form action="https://cloudflare.us5.list-manage.com/subscribe/post?u=d80d4d74266c0c044b0bcd7ca&amp;id=8dc0bf9dea" method="post" id="mc-embedded-subscribe-form" name="mc-embedded-subscribe-form" class="validate" target="_blank" novalidate>
<input type="email" value="" name="EMAIL" class="width-full required email" id="mce-EMAIL" placeholder="Enter your email address"/>
<div id="mce-responses" class="clearfix">
<div class="response" id="mce-error-response" style="display:none"></div>
<div class="response" id="mce-success-response" style="display:none"></div>
</div>
<div class="clearfix">
<button type="submit" name="subscribe" id="mc-embedded-subscribe" class="btn btn-primary width-full">Sign up for email updates</button>
</div>
</form>
</div>
</aside>
</div>
<footer id="footer" class="footer">
<div class="wrapper">
<nav class="navigation footer-nav">
<ul role="navigation">
<li id="cf_nav_menu-2" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">What We Do</h6>
<div class="menu-what-we-do-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/plans">Plans</a></li>
<li><a href="https://www.cloudflare.com/performance/">Performance</a></li>
<li><a href="https://www.cloudflare.com/security/">Security</a></li>
<li><a href="https://www.cloudflare.com/reliability/">Reliability</a></li>
<li><a href="https://www.cloudflare.com/apps">Apps</a></li>
<li><a href="https://www.cloudflare.com/network-map">Network</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-3" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Resources</h6>
<div class="menu-support-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/support">Help Center</a></li>
<li><a href="https://www.cloudflare.com/community">Community</a></li>
<li><a href="https://www.cloudflare.com/video">Video Guides</a></li>
<li><a href="https://www.cloudflarestatus.com">System Status</a></li>
<li><a href="https://www.cloudflare.com/contact">Contact Us</a></li>
<li class="active"><a href="/">Blog</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-4" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Not a Developer?</h6>
<div class="menu-resources-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/case-studies">Case Studies</a></li>
<li><a href="https://www.cloudflare.com/resources/">White Papers</a></li>
<li><a href="https://www.cloudflare.com/internet-summit/">Internet Summit</a></li>
<li><a href="https://www.cloudflare.com/hosting-partners">Partners</a></li>
<li><a href="https://www.cloudflare.com/hosting-partners">Integrations</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-5" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">About Us</h6>
<div class="menu-about-us-container">
<ul class="menu menu-footer">
<li><a href="https://www.cloudflare.com/people">Our Team</a></li>
<li><a href="https://www.cloudflare.com/join-our-team">Careers</a></li>
<li><a href="https://www.cloudflare.com/press-center">Press</a></li>
<li><a href="https://www.cloudflare.com/terms">Terms of Service</a></li>
<li><a href="https://www.cloudflare.com/security-policy/">Privacy &amp; Security</a></li>
<li><a href="https://www.cloudflare.com/abuse/">Trust &amp; Safety</a></li>
</ul>
</div>
</li>
<li id="cf_nav_menu-6" class="footer-column widget_cf_nav_menu">
<h6 class="widget-title">Connect</h6>
<div class="menu-connect-container">
<ul class="menu menu-footer">
<li><a href="http://twitter.com/cloudflare">Twitter</a></li>
<li><a href="https://www.facebook.com/Cloudflare">Facebook</a></li>
<li><a href="https://www.linkedin.com/company/cloudflare-inc-">LinkedIn</a></li>
<li><a href="https://www.youtube.com/cloudflare-">YouTube</a></li>
<li><a href="https://plus.google.com/+cloudflare/posts">Google+</a></li>
<li><a href="/rss/">RSS</a></li>
</ul>
</div>
</li>
</ul>
<div class="credits">All content &copy; 2017 <a href="https://cloudflare.com">Cloudflare</a>. Proudly published with <a href="https://ghost.org">Ghost</a>.</div>
</nav>
</div>
</footer>
<script>
var links = document.links;
for (var i = 0, linksLength = links.length; i < linksLength; i++) {
if (links[i].hostname != window.location.hostname) {
links[i].target = '_blank';
}
}
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/prism.min.js"></script>
<script type="text/javascript" src="/assets/js/jquery.fitvids.js?v=b6cf3f99a6"></script>
<script type="text/javascript">
$(document).ready(function(){ $(".post-content").fitVids(); });
</script>
<script type="text/javascript">
var disqus_shortname = 'cloudflare';
(function () {
var s = document.createElement('script'); s.async = true;
s.type = 'text/javascript';
s.src = '//' + disqus_shortname + '.disqus.com/count.js';
(document.getElementsByTagName('HEAD')[0] || document.getElementsByTagName('BODY')[0]).appendChild(s);
}());
</script>
</body>
</html>

74
h2mux/sample/jquery.fitvids.js
View File

@ -1,74 +0,0 @@
/*global jQuery */
/*jshint multistr:true browser:true */
/*!
* FitVids 1.0.3
*
* Copyright 2013, Chris Coyier - http://css-tricks.com + Dave Rupert - http://daverupert.com
* Credit to Thierry Koblentz - http://www.alistapart.com/articles/creating-intrinsic-ratios-for-video/
* Released under the WTFPL license - http://sam.zoy.org/wtfpl/
*
* Date: Thu Sept 01 18:00:00 2011 -0500
*/
(function( $ ){
"use strict";
$.fn.fitVids = function( options ) {
var settings = {
customSelector: null
};
if(!document.getElementById('fit-vids-style')) {
var div = document.createElement('div'),
ref = document.getElementsByTagName('base')[0] || document.getElementsByTagName('script')[0],
cssStyles = '&shy;<style>.fluid-width-video-wrapper{width:100%;position:relative;padding:0;}.fluid-width-video-wrapper iframe,.fluid-width-video-wrapper object,.fluid-width-video-wrapper embed {position:absolute;top:0;left:0;width:100%;height:100%;}</style>';
div.className = 'fit-vids-style';
div.id = 'fit-vids-style';
div.style.display = 'none';
div.innerHTML = cssStyles;
ref.parentNode.insertBefore(div,ref);
}
if ( options ) {
$.extend( settings, options );
}
return this.each(function(){
var selectors = [
"iframe[src*='player.vimeo.com']",
"iframe[src*='youtube.com']",
"iframe[src*='youtube-nocookie.com']",
"iframe[src*='kickstarter.com'][src*='video.html']",
"object",
"embed"
];
if (settings.customSelector) {
selectors.push(settings.customSelector);
}
var $allVideos = $(this).find(selectors.join(','));
$allVideos = $allVideos.not("object object"); // SwfObj conflict patch
$allVideos.each(function(){
var $this = $(this);
if (this.tagName.toLowerCase() === 'embed' && $this.parent('object').length || $this.parent('.fluid-width-video-wrapper').length) { return; }
var height = ( this.tagName.toLowerCase() === 'object' || ($this.attr('height') && !isNaN(parseInt($this.attr('height'), 10))) ) ? parseInt($this.attr('height'), 10) : $this.height(),
width = !isNaN(parseInt($this.attr('width'), 10)) ? parseInt($this.attr('width'), 10) : $this.width(),
aspectRatio = height / width;
if(!$this.attr('id')){
var videoID = 'fitvid' + Math.floor(Math.random()*999999);
$this.attr('id', videoID);
}
$this.wrap('<div class="fluid-width-video-wrapper"></div>').parent('.fluid-width-video-wrapper').css('padding-top', (aspectRatio * 100)+"%");
$this.removeAttr('height').removeAttr('width');
});
});
};
// Works with either jQuery or Zepto
})( window.jQuery || window.Zepto );

70
h2mux/sample/screen.css
View File

File diff suppressed because one or more lines are too long

67
h2mux/shared_buffer.go
View File

@ -1,67 +0,0 @@
package h2mux
import (
"bytes"
"io"
"sync"
)
type SharedBuffer struct {
cond *sync.Cond
buffer bytes.Buffer
eof bool
}
func NewSharedBuffer() *SharedBuffer {
return &SharedBuffer{
cond: sync.NewCond(&sync.Mutex{}),
}
}
func (s *SharedBuffer) Read(p []byte) (n int, err error) {
totalRead := 0
s.cond.L.Lock()
for totalRead == 0 {
n, err = s.buffer.Read(p[totalRead:])
totalRead += n
if err == io.EOF {
if s.eof {
break
}
err = nil
if n > 0 {
break
}
s.cond.Wait()
}
}
s.cond.L.Unlock()
return totalRead, err
}
func (s *SharedBuffer) Write(p []byte) (n int, err error) {
s.cond.L.Lock()
defer s.cond.L.Unlock()
if s.eof {
return 0, io.EOF
}
n, err = s.buffer.Write(p)
s.cond.Signal()
return
}
func (s *SharedBuffer) Close() error {
s.cond.L.Lock()
defer s.cond.L.Unlock()
if !s.eof {
s.eof = true
s.cond.Signal()
}
return nil
}
func (s *SharedBuffer) Closed() bool {
s.cond.L.Lock()
defer s.cond.L.Unlock()
return s.eof
}

129
h2mux/shared_buffer_test.go
View File

@ -1,129 +0,0 @@
package h2mux
import (
"bytes"
"io"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func AssertIOReturnIsGood(t *testing.T, expected int) func(int, error) {
return func(actual int, err error) {
if expected != actual {
t.Fatalf("Expected %d bytes, got %d", expected, actual)
}
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
}
}
func TestSharedBuffer(t *testing.T) {
b := NewSharedBuffer()
testData := []byte("Hello world")
AssertIOReturnIsGood(t, len(testData))(b.Write(testData))
bytesRead := make([]byte, len(testData))
AssertIOReturnIsGood(t, len(testData))(b.Read(bytesRead))
}
func TestSharedBufferBlockingRead(t *testing.T) {
b := NewSharedBuffer()
testData1 := []byte("Hello")
testData2 := []byte(" world")
result := make(chan []byte)
go func() {
bytesRead := make([]byte, len(testData1)+len(testData2))
nRead, err := b.Read(bytesRead)
AssertIOReturnIsGood(t, len(testData1))(nRead, err)
result <- bytesRead[:nRead]
nRead, err = b.Read(bytesRead)
AssertIOReturnIsGood(t, len(testData2))(nRead, err)
result <- bytesRead[:nRead]
}()
time.Sleep(time.Millisecond * 250)
select {
case <-result:
t.Fatalf("read returned early")
default:
}
AssertIOReturnIsGood(t, len(testData1))(b.Write([]byte(testData1)))
select {
case r := <-result:
assert.Equal(t, testData1, r)
case <-time.After(time.Second):
t.Fatalf("read timed out")
}
AssertIOReturnIsGood(t, len(testData2))(b.Write([]byte(testData2)))
select {
case r := <-result:
assert.Equal(t, testData2, r)
case <-time.After(time.Second):
t.Fatalf("read timed out")
}
}
// This is quite slow under the race detector
func TestSharedBufferConcurrentReadWrite(t *testing.T) {
b := NewSharedBuffer()
var expectedResult, actualResult bytes.Buffer
var wg sync.WaitGroup
wg.Add(2)
go func() {
block := make([]byte, 256)
for i := range block {
block[i] = byte(i)
}
for blockSize := 1; blockSize <= 256; blockSize++ {
for i := 0; i < 256; i++ {
expectedResult.Write(block[:blockSize])
n, err := b.Write(block[:blockSize])
if n != blockSize || err != nil {
t.Errorf("write error: %d %s", n, err)
return
}
}
}
wg.Done()
}()
go func() {
block := make([]byte, 256)
// Change block sizes in opposition to the write thread, to test blocking for new data.
for blockSize := 256; blockSize > 0; blockSize-- {
for i := 0; i < 256; i++ {
n, err := io.ReadFull(b, block[:blockSize])
if n != blockSize || err != nil {
t.Errorf("read error: %d %s", n, err)
return
}
actualResult.Write(block[:blockSize])
}
}
wg.Done()
}()
wg.Wait()
if bytes.Compare(expectedResult.Bytes(), actualResult.Bytes()) != 0 {
t.Fatal("Result diverged")
}
}
func TestSharedBufferClose(t *testing.T) {
b := NewSharedBuffer()
testData := []byte("Hello world")
AssertIOReturnIsGood(t, len(testData))(b.Write(testData))
err := b.Close()
if err != nil {
t.Fatalf("unexpected error from Close: %s", err)
}
bytesRead := make([]byte, len(testData))
AssertIOReturnIsGood(t, len(testData))(b.Read(bytesRead))
n, err := b.Read(bytesRead)
if n != 0 {
t.Fatalf("extra bytes received: %d", n)
}
if err != io.EOF {
t.Fatalf("expected EOF, got %s", err)
}
}

34
h2mux/signal.go
View File

@ -1,34 +0,0 @@
package h2mux
// Signal describes an event that can be waited on for at least one signal.
// Signalling the event while it is in the signalled state is a noop.
// When the waiter wakes up, the signal is set to unsignalled.
// It is a way for any number of writers to inform a reader (without blocking)
// that an event has happened.
type Signal struct {
c chan struct{}
}
// NewSignal creates a new Signal.
func NewSignal() Signal {
return Signal{c: make(chan struct{}, 1)}
}
// Signal signals the event.
func (s Signal) Signal() {
// This channel is buffered, so the nonblocking send will always succeed if the buffer is empty.
select {
case s.c <- struct{}{}:
default:
}
}
// Wait for the event to be signalled.
func (s Signal) Wait() {
<-s.c
}
// WaitChannel returns a channel that is readable after Signal is called.
func (s Signal) WaitChannel() <-chan struct{} {
return s.c
}

47
h2mux/streamerrormap.go
View File

@ -1,47 +0,0 @@
package h2mux
import (
"sync"
"golang.org/x/net/http2"
)
// StreamErrorMap is used to track stream errors. This is a separate structure to ActiveStreamMap because
// errors can be raised against non-existent or closed streams.
type StreamErrorMap struct {
sync.RWMutex
// errors tracks per-stream errors
errors map[uint32]http2.ErrCode
// hasError is signaled whenever an error is raised.
hasError Signal
}
// NewStreamErrorMap creates a new StreamErrorMap.
func NewStreamErrorMap() *StreamErrorMap {
return &StreamErrorMap{
errors: make(map[uint32]http2.ErrCode),
hasError: NewSignal(),
}
}
// RaiseError raises a stream error.
func (s *StreamErrorMap) RaiseError(streamID uint32, err http2.ErrCode) {
s.Lock()
s.errors[streamID] = err
s.Unlock()
s.hasError.Signal()
}
// GetSignalChan returns a channel that is signalled when an error is raised.
func (s *StreamErrorMap) GetSignalChan() <-chan struct{} {
return s.hasError.WaitChannel()
}
// GetErrors retrieves all errors currently raised. This resets the currently-tracked errors.
func (s *StreamErrorMap) GetErrors() map[uint32]http2.ErrCode {
s.Lock()
errors := s.errors
s.errors = make(map[uint32]http2.ErrCode)
s.Unlock()
return errors
}

7
proxy/logger.go
View File

@ -16,11 +16,14 @@ const (
logFieldLBProbe = "lbProbe"
logFieldRule = "ingressRule"
logFieldOriginService = "originService"
logFieldFlowID = "flowID"
logFieldConnIndex = "connIndex"
logFieldDestAddr = "destAddr"
)
var (
LogFieldFlowID = "flowID"
)
// newHTTPLogger creates a child zerolog.Logger from the provided with added context from the HTTP request, ingress
// services, and connection index.
func newHTTPLogger(logger *zerolog.Logger, connIndex uint8, req *http.Request, rule int, serviceName string) zerolog.Logger {
@ -47,7 +50,7 @@ func newTCPLogger(logger *zerolog.Logger, req *connection.TCPRequest) zerolog.Lo
Int(management.EventTypeKey, int(management.TCP)).
Uint8(logFieldConnIndex, req.ConnIndex).
Str(logFieldOriginService, ingress.ServiceWarpRouting).
Str(logFieldFlowID, req.FlowID).
Str(LogFieldFlowID, req.FlowID).
Str(logFieldDestAddr, req.Dest).
Uint8(logFieldConnIndex, req.ConnIndex).
Logger()

372
quic/v3/datagram.go Normal file
View File

@ -0,0 +1,372 @@
package v3
import (
"encoding/binary"
"net/netip"
"time"
)
type DatagramType byte
const (
// UDP Registration
UDPSessionRegistrationType DatagramType = 0x0
// UDP Session Payload
UDPSessionPayloadType DatagramType = 0x1
// DatagramTypeICMP (supporting both ICMPv4 and ICMPv6)
ICMPType DatagramType = 0x2
// UDP Session Registration Response
UDPSessionRegistrationResponseType DatagramType = 0x3
)
const (
// Total number of bytes representing the [DatagramType]
datagramTypeLen = 1
// 1280 is the default datagram packet length used before MTU discovery: https://github.com/quic-go/quic-go/blob/v0.45.0/internal/protocol/params.go#L12
maxDatagramLen = 1280
)
func parseDatagramType(data []byte) (DatagramType, error) {
if len(data) < datagramTypeLen {
return 0, ErrDatagramHeaderTooSmall
}
return DatagramType(data[0]), nil
}
// UDPSessionRegistrationDatagram handles a request to initialize a UDP session on the remote client.
type UDPSessionRegistrationDatagram struct {
RequestID RequestID
Dest netip.AddrPort
Traced bool
IdleDurationHint time.Duration
Payload []byte
}
const (
sessionRegistrationFlagsIPMask byte = 0b0000_0001
sessionRegistrationFlagsTracedMask byte = 0b0000_0010
sessionRegistrationFlagsBundledMask byte = 0b0000_0100
sessionRegistrationIPv4DatagramHeaderLen = datagramTypeLen +
1 + // Flag length
2 + // Destination port length
2 + // Idle duration seconds length
datagramRequestIdLen + // Request ID length
4 // IPv4 address length
// The IPv4 and IPv6 address share space, so adding 12 to the header length gets the space taken by the IPv6 field.
sessionRegistrationIPv6DatagramHeaderLen = sessionRegistrationIPv4DatagramHeaderLen + 12
)
// The datagram structure for UDPSessionRegistrationDatagram is:
//
// 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// 0| Type | Flags | Destination Port |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// 4| Idle Duration Seconds | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +
// 8| |
// + Session Identifier +
// 12| (16 Bytes) |
// + +
// 16| |
// + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// 20| | Destination IPv4 Address |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- - - - - - - - - - - - - - - -+
// 24| Destination IPv4 Address cont | |
// +- - - - - - - - - - - - - - - - +
// 28| Destination IPv6 Address |
// + (extension of IPv4 region) +
// 32| |
// + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// 36| | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +
// . .
// . Bundle Payload .
// . .
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
func (s *UDPSessionRegistrationDatagram) MarshalBinary() (data []byte, err error) {
ipv6 := s.Dest.Addr().Is6()
var flags byte
if s.Traced {
flags |= sessionRegistrationFlagsTracedMask
}
hasPayload := len(s.Payload) > 0
if hasPayload {
flags |= sessionRegistrationFlagsBundledMask
}
var maxPayloadLen int
if ipv6 {
maxPayloadLen = maxDatagramLen - sessionRegistrationIPv6DatagramHeaderLen
flags |= sessionRegistrationFlagsIPMask
} else {
maxPayloadLen = maxDatagramLen - sessionRegistrationIPv4DatagramHeaderLen
}
// Make sure that the payload being bundled can actually fit in the payload destination
if len(s.Payload) > maxPayloadLen {
return nil, wrapMarshalErr(ErrDatagramPayloadTooLarge)
}
// Allocate the buffer with the right size for the destination IP family
if ipv6 {
data = make([]byte, sessionRegistrationIPv6DatagramHeaderLen+len(s.Payload))
} else {
data = make([]byte, sessionRegistrationIPv4DatagramHeaderLen+len(s.Payload))
}
data[0] = byte(UDPSessionRegistrationType)
data[1] = byte(flags)
binary.BigEndian.PutUint16(data[2:4], s.Dest.Port())
binary.BigEndian.PutUint16(data[4:6], uint16(s.IdleDurationHint.Seconds()))
err = s.RequestID.MarshalBinaryTo(data[6:22])
if err != nil {
return nil, wrapMarshalErr(err)
}
var end int
if ipv6 {
copy(data[22:38], s.Dest.Addr().AsSlice())
end = 38
} else {
copy(data[22:26], s.Dest.Addr().AsSlice())
end = 26
}
if hasPayload {
copy(data[end:], s.Payload)
}
return data, nil
}
func (s *UDPSessionRegistrationDatagram) UnmarshalBinary(data []byte) error {
datagramType, err := parseDatagramType(data)
if err != nil {
return err
}
if datagramType != UDPSessionRegistrationType {
return wrapUnmarshalErr(ErrInvalidDatagramType)
}
requestID, err := RequestIDFromSlice(data[6:22])
if err != nil {
return wrapUnmarshalErr(err)
}
traced := (data[1] & sessionRegistrationFlagsTracedMask) == sessionRegistrationFlagsTracedMask
bundled := (data[1] & sessionRegistrationFlagsBundledMask) == sessionRegistrationFlagsBundledMask
ipv6 := (data[1] & sessionRegistrationFlagsIPMask) == sessionRegistrationFlagsIPMask
port := binary.BigEndian.Uint16(data[2:4])
var datagramHeaderSize int
var dest netip.AddrPort
if ipv6 {
datagramHeaderSize = sessionRegistrationIPv6DatagramHeaderLen
dest = netip.AddrPortFrom(netip.AddrFrom16([16]byte(data[22:38])), port)
} else {
datagramHeaderSize = sessionRegistrationIPv4DatagramHeaderLen
dest = netip.AddrPortFrom(netip.AddrFrom4([4]byte(data[22:26])), port)
}
idle := time.Duration(binary.BigEndian.Uint16(data[4:6])) * time.Second
var payload []byte
if bundled && len(data) >= datagramHeaderSize && len(data[datagramHeaderSize:]) > 0 {
payload = data[datagramHeaderSize:]
}
*s = UDPSessionRegistrationDatagram{
RequestID: requestID,
Dest: dest,
Traced: traced,
IdleDurationHint: idle,
Payload: payload,
}
return nil
}
// UDPSessionPayloadDatagram provides the payload for a session to be send to either the origin or the client.
type UDPSessionPayloadDatagram struct {
RequestID RequestID
Payload []byte
}
const (
datagramPayloadHeaderLen = datagramTypeLen + datagramRequestIdLen
// The maximum size that a proxied UDP payload can be in a [UDPSessionPayloadDatagram]
maxPayloadPlusHeaderLen = maxDatagramLen - datagramPayloadHeaderLen
)
// The datagram structure for UDPSessionPayloadDatagram is:
//
// 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// 0| Type | |
// +-+-+-+-+-+-+-+-+ +
// 4| |
// + +
// 8| Session Identifier |
// + (16 Bytes) +
// 12| |
// + +-+-+-+-+-+-+-+-+
// 16| | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +
// . .
// . Payload .
// . .
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// MarshalPayloadHeaderTo provides a way to insert the Session Payload header into an already existing byte slice
// without having to allocate and copy the payload into the destination.
//
// This method should be used in-place of MarshalBinary which will allocate in-place the required byte array to return.
func MarshalPayloadHeaderTo(requestID RequestID, payload []byte) error {
if len(payload) < 17 {
return wrapMarshalErr(ErrDatagramPayloadHeaderTooSmall)
}
payload[0] = byte(UDPSessionPayloadType)
return requestID.MarshalBinaryTo(payload[1:17])
}
func (s *UDPSessionPayloadDatagram) UnmarshalBinary(data []byte) error {
datagramType, err := parseDatagramType(data)
if err != nil {
return err
}
if datagramType != UDPSessionPayloadType {
return wrapUnmarshalErr(ErrInvalidDatagramType)
}
// Make sure that the slice provided is the right size to be parsed.
if len(data) < 17 || len(data) > maxPayloadPlusHeaderLen {
return wrapUnmarshalErr(ErrDatagramPayloadInvalidSize)
}
requestID, err := RequestIDFromSlice(data[1:17])
if err != nil {
return wrapUnmarshalErr(err)
}
*s = UDPSessionPayloadDatagram{
RequestID: requestID,
Payload: data[17:],
}
return nil
}
// UDPSessionRegistrationResponseDatagram is used to either return a successful registration or error to the client
// that requested the registration of a UDP session.
type UDPSessionRegistrationResponseDatagram struct {
RequestID RequestID
ResponseType SessionRegistrationResp
ErrorMsg string
}
const (
datagramRespTypeLen = 1
datagramRespErrMsgLen = 2
datagramSessionRegistrationResponseLen = datagramTypeLen + datagramRespTypeLen + datagramRequestIdLen + datagramRespErrMsgLen
// The maximum size that an error message can be in a [UDPSessionRegistrationResponseDatagram].
maxResponseErrorMessageLen = maxDatagramLen - datagramSessionRegistrationResponseLen
)
// SessionRegistrationResp represents all of the responses that a UDP session registration response
// can return back to the client.
type SessionRegistrationResp byte
const (
// Session was received and is ready to proxy.
ResponseOk SessionRegistrationResp = 0x00
// Session registration was unable to reach the requested origin destination.
ResponseDestinationUnreachable SessionRegistrationResp = 0x01
// Session registration was unable to bind to a local UDP socket.
ResponseUnableToBindSocket SessionRegistrationResp = 0x02
// Session registration failed with an unexpected error but provided a message.
ResponseErrorWithMsg SessionRegistrationResp = 0xff
)
// The datagram structure for UDPSessionRegistrationResponseDatagram is:
//
// 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// 0| Type | Resp Type | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +
// 4| |
// + Session Identifier +
// 8| (16 Bytes) |
// + +
// 12| |
// + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// 16| | Error Length |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// . .
// . .
// . .
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
func (s *UDPSessionRegistrationResponseDatagram) MarshalBinary() (data []byte, err error) {
if len(s.ErrorMsg) > maxResponseErrorMessageLen {
return nil, wrapMarshalErr(ErrDatagramResponseMsgInvalidSize)
}
errMsgLen := uint16(len(s.ErrorMsg))
data = make([]byte, datagramSessionRegistrationResponseLen+errMsgLen)
data[0] = byte(UDPSessionRegistrationResponseType)
data[1] = byte(s.ResponseType)
err = s.RequestID.MarshalBinaryTo(data[2:18])
if err != nil {
return nil, wrapMarshalErr(err)
}
if errMsgLen > 0 {
binary.BigEndian.PutUint16(data[18:20], errMsgLen)
copy(data[20:], []byte(s.ErrorMsg))
}
return data, nil
}
func (s *UDPSessionRegistrationResponseDatagram) UnmarshalBinary(data []byte) error {
datagramType, err := parseDatagramType(data)
if err != nil {
return wrapUnmarshalErr(err)
}
if datagramType != UDPSessionRegistrationResponseType {
return wrapUnmarshalErr(ErrInvalidDatagramType)
}
if len(data) < datagramSessionRegistrationResponseLen {
return wrapUnmarshalErr(ErrDatagramResponseInvalidSize)
}
respType := SessionRegistrationResp(data[1])
requestID, err := RequestIDFromSlice(data[2:18])
if err != nil {
return wrapUnmarshalErr(err)
}
errMsgLen := binary.BigEndian.Uint16(data[18:20])
if errMsgLen > maxResponseErrorMessageLen {
return wrapUnmarshalErr(ErrDatagramResponseMsgTooLargeMaximum)
}
if len(data[20:]) < int(errMsgLen) {
return wrapUnmarshalErr(ErrDatagramResponseMsgTooLargeDatagram)
}
var errMsg string
if errMsgLen > 0 {
errMsg = string(data[20:])
}
*s = UDPSessionRegistrationResponseDatagram{
RequestID: requestID,
ResponseType: respType,
ErrorMsg: errMsg,
}
return nil
}

26
quic/v3/datagram_errors.go Normal file
View File

@ -0,0 +1,26 @@
package v3
import (
"errors"
"fmt"
)
var (
ErrInvalidDatagramType error = errors.New("invalid datagram type expected")
ErrDatagramHeaderTooSmall error = fmt.Errorf("datagram should have at least %d bytes", datagramTypeLen)
ErrDatagramPayloadTooLarge error = errors.New("payload length is too large to be bundled in datagram")
ErrDatagramPayloadHeaderTooSmall error = errors.New("payload length is too small to fit the datagram header")
ErrDatagramPayloadInvalidSize error = errors.New("datagram provided is an invalid size")
ErrDatagramResponseMsgInvalidSize error = errors.New("datagram response message is an invalid size")
ErrDatagramResponseInvalidSize error = errors.New("datagram response is an invalid size")
ErrDatagramResponseMsgTooLargeMaximum error = fmt.Errorf("datagram response error message length exceeds the length of the datagram maximum: %d", maxResponseErrorMessageLen)
ErrDatagramResponseMsgTooLargeDatagram error = fmt.Errorf("datagram response error message length exceeds the length of the provided datagram")
)
func wrapMarshalErr(err error) error {
return fmt.Errorf("datagram marshal error: %w", err)
}
func wrapUnmarshalErr(err error) error {
return fmt.Errorf("datagram unmarshal error: %w", err)
}

352
quic/v3/datagram_test.go Normal file
View File

@ -0,0 +1,352 @@
package v3_test
import (
"encoding/binary"
"errors"
"net/netip"
"testing"
"time"
"github.com/stretchr/testify/require"
v3 "github.com/cloudflare/cloudflared/quic/v3"
)
func makePayload(size int) []byte {
payload := make([]byte, size)
for i := range len(payload) {
payload[i] = 0xfc
}
return payload
}
func TestSessionRegistration_MarshalUnmarshal(t *testing.T) {
payload := makePayload(1254)
tests := []*v3.UDPSessionRegistrationDatagram{
// Default (IPv4)
{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 5 * time.Second,
Payload: nil,
},
// Request ID (max)
{
RequestID: mustRequestID([16]byte{
^uint8(0), ^uint8(0), ^uint8(0), ^uint8(0),
^uint8(0), ^uint8(0), ^uint8(0), ^uint8(0),
^uint8(0), ^uint8(0), ^uint8(0), ^uint8(0),
^uint8(0), ^uint8(0), ^uint8(0), ^uint8(0),
}),
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 5 * time.Second,
Payload: nil,
},
// IPv6
{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("[fc00::0]:8080"),
Traced: false,
IdleDurationHint: 5 * time.Second,
Payload: nil,
},
// Traced
{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: true,
IdleDurationHint: 5 * time.Second,
Payload: nil,
},
// IdleDurationHint (max)
{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 65535 * time.Second,
Payload: nil,
},
// Payload
{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 5 * time.Second,
Payload: []byte{0xff, 0xaa, 0xcc, 0x44},
},
// Payload (max: 1254) for IPv4
{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 5 * time.Second,
Payload: payload,
},
// Payload (max: 1242) for IPv4
{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 5 * time.Second,
Payload: payload[:1242],
},
}
for _, tt := range tests {
marshaled, err := tt.MarshalBinary()
if err != nil {
t.Error(err)
}
unmarshaled := v3.UDPSessionRegistrationDatagram{}
err = unmarshaled.UnmarshalBinary(marshaled)
if err != nil {
t.Error(err)
}
if !compareRegistrationDatagrams(t, tt, &unmarshaled) {
t.Errorf("not equal:\n%+v\n%+v", tt, &unmarshaled)
}
}
}
func TestSessionRegistration_MarshalBinary(t *testing.T) {
t.Run("idle hint too large", func(t *testing.T) {
// idle hint duration overflows back to 1
datagram := &v3.UDPSessionRegistrationDatagram{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 65537 * time.Second,
Payload: nil,
}
expected := &v3.UDPSessionRegistrationDatagram{
RequestID: testRequestID,
Dest: netip.MustParseAddrPort("1.1.1.1:8080"),
Traced: false,
IdleDurationHint: 1 * time.Second,
Payload: nil,
}
marshaled, err := datagram.MarshalBinary()
if err != nil {
t.Error(err)
}
unmarshaled := v3.UDPSessionRegistrationDatagram{}
err = unmarshaled.UnmarshalBinary(marshaled)
if err != nil {
t.Error(err)
}
if !compareRegistrationDatagrams(t, expected, &unmarshaled) {
t.Errorf("not equal:\n%+v\n%+v", expected, &unmarshaled)
}
})
}
func TestTypeUnmarshalErrors(t *testing.T) {
t.Run("invalid length", func(t *testing.T) {
d1 := v3.UDPSessionRegistrationDatagram{}
err := d1.UnmarshalBinary([]byte{})
if !errors.Is(err, v3.ErrDatagramHeaderTooSmall) {
t.Errorf("expected invalid length to throw error")
}
d2 := v3.UDPSessionPayloadDatagram{}
err = d2.UnmarshalBinary([]byte{})
if !errors.Is(err, v3.ErrDatagramHeaderTooSmall) {
t.Errorf("expected invalid length to throw error")
}
d3 := v3.UDPSessionRegistrationResponseDatagram{}
err = d3.UnmarshalBinary([]byte{})
if !errors.Is(err, v3.ErrDatagramHeaderTooSmall) {
t.Errorf("expected invalid length to throw error")
}
})
t.Run("invalid types", func(t *testing.T) {
d1 := v3.UDPSessionRegistrationDatagram{}
err := d1.UnmarshalBinary([]byte{byte(v3.UDPSessionRegistrationResponseType)})
if !errors.Is(err, v3.ErrInvalidDatagramType) {
t.Errorf("expected invalid type to throw error")
}
d2 := v3.UDPSessionPayloadDatagram{}
err = d2.UnmarshalBinary([]byte{byte(v3.UDPSessionRegistrationType)})
if !errors.Is(err, v3.ErrInvalidDatagramType) {
t.Errorf("expected invalid type to throw error")
}
d3 := v3.UDPSessionRegistrationResponseDatagram{}
err = d3.UnmarshalBinary([]byte{byte(v3.UDPSessionPayloadType)})
if !errors.Is(err, v3.ErrInvalidDatagramType) {
t.Errorf("expected invalid type to throw error")
}
})
}
func TestSessionPayload(t *testing.T) {
t.Run("basic", func(t *testing.T) {
payload := makePayload(128)
err := v3.MarshalPayloadHeaderTo(testRequestID, payload[0:17])
if err != nil {
t.Error(err)
}
unmarshaled := v3.UDPSessionPayloadDatagram{}
err = unmarshaled.UnmarshalBinary(payload)
if err != nil {
t.Error(err)
}
require.Equal(t, testRequestID, unmarshaled.RequestID)
require.Equal(t, payload[17:], unmarshaled.Payload)
})
t.Run("empty", func(t *testing.T) {
payload := makePayload(17)
err := v3.MarshalPayloadHeaderTo(testRequestID, payload)
if err != nil {
t.Error(err)
}
unmarshaled := v3.UDPSessionPayloadDatagram{}
err = unmarshaled.UnmarshalBinary(payload)
if err != nil {
t.Error(err)
}
require.Equal(t, testRequestID, unmarshaled.RequestID)
require.Equal(t, payload[17:], unmarshaled.Payload)
})
t.Run("header size too small", func(t *testing.T) {
payload := makePayload(16)
err := v3.MarshalPayloadHeaderTo(testRequestID, payload)
if !errors.Is(err, v3.ErrDatagramPayloadHeaderTooSmall) {
t.Errorf("expected an error")
}
})
t.Run("payload size too small", func(t *testing.T) {
payload := makePayload(17)
err := v3.MarshalPayloadHeaderTo(testRequestID, payload)
if err != nil {
t.Error(err)
}
unmarshaled := v3.UDPSessionPayloadDatagram{}
err = unmarshaled.UnmarshalBinary(payload[:16])
if !errors.Is(err, v3.ErrDatagramPayloadInvalidSize) {
t.Errorf("expected an error: %s", err)
}
})
t.Run("payload size too large", func(t *testing.T) {
datagram := makePayload(17 + 1264) // 1263 is the largest payload size allowed
err := v3.MarshalPayloadHeaderTo(testRequestID, datagram)
if err != nil {
t.Error(err)
}
unmarshaled := v3.UDPSessionPayloadDatagram{}
err = unmarshaled.UnmarshalBinary(datagram[:])
if !errors.Is(err, v3.ErrDatagramPayloadInvalidSize) {
t.Errorf("expected an error: %s", err)
}
})
}
func TestSessionRegistrationResponse(t *testing.T) {
validRespTypes := []v3.SessionRegistrationResp{
v3.ResponseOk,
v3.ResponseDestinationUnreachable,
v3.ResponseUnableToBindSocket,
v3.ResponseErrorWithMsg,
}
t.Run("basic", func(t *testing.T) {
for _, responseType := range validRespTypes {
datagram := &v3.UDPSessionRegistrationResponseDatagram{
RequestID: testRequestID,
ResponseType: responseType,
ErrorMsg: "test",
}
marshaled, err := datagram.MarshalBinary()
if err != nil {
t.Error(err)
}
unmarshaled := &v3.UDPSessionRegistrationResponseDatagram{}
err = unmarshaled.UnmarshalBinary(marshaled)
if err != nil {
t.Error(err)
}
require.Equal(t, datagram, unmarshaled)
}
})
t.Run("unsupported resp type is valid", func(t *testing.T) {
datagram := &v3.UDPSessionRegistrationResponseDatagram{
RequestID: testRequestID,
ResponseType: v3.SessionRegistrationResp(0xfc),
ErrorMsg: "",
}
marshaled, err := datagram.MarshalBinary()
if err != nil {
t.Error(err)
}
unmarshaled := &v3.UDPSessionRegistrationResponseDatagram{}
err = unmarshaled.UnmarshalBinary(marshaled)
if err != nil {
t.Error(err)
}
require.Equal(t, datagram, unmarshaled)
})
t.Run("too small to unmarshal", func(t *testing.T) {
payload := makePayload(17)
payload[0] = byte(v3.UDPSessionRegistrationResponseType)
unmarshaled := &v3.UDPSessionRegistrationResponseDatagram{}
err := unmarshaled.UnmarshalBinary(payload)
if !errors.Is(err, v3.ErrDatagramResponseInvalidSize) {
t.Errorf("expected an error")
}
})
t.Run("error message too long", func(t *testing.T) {
message := ""
for i := 0; i < 1280; i++ {
message += "a"
}
datagram := &v3.UDPSessionRegistrationResponseDatagram{
RequestID: testRequestID,
ResponseType: v3.SessionRegistrationResp(0xfc),
ErrorMsg: message,
}
_, err := datagram.MarshalBinary()
if !errors.Is(err, v3.ErrDatagramResponseMsgInvalidSize) {
t.Errorf("expected an error")
}
})
t.Run("error message too large to unmarshal", func(t *testing.T) {
payload := makePayload(1280)
payload[0] = byte(v3.UDPSessionRegistrationResponseType)
binary.BigEndian.PutUint16(payload[18:20], 1280) // larger than the datagram size could be
unmarshaled := &v3.UDPSessionRegistrationResponseDatagram{}
err := unmarshaled.UnmarshalBinary(payload)
if !errors.Is(err, v3.ErrDatagramResponseMsgTooLargeMaximum) {
t.Errorf("expected an error: %v", err)
}
})
t.Run("error message larger than provided buffer", func(t *testing.T) {
payload := makePayload(1000)
payload[0] = byte(v3.UDPSessionRegistrationResponseType)
binary.BigEndian.PutUint16(payload[18:20], 1001) // larger than the datagram size provided
unmarshaled := &v3.UDPSessionRegistrationResponseDatagram{}
err := unmarshaled.UnmarshalBinary(payload)
if !errors.Is(err, v3.ErrDatagramResponseMsgTooLargeDatagram) {
t.Errorf("expected an error: %v", err)
}
})
}
func compareRegistrationDatagrams(t *testing.T, l *v3.UDPSessionRegistrationDatagram, r *v3.UDPSessionRegistrationDatagram) bool {
require.Equal(t, l.Payload, r.Payload)
return l.RequestID == r.RequestID &&
l.Dest == r.Dest &&
l.IdleDurationHint == r.IdleDurationHint &&
l.Traced == r.Traced
}

72
quic/v3/request.go Normal file
View File

@ -0,0 +1,72 @@
package v3
import (
"encoding/binary"
"errors"
)
const (
datagramRequestIdLen = 16
)
var (
// ErrInvalidRequestIDLen is returned when the provided request id can not be parsed from the provided byte slice.
ErrInvalidRequestIDLen error = errors.New("invalid request id length provided")
// ErrInvalidPayloadDestLen is returned when the provided destination byte slice cannot fit the whole request id.
ErrInvalidPayloadDestLen error = errors.New("invalid payload size provided")
)
// RequestID is the request-id-v2 identifier, it is used to distinguish between specific flows or sessions proxied
// from the edge to cloudflared.
type RequestID uint128
type uint128 struct {
hi uint64
lo uint64
}
// RequestIDFromSlice reads a request ID from a byte slice.
func RequestIDFromSlice(data []byte) (RequestID, error) {
if len(data) != datagramRequestIdLen {
return RequestID{}, ErrInvalidRequestIDLen
}
return RequestID{
hi: binary.BigEndian.Uint64(data[:8]),
lo: binary.BigEndian.Uint64(data[8:]),
}, nil
}
// Compare returns an integer comparing two IPs.
// The result will be 0 if id == id2, -1 if id < id2, and +1 if id > id2.
// The definition of "less than" is the same as the [RequestID.Less] method.
func (id RequestID) Compare(id2 RequestID) int {
hi1, hi2 := id.hi, id2.hi
if hi1 < hi2 {
return -1
}
if hi1 > hi2 {
return 1
}
lo1, lo2 := id.lo, id2.lo
if lo1 < lo2 {
return -1
}
if lo1 > lo2 {
return 1
}
return 0
}
// Less reports whether id sorts before id2.
func (id RequestID) Less(id2 RequestID) bool { return id.Compare(id2) == -1 }
// MarshalBinaryTo writes the id to the provided destination byte slice; the byte slice must be of at least size 16.
func (id RequestID) MarshalBinaryTo(data []byte) error {
if len(data) < datagramRequestIdLen {
return ErrInvalidPayloadDestLen
}
binary.BigEndian.PutUint64(data[:8], id.hi)
binary.BigEndian.PutUint64(data[8:], id.lo)
return nil
}

50
quic/v3/request_test.go Normal file
View File

@ -0,0 +1,50 @@
package v3_test
import (
"crypto/rand"
"slices"
"testing"
v3 "github.com/cloudflare/cloudflared/quic/v3"
)
var (
testRequestIDBytes = [16]byte{
0x00, 0x11, 0x22, 0x33,
0x44, 0x55, 0x66, 0x77,
0x88, 0x99, 0xaa, 0xbb,
0xcc, 0xdd, 0xee, 0xff,
}
testRequestID = mustRequestID(testRequestIDBytes)
)
func mustRequestID(data [16]byte) v3.RequestID {
id, err := v3.RequestIDFromSlice(data[:])
if err != nil {
panic(err)
}
return id
}
func TestRequestIDParsing(t *testing.T) {
buf1 := make([]byte, 16)
n, err := rand.Read(buf1)
if err != nil {
t.Fatal(err)
}
if n != 16 {
t.Fatalf("did not read 16 bytes: %d", n)
}
id, err := v3.RequestIDFromSlice(buf1)
if err != nil {
t.Fatal(err)
}
buf2 := make([]byte, 16)
err = id.MarshalBinaryTo(buf2)
if err != nil {
t.Fatal(err)
}
if !slices.Equal(buf1, buf2) {
t.Fatalf("buf1 != buf2: %+v %+v", buf1, buf2)
}
}

16
h2mux/booleanfuse.go → supervisor/fuse.go
View File

@ -1,23 +1,23 @@
package h2mux
package supervisor
import "sync"
// BooleanFuse is a data structure that can be set once to a particular value using Fuse(value).
// booleanFuse is a data structure that can be set once to a particular value using Fuse(value).
// Subsequent calls to Fuse() will have no effect.
type BooleanFuse struct {
type booleanFuse struct {
value int32
mu sync.Mutex
cond *sync.Cond
}
func NewBooleanFuse() *BooleanFuse {
f := &BooleanFuse{}
func newBooleanFuse() *booleanFuse {
f := &booleanFuse{}
f.cond = sync.NewCond(&f.mu)
return f
}
// Value gets the value
func (f *BooleanFuse) Value() bool {
func (f *booleanFuse) Value() bool {
// 0: unset
// 1: set true
// 2: set false
@ -26,7 +26,7 @@ func (f *BooleanFuse) Value() bool {
return f.value == 1
}
func (f *BooleanFuse) Fuse(result bool) {
func (f *booleanFuse) Fuse(result bool) {
f.mu.Lock()
defer f.mu.Unlock()
newValue := int32(2)
@ -40,7 +40,7 @@ func (f *BooleanFuse) Fuse(result bool) {
}
// Await blocks until Fuse has been called at least once.
func (f *BooleanFuse) Await() bool {
func (f *booleanFuse) Await() bool {
f.mu.Lock()
defer f.mu.Unlock()
for f.value == 0 {

67
supervisor/tunnel.go
View File

@ -5,6 +5,7 @@ import (
"crypto/tls"
"fmt"
"net"
"net/netip"
"runtime/debug"
"strings"
"sync"
@ -19,7 +20,6 @@ import (
"github.com/cloudflare/cloudflared/edgediscovery"
"github.com/cloudflare/cloudflared/edgediscovery/allregions"
"github.com/cloudflare/cloudflared/features"
"github.com/cloudflare/cloudflared/h2mux"
"github.com/cloudflare/cloudflared/ingress"
"github.com/cloudflare/cloudflared/management"
"github.com/cloudflare/cloudflared/orchestration"
@ -199,7 +199,7 @@ func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolF
haConnections.Inc()
defer haConnections.Dec()
connectedFuse := h2mux.NewBooleanFuse()
connectedFuse := newBooleanFuse()
go func() {
if connectedFuse.Await() {
connectedSignal.Notify()
@ -375,7 +375,7 @@ func (e *EdgeTunnelServer) serveTunnel(
connLog *ConnAwareLogger,
addr *allregions.EdgeAddr,
connIndex uint8,
fuse *h2mux.BooleanFuse,
fuse *booleanFuse,
backoff *protocolFallback,
protocol connection.Protocol,
) (err error, recoverable bool) {
@ -441,7 +441,7 @@ func (e *EdgeTunnelServer) serveConnection(
connLog *ConnAwareLogger,
addr *allregions.EdgeAddr,
connIndex uint8,
fuse *h2mux.BooleanFuse,
fuse *booleanFuse,
backoff *protocolFallback,
protocol connection.Protocol,
) (err error, recoverable bool) {
@ -466,7 +466,7 @@ func (e *EdgeTunnelServer) serveConnection(
case connection.QUIC:
connOptions := e.config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
return e.serveQUIC(ctx,
addr.UDP,
addr.UDP.AddrPort(),
connLog,
connOptions,
controlStream,
@ -549,7 +549,7 @@ func (e *EdgeTunnelServer) serveHTTP2(
func (e *EdgeTunnelServer) serveQUIC(
ctx context.Context,
edgeAddr *net.UDPAddr,
edgeAddr netip.AddrPort,
connLogger *ConnAwareLogger,
connOptions *pogs.ConnectionOptions,
controlStreamHandler connection.ControlStreamHandler,
@ -572,7 +572,7 @@ func (e *EdgeTunnelServer) serveQUIC(
// quic-go 0.44 increases the initial packet size to 1280 by default. That breaks anyone running tunnel through WARP
// because WARP MTU is 1280.
var initialPacketSize uint16 = 1252
if edgeAddr.IP.To4() == nil {
if edgeAddr.Addr().Is4() {
initialPacketSize = 1232
}
@ -590,32 +590,55 @@ func (e *EdgeTunnelServer) serveQUIC(
InitialPacketSize: initialPacketSize,
}
quicConn, err := connection.NewQUICConnection(
// Dial the QUIC connection to the edge
conn, err := connection.DialQuic(
ctx,
quicConfig,
tlsConfig,
edgeAddr,
e.edgeBindAddr,
connIndex,
tlsConfig,
e.orchestrator,
connOptions,
controlStreamHandler,
connLogger.Logger(),
e.config.PacketConfig,
e.config.RPCTimeout,
e.config.WriteStreamTimeout,
e.config.GracePeriod,
)
if err != nil {
connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")
connLogger.ConnAwareLogger().Err(err).Msgf("Failed to dial a quic connection")
return err, true
}
datagramSessionManager := connection.NewDatagramV2Connection(
ctx,
conn,
e.config.PacketConfig,
e.config.RPCTimeout,
e.config.WriteStreamTimeout,
connLogger.Logger(),
)
// Wrap the [quic.Connection] as a TunnelConnection
tunnelConn, err := connection.NewTunnelConnection(
ctx,
conn,
connIndex,
e.orchestrator,
datagramSessionManager,
controlStreamHandler,
connOptions,
e.config.RPCTimeout,
e.config.WriteStreamTimeout,
e.config.GracePeriod,
connLogger.Logger(),
)
if err != nil {
connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new tunnel connection")
return err, true
}
// Serve the TunnelConnection
errGroup, serveCtx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
err := quicConn.Serve(serveCtx)
err := tunnelConn.Serve(serveCtx)
if err != nil {
connLogger.ConnAwareLogger().Err(err).Msg("Failed to serve quic connection")
connLogger.ConnAwareLogger().Err(err).Msg("Failed to serve tunnel connection")
}
return err
})
@ -624,8 +647,8 @@ func (e *EdgeTunnelServer) serveQUIC(
err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
if err != nil {
// forcefully break the connection (this is only used for testing)
// errgroup will return context canceled for the quicConn.Serve
connLogger.Logger().Debug().Msg("Forcefully breaking quic connection")
// errgroup will return context canceled for the tunnelConn.Serve
connLogger.Logger().Debug().Msg("Forcefully breaking tunnel connection")
}
return err
})
@ -645,7 +668,7 @@ func listenReconnect(ctx context.Context, reconnectCh <-chan ReconnectSignal, gr
}
type connectedFuse struct {
fuse *h2mux.BooleanFuse
fuse *booleanFuse
backoff *protocolFallback
}

6
vendor/github.com/coredns/coredns/core/dnsserver/config.go generated vendored
View File

@ -5,6 +5,7 @@ import (
"crypto/tls"
"fmt"
"net/http"
"time"
"github.com/coredns/caddy"
"github.com/coredns/coredns/plugin"
@ -53,6 +54,11 @@ type Config struct {
// TLSConfig when listening for encrypted connections (gRPC, DNS-over-TLS).
TLSConfig *tls.Config
// Timeouts for TCP, TLS and HTTPS servers.
ReadTimeout time.Duration
WriteTimeout time.Duration
IdleTimeout time.Duration
// TSIG secrets, [name]key.
TsigSecret map[string]string

51
vendor/github.com/coredns/coredns/core/dnsserver/https.go generated vendored
View File

@ -4,13 +4,11 @@ import (
"net"
"net/http"
"github.com/coredns/coredns/plugin/pkg/nonwriter"
"github.com/miekg/dns"
)
// DoHWriter is a nonwriter.Writer that adds more specific LocalAddr and RemoteAddr methods.
// DoHWriter is a dns.ResponseWriter that adds more specific LocalAddr and RemoteAddr methods.
type DoHWriter struct {
nonwriter.Writer
// raddr is the remote's address. This can be optionally set.
raddr net.Addr
// laddr is our address. This can be optionally set.
@ -18,13 +16,50 @@ type DoHWriter struct {
// request is the HTTP request we're currently handling.
request *http.Request
// Msg is a response to be written to the client.
Msg *dns.Msg
}
// WriteMsg stores the message to be written to the client.
func (d *DoHWriter) WriteMsg(m *dns.Msg) error {
d.Msg = m
return nil
}
// Write stores the message to be written to the client.
func (d *DoHWriter) Write(b []byte) (int, error) {
d.Msg = new(dns.Msg)
return len(b), d.Msg.Unpack(b)
}
// RemoteAddr returns the remote address.
func (d *DoHWriter) RemoteAddr() net.Addr { return d.raddr }
func (d *DoHWriter) RemoteAddr() net.Addr {
return d.raddr
}
// LocalAddr returns the local address.
func (d *DoHWriter) LocalAddr() net.Addr { return d.laddr }
func (d *DoHWriter) LocalAddr() net.Addr {
return d.laddr
}
// Request returns the HTTP request
func (d *DoHWriter) Request() *http.Request { return d.request }
// Request returns the HTTP request.
func (d *DoHWriter) Request() *http.Request {
return d.request
}
// Close no-op implementation.
func (d *DoHWriter) Close() error {
return nil
}
// TsigStatus no-op implementation.
func (d *DoHWriter) TsigStatus() error {
return nil
}
// TsigTimersOnly no-op implementation.
func (d *DoHWriter) TsigTimersOnly(_ bool) {}
// Hijack no-op implementation.
func (d *DoHWriter) Hijack() {}

60
vendor/github.com/coredns/coredns/core/dnsserver/quic.go generated vendored Normal file
View File

@ -0,0 +1,60 @@
package dnsserver
import (
"encoding/binary"
"net"
"github.com/miekg/dns"
"github.com/quic-go/quic-go"
)
type DoQWriter struct {
localAddr net.Addr
remoteAddr net.Addr
stream quic.Stream
Msg *dns.Msg
}
func (w *DoQWriter) Write(b []byte) (int, error) {
b = AddPrefix(b)
return w.stream.Write(b)
}
func (w *DoQWriter) WriteMsg(m *dns.Msg) error {
bytes, err := m.Pack()
if err != nil {
return err
}
_, err = w.Write(bytes)
if err != nil {
return err
}
return w.Close()
}
// Close sends the STREAM FIN signal.
// The server MUST send the response(s) on the same stream and MUST
// indicate, after the last response, through the STREAM FIN
// mechanism that no further data will be sent on that stream.
// See https://www.rfc-editor.org/rfc/rfc9250#section-4.2-7
func (w *DoQWriter) Close() error {
return w.stream.Close()
}
// AddPrefix adds a 2-byte prefix with the DNS message length.
func AddPrefix(b []byte) (m []byte) {
m = make([]byte, 2+len(b))
binary.BigEndian.PutUint16(m, uint16(len(b)))
copy(m[2:], b)
return m
}
// These methods implement the dns.ResponseWriter interface from Go DNS.
func (w *DoQWriter) TsigStatus() error { return nil }
func (w *DoQWriter) TsigTimersOnly(b bool) {}
func (w *DoQWriter) Hijack() {}
func (w *DoQWriter) LocalAddr() net.Addr { return w.localAddr }
func (w *DoQWriter) RemoteAddr() net.Addr { return w.remoteAddr }

25
vendor/github.com/coredns/coredns/core/dnsserver/register.go generated vendored
View File

@ -1,7 +1,6 @@
package dnsserver
import (
"flag"
"fmt"
"net"
"time"
@ -17,12 +16,7 @@ import (
const serverType = "dns"
// Any flags defined here, need to be namespaced to the serverType other
// wise they potentially clash with other server types.
func init() {
flag.StringVar(&Port, serverType+".port", DefaultPort, "Default port")
flag.StringVar(&Port, "p", DefaultPort, "Default port")
caddy.RegisterServerType(serverType, caddy.ServerType{
Directives: func() []string { return Directives },
DefaultInput: func() caddy.Input {
@ -88,6 +82,8 @@ func (h *dnsContext) InspectServerBlocks(sourceFile string, serverBlocks []caddy
port = Port
case transport.TLS:
port = transport.TLSPort
case transport.QUIC:
port = transport.QUICPort
case transport.GRPC:
port = transport.GRPCPort
case transport.HTTPS:
@ -147,7 +143,12 @@ func (h *dnsContext) MakeServers() ([]caddy.Server, error) {
c.ListenHosts = c.firstConfigInBlock.ListenHosts
c.Debug = c.firstConfigInBlock.Debug
c.Stacktrace = c.firstConfigInBlock.Stacktrace
c.TLSConfig = c.firstConfigInBlock.TLSConfig
// Fork TLSConfig for each encrypted connection
c.TLSConfig = c.firstConfigInBlock.TLSConfig.Clone()
c.ReadTimeout = c.firstConfigInBlock.ReadTimeout
c.WriteTimeout = c.firstConfigInBlock.WriteTimeout
c.IdleTimeout = c.firstConfigInBlock.IdleTimeout
c.TsigSecret = c.firstConfigInBlock.TsigSecret
}
@ -175,6 +176,13 @@ func (h *dnsContext) MakeServers() ([]caddy.Server, error) {
}
servers = append(servers, s)
case transport.QUIC:
s, err := NewServerQUIC(addr, group)
if err != nil {
return nil, err
}
servers = append(servers, s)
case transport.GRPC:
s, err := NewServergRPC(addr, group)
if err != nil {
@ -221,6 +229,7 @@ func (c *Config) AddPlugin(m plugin.Plugin) {
}
// registerHandler adds a handler to a site's handler registration. Handlers
//
// use this to announce that they exist to other plugin.
func (c *Config) registerHandler(h plugin.Handler) {
if c.registry == nil {
@ -287,7 +296,7 @@ func (h *dnsContext) validateZonesAndListeningAddresses() error {
return nil
}
// groupSiteConfigsByListenAddr groups site configs by their listen
// groupConfigsByListenAddr groups site configs by their listen
// (bind) address, so sites that use the same listener can be served
// on the same server instance. The return value maps the listen
// address (what you pass into net.Listen) to the list of site configs.

34
vendor/github.com/coredns/coredns/core/dnsserver/server.go generated vendored
View File

@ -44,6 +44,9 @@ type Server struct {
debug bool // disable recover()
stacktrace bool // enable stacktrace in recover error log
classChaos bool // allow non-INET class queries
idleTimeout time.Duration // Idle timeout for TCP
readTimeout time.Duration // Read timeout for TCP
writeTimeout time.Duration // Write timeout for TCP
tsigSecret map[string]string
}
@ -60,6 +63,9 @@ func NewServer(addr string, group []*Config) (*Server, error) {
Addr: addr,
zones: make(map[string][]*Config),
graceTimeout: 5 * time.Second,
idleTimeout: 10 * time.Second,
readTimeout: 3 * time.Second,
writeTimeout: 5 * time.Second,
tsigSecret: make(map[string]string),
}
@ -81,6 +87,17 @@ func NewServer(addr string, group []*Config) (*Server, error) {
// append the config to the zone's configs
s.zones[site.Zone] = append(s.zones[site.Zone], site)
// set timeouts
if site.ReadTimeout != 0 {
s.readTimeout = site.ReadTimeout
}
if site.WriteTimeout != 0 {
s.writeTimeout = site.WriteTimeout
}
if site.IdleTimeout != 0 {
s.idleTimeout = site.IdleTimeout
}
// copy tsig secrets
for key, secret := range site.TsigSecret {
s.tsigSecret[key] = secret
@ -130,11 +147,22 @@ var _ caddy.GracefulServer = &Server{}
// This implements caddy.TCPServer interface.
func (s *Server) Serve(l net.Listener) error {
s.m.Lock()
s.server[tcp] = &dns.Server{Listener: l, Net: "tcp", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
s.server[tcp] = &dns.Server{Listener: l,
Net: "tcp",
TsigSecret: s.tsigSecret,
MaxTCPQueries: tcpMaxQueries,
ReadTimeout: s.readTimeout,
WriteTimeout: s.writeTimeout,
IdleTimeout: func() time.Duration {
return s.idleTimeout
},
Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
ctx := context.WithValue(context.Background(), Key{}, s)
ctx = context.WithValue(ctx, LoopKey{}, 0)
s.ServeDNS(ctx, w, r)
}), TsigSecret: s.tsigSecret}
})}
s.m.Unlock()
return s.server[tcp].ActivateAndServe()
@ -404,6 +432,8 @@ func errorAndMetricsFunc(server string, w dns.ResponseWriter, r *dns.Msg, rc int
const (
tcp = 0
udp = 1
tcpMaxQueries = -1
)
type (

6
vendor/github.com/coredns/coredns/core/dnsserver/server_https.go generated vendored
View File

@ -75,9 +75,9 @@ func NewServerHTTPS(addr string, group []*Config) (*ServerHTTPS, error) {
}
srv := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
ReadTimeout: s.readTimeout,
WriteTimeout: s.writeTimeout,
IdleTimeout: s.idleTimeout,
ErrorLog: stdlog.New(&loggerAdapter{}, "", 0),
}
sh := &ServerHTTPS{

346
vendor/github.com/coredns/coredns/core/dnsserver/server_quic.go generated vendored Normal file
View File

@ -0,0 +1,346 @@
package dnsserver
import (
"context"
"crypto/tls"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"net"
"github.com/coredns/coredns/plugin/metrics/vars"
clog "github.com/coredns/coredns/plugin/pkg/log"
"github.com/coredns/coredns/plugin/pkg/reuseport"
"github.com/coredns/coredns/plugin/pkg/transport"
"github.com/miekg/dns"
"github.com/quic-go/quic-go"
)
const (
// DoQCodeNoError is used when the connection or stream needs to be
// closed, but there is no error to signal.
DoQCodeNoError quic.ApplicationErrorCode = 0
// DoQCodeInternalError signals that the DoQ implementation encountered
// an internal error and is incapable of pursuing the transaction or the
// connection.
DoQCodeInternalError quic.ApplicationErrorCode = 1
// DoQCodeProtocolError signals that the DoQ implementation encountered
// a protocol error and is forcibly aborting the connection.
DoQCodeProtocolError quic.ApplicationErrorCode = 2
)
// ServerQUIC represents an instance of a DNS-over-QUIC server.
type ServerQUIC struct {
*Server
listenAddr net.Addr
tlsConfig *tls.Config
quicConfig *quic.Config
quicListener *quic.Listener
}
// NewServerQUIC returns a new CoreDNS QUIC server and compiles all plugin in to it.
func NewServerQUIC(addr string, group []*Config) (*ServerQUIC, error) {
s, err := NewServer(addr, group)
if err != nil {
return nil, err
}
// The *tls* plugin must make sure that multiple conflicting
// TLS configuration returns an error: it can only be specified once.
var tlsConfig *tls.Config
for _, z := range s.zones {
for _, conf := range z {
// Should we error if some configs *don't* have TLS?
tlsConfig = conf.TLSConfig
}
}
if tlsConfig != nil {
tlsConfig.NextProtos = []string{"doq"}
}
var quicConfig *quic.Config
quicConfig = &quic.Config{
MaxIdleTimeout: s.idleTimeout,
MaxIncomingStreams: math.MaxUint16,
MaxIncomingUniStreams: math.MaxUint16,
// Enable 0-RTT by default for all connections on the server-side.
Allow0RTT: true,
}
return &ServerQUIC{Server: s, tlsConfig: tlsConfig, quicConfig: quicConfig}, nil
}
// ServePacket implements caddy.UDPServer interface.
func (s *ServerQUIC) ServePacket(p net.PacketConn) error {
s.m.Lock()
s.listenAddr = s.quicListener.Addr()
s.m.Unlock()
return s.ServeQUIC()
}
// ServeQUIC listens for incoming QUIC packets.
func (s *ServerQUIC) ServeQUIC() error {
for {
conn, err := s.quicListener.Accept(context.Background())
if err != nil {
if s.isExpectedErr(err) {
s.closeQUICConn(conn, DoQCodeNoError)
return err
}
s.closeQUICConn(conn, DoQCodeInternalError)
return err
}
go s.serveQUICConnection(conn)
}
}
// serveQUICConnection handles a new QUIC connection. It waits for new streams
// and passes them to serveQUICStream.
func (s *ServerQUIC) serveQUICConnection(conn quic.Connection) {
for {
// In DoQ, one query consumes one stream.
// The client MUST select the next available client-initiated bidirectional
// stream for each subsequent query on a QUIC connection.
stream, err := conn.AcceptStream(context.Background())
if err != nil {
if s.isExpectedErr(err) {
s.closeQUICConn(conn, DoQCodeNoError)
return
}
s.closeQUICConn(conn, DoQCodeInternalError)
return
}
go s.serveQUICStream(stream, conn)
}
}
func (s *ServerQUIC) serveQUICStream(stream quic.Stream, conn quic.Connection) {
buf, err := readDOQMessage(stream)
// io.EOF does not really mean that there's any error, it is just
// the STREAM FIN indicating that there will be no data to read
// anymore from this stream.
if err != nil && err != io.EOF {
s.closeQUICConn(conn, DoQCodeProtocolError)
return
}
req := &dns.Msg{}
err = req.Unpack(buf)
if err != nil {
clog.Debugf("unpacking quic packet: %s", err)
s.closeQUICConn(conn, DoQCodeProtocolError)
return
}
if !validRequest(req) {
// If a peer encounters such an error condition, it is considered a
// fatal error. It SHOULD forcibly abort the connection using QUIC's
// CONNECTION_CLOSE mechanism and SHOULD use the DoQ error code
// DOQ_PROTOCOL_ERROR.
// See https://www.rfc-editor.org/rfc/rfc9250#section-4.3.3-3
s.closeQUICConn(conn, DoQCodeProtocolError)
return
}
w := &DoQWriter{
localAddr: conn.LocalAddr(),
remoteAddr: conn.RemoteAddr(),
stream: stream,
Msg: req,
}
dnsCtx := context.WithValue(stream.Context(), Key{}, s.Server)
dnsCtx = context.WithValue(dnsCtx, LoopKey{}, 0)
s.ServeDNS(dnsCtx, w, req)
s.countResponse(DoQCodeNoError)
}
// ListenPacket implements caddy.UDPServer interface.
func (s *ServerQUIC) ListenPacket() (net.PacketConn, error) {
p, err := reuseport.ListenPacket("udp", s.Addr[len(transport.QUIC+"://"):])
if err != nil {
return nil, err
}
s.m.Lock()
defer s.m.Unlock()
s.quicListener, err = quic.Listen(p, s.tlsConfig, s.quicConfig)
if err != nil {
return nil, err
}
return p, nil
}
// OnStartupComplete lists the sites served by this server
// and any relevant information, assuming Quiet is false.
func (s *ServerQUIC) OnStartupComplete() {
if Quiet {
return
}
out := startUpZones(transport.QUIC+"://", s.Addr, s.zones)
if out != "" {
fmt.Print(out)
}
}
// Stop stops the server non-gracefully. It blocks until the server is totally stopped.
func (s *ServerQUIC) Stop() error {
s.m.Lock()
defer s.m.Unlock()
if s.quicListener != nil {
return s.quicListener.Close()
}
return nil
}
// Serve implements caddy.TCPServer interface.
func (s *ServerQUIC) Serve(l net.Listener) error { return nil }
// Listen implements caddy.TCPServer interface.
func (s *ServerQUIC) Listen() (net.Listener, error) { return nil, nil }
// closeQUICConn quietly closes the QUIC connection.
func (s *ServerQUIC) closeQUICConn(conn quic.Connection, code quic.ApplicationErrorCode) {
if conn == nil {
return
}
clog.Debugf("closing quic conn %s with code %d", conn.LocalAddr(), code)
err := conn.CloseWithError(code, "")
if err != nil {
clog.Debugf("failed to close quic connection with code %d: %s", code, err)
}
// DoQCodeNoError metrics are already registered after s.ServeDNS()
if code != DoQCodeNoError {
s.countResponse(code)
}
}
// validRequest checks for protocol errors in the unpacked DNS message.
// See https://www.rfc-editor.org/rfc/rfc9250.html#name-protocol-errors
func validRequest(req *dns.Msg) (ok bool) {
// 1. a client or server receives a message with a non-zero Message ID.
if req.Id != 0 {
return false
}
// 2. an implementation receives a message containing the edns-tcp-keepalive
// EDNS(0) Option [RFC7828].
if opt := req.IsEdns0(); opt != nil {
for _, option := range opt.Option {
if option.Option() == dns.EDNS0TCPKEEPALIVE {
clog.Debug("client sent EDNS0 TCP keepalive option")
return false
}
}
}
// 3. the client or server does not indicate the expected STREAM FIN after
// sending requests or responses.
//
// This is quite problematic to validate this case since this would imply
// we have to wait until STREAM FIN is arrived before we start processing
// the message. So we're consciously ignoring this case in this
// implementation.
// 4. a server receives a "replayable" transaction in 0-RTT data
//
// The information necessary to validate this is not exposed by quic-go.
return true
}
// readDOQMessage reads a DNS over QUIC (DOQ) message from the given stream
// and returns the message bytes.
// Drafts of the RFC9250 did not require the 2-byte prefixed message length.
// Thus, we are only supporting the official version (DoQ v1).
func readDOQMessage(r io.Reader) ([]byte, error) {
// All DNS messages (queries and responses) sent over DoQ connections MUST
// be encoded as a 2-octet length field followed by the message content as
// specified in [RFC1035].
// See https://www.rfc-editor.org/rfc/rfc9250.html#section-4.2-4
sizeBuf := make([]byte, 2)
_, err := io.ReadFull(r, sizeBuf)
if err != nil {
return nil, err
}
size := binary.BigEndian.Uint16(sizeBuf)
if size == 0 {
return nil, fmt.Errorf("message size is 0: probably unsupported DoQ version")
}
buf := make([]byte, size)
_, err = io.ReadFull(r, buf)
// A client or server receives a STREAM FIN before receiving all the bytes
// for a message indicated in the 2-octet length field.
// See https://www.rfc-editor.org/rfc/rfc9250#section-4.3.3-2.2
if size != uint16(len(buf)) {
return nil, fmt.Errorf("message size does not match 2-byte prefix")
}
return buf, err
}
// isExpectedErr returns true if err is an expected error, likely related to
// the current implementation.
func (s *ServerQUIC) isExpectedErr(err error) bool {
if err == nil {
return false
}
// This error is returned when the QUIC listener was closed by us. As
// graceful shutdown is not implemented, the connection will be abruptly
// closed but there is no error to signal.
if errors.Is(err, quic.ErrServerClosed) {
return true
}
// This error happens when the connection was closed due to a DoQ
// protocol error but there's still something to read in the closed stream.
// For example, when the message was sent without the prefixed length.
var qAppErr *quic.ApplicationError
if errors.As(err, &qAppErr) && qAppErr.ErrorCode == 2 {
return true
}
// When a connection hits the idle timeout, quic.AcceptStream() returns
// an IdleTimeoutError. In this, case, we should just drop the connection
// with DoQCodeNoError.
var qIdleErr *quic.IdleTimeoutError
return errors.As(err, &qIdleErr)
}
func (s *ServerQUIC) countResponse(code quic.ApplicationErrorCode) {
switch code {
case DoQCodeNoError:
vars.QUICResponsesCount.WithLabelValues(s.Addr, "0x0").Inc()
case DoQCodeInternalError:
vars.QUICResponsesCount.WithLabelValues(s.Addr, "0x1").Inc()
case DoQCodeProtocolError:
vars.QUICResponsesCount.WithLabelValues(s.Addr, "0x2").Inc()
}
}

16
vendor/github.com/coredns/coredns/core/dnsserver/server_tls.go generated vendored
View File

@ -5,6 +5,7 @@ import (
"crypto/tls"
"fmt"
"net"
"time"
"github.com/coredns/caddy"
"github.com/coredns/coredns/plugin/pkg/reuseport"
@ -50,11 +51,20 @@ func (s *ServerTLS) Serve(l net.Listener) error {
}
// Only fill out the TCP server for this one.
s.server[tcp] = &dns.Server{Listener: l, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
s.server[tcp] = &dns.Server{Listener: l,
Net: "tcp-tls",
MaxTCPQueries: tlsMaxQueries,
ReadTimeout: s.readTimeout,
WriteTimeout: s.writeTimeout,
IdleTimeout: func() time.Duration {
return s.idleTimeout
},
Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) {
ctx := context.WithValue(context.Background(), Key{}, s.Server)
ctx = context.WithValue(ctx, LoopKey{}, 0)
s.ServeDNS(ctx, w, r)
})}
s.m.Unlock()
return s.server[tcp].ActivateAndServe()
@ -87,3 +97,7 @@ func (s *ServerTLS) OnStartupComplete() {
fmt.Print(out)
}
}
const (
tlsMaxQueries = -1
)

3
vendor/github.com/coredns/coredns/core/dnsserver/zdirectives.go generated vendored
View File

@ -10,14 +10,15 @@ package dnsserver
// (after) them during a request, but they must not
// care what plugin above them are doing.
var Directives = []string{
"root",
"metadata",
"geoip",
"cancel",
"tls",
"timeouts",
"reload",
"nsid",
"bufsize",
"root",
"bind",
"debug",
"trace",

9
vendor/github.com/coredns/coredns/coremain/run.go generated vendored
View File

@ -28,6 +28,9 @@ func init() {
caddy.RegisterCaddyfileLoader("flag", caddy.LoaderFunc(confLoader))
caddy.SetDefaultCaddyfileLoader("default", caddy.LoaderFunc(defaultLoader))
flag.StringVar(&dnsserver.Port, serverType+".port", dnsserver.DefaultPort, "Default port")
flag.StringVar(&dnsserver.Port, "p", dnsserver.DefaultPort, "Default port")
caddy.AppName = coreName
caddy.AppVersion = CoreVersion
}
@ -42,7 +45,7 @@ func Run() {
}
log.SetOutput(os.Stdout)
log.SetFlags(0) // Set to 0 because we're doing our own time, with timezone
log.SetFlags(LogFlags)
if version {
showVersion()
@ -166,10 +169,14 @@ var (
conf string
version bool
plugins bool
// LogFlags are initially set to 0 for no extra output
LogFlags int
)
// Build information obtained with the help of -ldflags
var (
// nolint
appVersion = "(untracked dev build)" // inferred at startup
devBuild = true // inferred at startup

2
vendor/github.com/coredns/coredns/coremain/version.go generated vendored
View File

@ -2,7 +2,7 @@ package coremain
// Various CoreDNS constants.
const (
CoreVersion = "1.10.0"
CoreVersion = "1.11.3"
coreName = "CoreDNS"
serverType = "dns"
)

9
vendor/github.com/coredns/coredns/plugin/cache/README.md generated vendored
View File

@ -10,8 +10,7 @@ With *cache* enabled, all records except zone transfers and metadata records wil
3600s. Caching is mostly useful in a scenario when fetching data from the backend (upstream,
database, etc.) is expensive.
*Cache* will change the query to enable DNSSEC (DNSSEC OK; DO) if it passes through the plugin. If
the client didn't request any DNSSEC (records), these are filtered out when replying.
*Cache* will pass DNSSEC (DNSSEC OK; DO) options through the plugin for upstream queries.
This plugin can only be used once per Server Block.
@ -40,6 +39,7 @@ cache [TTL] [ZONES...] {
serve_stale [DURATION] [REFRESH_MODE]
servfail DURATION
disable success|denial [ZONES...]
keepttl
}
~~~
@ -70,6 +70,11 @@ cache [TTL] [ZONES...] {
greater than 5 minutes.
* `disable` disable the success or denial cache for the listed **ZONES**. If no **ZONES** are given, the specified
cache will be disabled for all zones.
* `keepttl` do not age TTL when serving responses from cache. The entry will still be removed from cache
when the TTL expires as normal, but until it expires responses will include the original TTL instead
of the remaining TTL. This can be useful if CoreDNS is used as an authoritative server and you want
to serve a consistent TTL to downstream clients. This is **NOT** recommended when CoreDNS is caching
records it is not authoritative for because it could result in downstream clients using stale answers.
## Capacity and Eviction

37
vendor/github.com/coredns/coredns/plugin/cache/cache.go generated vendored
View File

@ -48,6 +48,9 @@ type Cache struct {
pexcept []string
nexcept []string
// Keep ttl option
keepttl bool
// Testing.
now func() time.Time
}
@ -76,7 +79,7 @@ func New() *Cache {
// key returns key under which we store the item, -1 will be returned if we don't store the message.
// Currently we do not cache Truncated, errors zone transfers or dynamic update messages.
// qname holds the already lowercased qname.
func key(qname string, m *dns.Msg, t response.Type) (bool, uint64) {
func key(qname string, m *dns.Msg, t response.Type, do, cd bool) (bool, uint64) {
// We don't store truncated responses.
if m.Truncated {
return false, 0
@ -86,11 +89,27 @@ func key(qname string, m *dns.Msg, t response.Type) (bool, uint64) {
return false, 0
}
return true, hash(qname, m.Question[0].Qtype)
return true, hash(qname, m.Question[0].Qtype, do, cd)
}
func hash(qname string, qtype uint16) uint64 {
var one = []byte("1")
var zero = []byte("0")
func hash(qname string, qtype uint16, do, cd bool) uint64 {
h := fnv.New64()
if do {
h.Write(one)
} else {
h.Write(zero)
}
if cd {
h.Write(one)
} else {
h.Write(zero)
}
h.Write([]byte{byte(qtype >> 8)})
h.Write([]byte{byte(qtype)})
h.Write([]byte(qname))
@ -116,6 +135,7 @@ type ResponseWriter struct {
server string // Server handling the request.
do bool // When true the original request had the DO bit set.
cd bool // When true the original request had the CD bit set.
ad bool // When true the original request had the AD bit set.
prefetch bool // When true write nothing back to the client.
remoteAddr net.Addr
@ -145,6 +165,8 @@ func newPrefetchResponseWriter(server string, state request.Request, c *Cache) *
Cache: c,
state: state,
server: server,
do: state.Do(),
cd: state.Req.CheckingDisabled,
prefetch: true,
remoteAddr: addr,
}
@ -163,7 +185,7 @@ func (w *ResponseWriter) WriteMsg(res *dns.Msg) error {
mt, _ := response.Typify(res, w.now().UTC())
// key returns empty string for anything we don't want to cache.
hasKey, key := key(w.state.Name(), res, mt)
hasKey, key := key(w.state.Name(), res, mt, w.do, w.cd)
msgTTL := dnsutil.MinimalTTL(res, mt)
var duration time.Duration
@ -191,11 +213,10 @@ func (w *ResponseWriter) WriteMsg(res *dns.Msg) error {
}
// Apply capped TTL to this reply to avoid jarring TTL experience 1799 -> 8 (e.g.)
// We also may need to filter out DNSSEC records, see toMsg() for similar code.
ttl := uint32(duration.Seconds())
res.Answer = filterRRSlice(res.Answer, ttl, w.do, false)
res.Ns = filterRRSlice(res.Ns, ttl, w.do, false)
res.Extra = filterRRSlice(res.Extra, ttl, w.do, false)
res.Answer = filterRRSlice(res.Answer, ttl, false)
res.Ns = filterRRSlice(res.Ns, ttl, false)
res.Extra = filterRRSlice(res.Extra, ttl, false)
if !w.do && !w.ad {
// unset AD bit if requester is not OK with DNSSEC

28
vendor/github.com/coredns/coredns/plugin/cache/dnssec.go generated vendored
View File

@ -2,35 +2,13 @@ package cache
import "github.com/miekg/dns"
// isDNSSEC returns true if r is a DNSSEC record. NSEC,NSEC3,DS and RRSIG/SIG
// are DNSSEC records. DNSKEYs is not in this list on the assumption that the
// client explicitly asked for it.
func isDNSSEC(r dns.RR) bool {
switch r.Header().Rrtype {
case dns.TypeNSEC:
return true
case dns.TypeNSEC3:
return true
case dns.TypeDS:
return true
case dns.TypeRRSIG:
return true
case dns.TypeSIG:
return true
}
return false
}
// filterRRSlice filters rrs and removes DNSSEC RRs when do is false. In the returned slice
// the TTLs are set to ttl. If dup is true the RRs in rrs are _copied_ into the slice that is
// filterRRSlice filters out OPT RRs, and sets all RR TTLs to ttl.
// If dup is true the RRs in rrs are _copied_ into the slice that is
// returned.
func filterRRSlice(rrs []dns.RR, ttl uint32, do, dup bool) []dns.RR {
func filterRRSlice(rrs []dns.RR, ttl uint32, dup bool) []dns.RR {
j := 0
rs := make([]dns.RR, len(rrs))
for _, r := range rrs {
if !do && isDNSSEC(r) {
continue
}
if r.Header().Rrtype == dns.TypeOPT {
continue
}

46
vendor/github.com/coredns/coredns/plugin/cache/handler.go generated vendored
View File

@ -18,6 +18,7 @@ func (c *Cache) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
rc := r.Copy() // We potentially modify r, to prevent other plugins from seeing this (r is a pointer), copy r into rc.
state := request.Request{W: w, Req: rc}
do := state.Do()
cd := r.CheckingDisabled
ad := r.AuthenticatedData
zone := plugin.Zones(c.Zones).Matches(state.Name())
@ -28,17 +29,15 @@ func (c *Cache) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
now := c.now().UTC()
server := metrics.WithServer(ctx)
// On cache miss, if the request has the OPT record and the DO bit set we leave the message as-is. If there isn't a DO bit
// set we will modify the request to _add_ one. This means we will always do DNSSEC lookups on cache misses.
// When writing to cache, any DNSSEC RRs in the response are written to cache with the response.
// When sending a response to a non-DNSSEC client, we remove DNSSEC RRs from the response. We use a 2048 buffer size, which is
// less than 4096 (and older default) and more than 1024 which may be too small. We might need to tweaks this
// value to be smaller still to prevent UDP fragmentation?
// On cache refresh, we will just use the DO bit from the incoming query for the refresh since we key our cache
// with the query DO bit. That means two separate cache items for the query DO bit true or false. In the situation
// in which upstream doesn't support DNSSEC, the two cache items will effectively be the same. Regardless, any
// DNSSEC RRs in the response are written to cache with the response.
ttl := 0
i := c.getIgnoreTTL(now, state, server)
if i == nil {
crr := &ResponseWriter{ResponseWriter: w, Cache: c, state: state, server: server, do: do, ad: ad,
crr := &ResponseWriter{ResponseWriter: w, Cache: c, state: state, server: server, do: do, ad: ad, cd: cd,
nexcept: c.nexcept, pexcept: c.pexcept, wildcardFunc: wildcardFunc(ctx)}
return c.doRefresh(ctx, state, crr)
}
@ -46,7 +45,7 @@ func (c *Cache) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
if ttl < 0 {
// serve stale behavior
if c.verifyStale {
crr := &ResponseWriter{ResponseWriter: w, Cache: c, state: state, server: server, do: do}
crr := &ResponseWriter{ResponseWriter: w, Cache: c, state: state, server: server, do: do, cd: cd}
cw := newVerifyStaleResponseWriter(crr)
ret, err := c.doRefresh(ctx, state, cw)
if cw.refreshed {
@ -73,6 +72,11 @@ func (c *Cache) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
})
}
if c.keepttl {
// If keepttl is enabled we fake the current time to the stored
// one so that we always get the original TTL
now = i.stored
}
resp := i.toMsg(r, now, do, ad)
w.WriteMsg(resp)
return dns.RcodeSuccess, nil
@ -101,9 +105,6 @@ func (c *Cache) doPrefetch(ctx context.Context, state request.Request, cw *Respo
}
func (c *Cache) doRefresh(ctx context.Context, state request.Request, cw dns.ResponseWriter) (int, error) {
if !state.Do() {
setDo(state.Req)
}
return plugin.NextOrFailure(c.Name(), c.Next, ctx, cw, state.Req)
}
@ -121,7 +122,7 @@ func (c *Cache) Name() string { return "cache" }
// getIgnoreTTL unconditionally returns an item if it exists in the cache.
func (c *Cache) getIgnoreTTL(now time.Time, state request.Request, server string) *item {
k := hash(state.Name(), state.QType())
k := hash(state.Name(), state.QType(), state.Do(), state.Req.CheckingDisabled)
cacheRequests.WithLabelValues(server, c.zonesMetricLabel, c.viewMetricLabel).Inc()
if i, ok := c.ncache.Get(k); ok {
@ -145,7 +146,7 @@ func (c *Cache) getIgnoreTTL(now time.Time, state request.Request, server string
}
func (c *Cache) exists(state request.Request) *item {
k := hash(state.Name(), state.QType())
k := hash(state.Name(), state.QType(), state.Do(), state.Req.CheckingDisabled)
if i, ok := c.ncache.Get(k); ok {
return i.(*item)
}
@ -154,22 +155,3 @@ func (c *Cache) exists(state request.Request) *item {
}
return nil
}
// setDo sets the DO bit and UDP buffer size in the message m.
func setDo(m *dns.Msg) {
o := m.IsEdns0()
if o != nil {
o.SetDo()
o.SetUDPSize(defaultUDPBufSize)
return
}
o = &dns.OPT{Hdr: dns.RR_Header{Name: ".", Rrtype: dns.TypeOPT}}
o.SetDo()
o.SetUDPSize(defaultUDPBufSize)
m.Extra = append(m.Extra, o)
}
// defaultUDPBufsize is the bufsize the cache plugin uses on outgoing requests that don't
// have an OPT RR.
const defaultUDPBufSize = 2048

6
vendor/github.com/coredns/coredns/plugin/cache/item.go generated vendored
View File

@ -87,9 +87,9 @@ func (i *item) toMsg(m *dns.Msg, now time.Time, do bool, ad bool) *dns.Msg {
m1.Extra = make([]dns.RR, len(i.Extra))
ttl := uint32(i.ttl(now))
m1.Answer = filterRRSlice(i.Answer, ttl, do, true)
m1.Ns = filterRRSlice(i.Ns, ttl, do, true)
m1.Extra = filterRRSlice(i.Extra, ttl, do, true)
m1.Answer = filterRRSlice(i.Answer, ttl, true)
m1.Ns = filterRRSlice(i.Ns, ttl, true)
m1.Extra = filterRRSlice(i.Extra, ttl, true)
return m1
}

6
vendor/github.com/coredns/coredns/plugin/cache/setup.go generated vendored
View File

@ -240,6 +240,12 @@ func cacheParse(c *caddy.Controller) (*Cache, error) {
default:
return nil, fmt.Errorf("cache type for disable must be %q or %q", Success, Denial)
}
case "keepttl":
args := c.RemainingArgs()
if len(args) != 0 {
return nil, c.ArgErr()
}
ca.keepttl = true
default:
return nil, c.ArgErr()
}

1
vendor/github.com/coredns/coredns/plugin/metadata/provider.go generated vendored
View File

@ -34,7 +34,6 @@
// valueFunc := metadata.ValueFunc(ctx, "test/something")
// value := valueFunc()
// // use 'value'
//
package metadata
import (

1
vendor/github.com/coredns/coredns/plugin/metrics/README.md generated vendored
View File

@ -21,6 +21,7 @@ the following metrics are exported:
* `coredns_dns_response_size_bytes{server, zone, view, proto}` - response size in bytes.
* `coredns_dns_responses_total{server, zone, view, rcode, plugin}` - response per zone, rcode and plugin.
* `coredns_dns_https_responses_total{server, status}` - responses per server and http status code.
* `coredns_dns_quic_responses_total{server, status}` - responses per server and QUIC application code.
* `coredns_plugin_enabled{server, zone, view, name}` - indicates whether a plugin is enabled on per server, zone and view basis.
Almost each counter has a label `zone` which is the zonename used for the request/response.

10
vendor/github.com/coredns/coredns/plugin/metrics/vars/vars.go generated vendored
View File

@ -21,6 +21,7 @@ var (
Subsystem: subsystem,
Name: "request_duration_seconds",
Buckets: plugin.TimeBuckets,
NativeHistogramBucketFactor: plugin.NativeHistogramBucketFactor,
Help: "Histogram of the time (in seconds) each request took per zone.",
}, []string{"server", "zone", "view"})
@ -30,6 +31,7 @@ var (
Name: "request_size_bytes",
Help: "Size of the EDNS0 UDP buffer in bytes (64K for TCP) per zone and protocol.",
Buckets: []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
NativeHistogramBucketFactor: plugin.NativeHistogramBucketFactor,
}, []string{"server", "zone", "view", "proto"})
RequestDo = promauto.NewCounterVec(prometheus.CounterOpts{
@ -45,6 +47,7 @@ var (
Name: "response_size_bytes",
Help: "Size of the returned response in bytes.",
Buckets: []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
NativeHistogramBucketFactor: plugin.NativeHistogramBucketFactor,
}, []string{"server", "zone", "view", "proto"})
ResponseRcode = promauto.NewCounterVec(prometheus.CounterOpts{
@ -72,6 +75,13 @@ var (
Name: "https_responses_total",
Help: "Counter of DoH responses per server and http status code.",
}, []string{"server", "status"})
QUICResponsesCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: plugin.Namespace,
Subsystem: subsystem,
Name: "quic_responses_total",
Help: "Counter of DoQ responses per server and QUIC application code.",
}, []string{"server", "status"})
)
const (

1
vendor/github.com/coredns/coredns/plugin/pkg/dnsutil/ttl.go generated vendored
View File

@ -48,5 +48,6 @@ const (
// MinimalDefaultTTL is the absolute lowest TTL we use in CoreDNS.
MinimalDefaultTTL = 5 * time.Second
// MaximumDefaulTTL is the maximum TTL was use on RRsets in CoreDNS.
// TODO: rename as MaximumDefaultTTL
MaximumDefaulTTL = 1 * time.Hour
)

23
vendor/github.com/coredns/coredns/plugin/pkg/doh/doh.go generated vendored
View File

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
"strings"
"github.com/miekg/dns"
)
@ -16,18 +17,30 @@ const MimeType = "application/dns-message"
// Path is the URL path that should be used.
const Path = "/dns-query"
// NewRequest returns a new DoH request given a method, URL (without any paths, so exclude /dns-query) and dns.Msg.
// NewRequest returns a new DoH request given a HTTP method, URL and dns.Msg.
//
// The URL should not have a path, so please exclude /dns-query. The URL will
// be prefixed with https:// by default, unless it's already prefixed with
// either http:// or https://.
func NewRequest(method, url string, m *dns.Msg) (*http.Request, error) {
buf, err := m.Pack()
if err != nil {
return nil, err
}
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
url = fmt.Sprintf("https://%s", url)
}
switch method {
case http.MethodGet:
b64 := base64.RawURLEncoding.EncodeToString(buf)
req, err := http.NewRequest(http.MethodGet, "https://"+url+Path+"?dns="+b64, nil)
req, err := http.NewRequest(
http.MethodGet,
fmt.Sprintf("%s%s?dns=%s", url, Path, b64),
nil,
)
if err != nil {
return req, err
}
@ -37,7 +50,11 @@ func NewRequest(method, url string, m *dns.Msg) (*http.Request, error) {
return req, nil
case http.MethodPost:
req, err := http.NewRequest(http.MethodPost, "https://"+url+Path+"?bla=foo:443", bytes.NewReader(buf))
req, err := http.NewRequest(
http.MethodPost,
fmt.Sprintf("%s%s?bla=foo:443", url, Path),
bytes.NewReader(buf),
)
if err != nil {
return req, err
}

5
vendor/github.com/coredns/coredns/plugin/pkg/edns/edns.go generated vendored
View File

@ -36,8 +36,7 @@ func SupportedOption(option uint16) bool {
// Version checks the EDNS version in the request. If error
// is nil everything is OK and we can invoke the plugin. If non-nil, the
// returned Msg is valid to be returned to the client (and should). For some
// reason this response should not contain a question RR in the question section.
// returned Msg is valid to be returned to the client (and should).
func Version(req *dns.Msg) (*dns.Msg, error) {
opt := req.IsEdns0()
if opt == nil {
@ -48,8 +47,6 @@ func Version(req *dns.Msg) (*dns.Msg, error) {
}
m := new(dns.Msg)
m.SetReply(req)
// zero out question section, wtf.
m.Question = nil
o := new(dns.OPT)
o.Hdr.Name = "."

18
vendor/github.com/coredns/coredns/plugin/pkg/log/log.go generated vendored
View File

@ -13,7 +13,7 @@ import (
"io"
golog "log"
"os"
"sync"
"sync/atomic"
)
// D controls whether we should output debug logs. If true, we do, once set
@ -21,30 +21,22 @@ import (
var D = &d{}
type d struct {
on bool
sync.RWMutex
on atomic.Bool
}
// Set enables debug logging.
func (d *d) Set() {
d.Lock()
d.on = true
d.Unlock()
d.on.Store(true)
}
// Clear disables debug logging.
func (d *d) Clear() {
d.Lock()
d.on = false
d.Unlock()
d.on.Store(false)
}
// Value returns if debug logging is enabled.
func (d *d) Value() bool {
d.RLock()
b := d.on
d.RUnlock()
return b
return d.on.Load()
}
// logf calls log.Printf prefixed with level.

21
vendor/github.com/coredns/coredns/plugin/pkg/nonwriter/nonwriter.go generated vendored
View File

@ -1,21 +0,0 @@
// Package nonwriter implements a dns.ResponseWriter that never writes, but captures the dns.Msg being written.
package nonwriter
import (
"github.com/miekg/dns"
)
// Writer is a type of ResponseWriter that captures the message, but never writes to the client.
type Writer struct {
dns.ResponseWriter
Msg *dns.Msg
}
// New makes and returns a new NonWriter.
func New(w dns.ResponseWriter) *Writer { return &Writer{ResponseWriter: w} }
// WriteMsg records the message, but doesn't write it itself.
func (w *Writer) WriteMsg(res *dns.Msg) error {
w.Msg = res
return nil
}

12
vendor/github.com/coredns/coredns/plugin/pkg/parse/host.go generated vendored
View File

@ -33,6 +33,14 @@ func HostPortOrFile(s ...string) ([]string, error) {
var servers []string
for _, h := range s {
trans, host := Transport(h)
if len(host) == 0 {
return servers, fmt.Errorf("invalid address: %q", h)
}
if trans == transport.UNIX {
servers = append(servers, trans+"://"+host)
continue
}
addr, _, err := net.SplitHostPort(host)
@ -53,6 +61,8 @@ func HostPortOrFile(s ...string) ([]string, error) {
ss = net.JoinHostPort(host, transport.Port)
case transport.TLS:
ss = transport.TLS + "://" + net.JoinHostPort(host, transport.TLSPort)
case transport.QUIC:
ss = transport.QUIC + "://" + net.JoinHostPort(host, transport.QUICPort)
case transport.GRPC:
ss = transport.GRPC + "://" + net.JoinHostPort(host, transport.GRPCPort)
case transport.HTTPS:
@ -89,7 +99,7 @@ func tryFile(s string) ([]string, error) {
servers := []string{}
for _, s := range c.Servers {
servers = append(servers, net.JoinHostPort(s, c.Port))
servers = append(servers, net.JoinHostPort(stripZone(s), c.Port))
}
return servers, nil
}

7
vendor/github.com/coredns/coredns/plugin/pkg/parse/transport.go generated vendored
View File

@ -19,6 +19,10 @@ func Transport(s string) (trans string, addr string) {
s = s[len(transport.DNS+"://"):]
return transport.DNS, s
case strings.HasPrefix(s, transport.QUIC+"://"):
s = s[len(transport.QUIC+"://"):]
return transport.QUIC, s
case strings.HasPrefix(s, transport.GRPC+"://"):
s = s[len(transport.GRPC+"://"):]
return transport.GRPC, s
@ -27,6 +31,9 @@ func Transport(s string) (trans string, addr string) {
s = s[len(transport.HTTPS+"://"):]
return transport.HTTPS, s
case strings.HasPrefix(s, transport.UNIX+"://"):
s = s[len(transport.UNIX+"://"):]
return transport.UNIX, s
}
return transport.DNS, s

4
vendor/github.com/coredns/coredns/plugin/pkg/transport/transport.go generated vendored
View File

@ -4,8 +4,10 @@ package transport
const (
DNS = "dns"
TLS = "tls"
QUIC = "quic"
GRPC = "grpc"
HTTPS = "https"
UNIX = "unix"
)
// Port numbers for the various transports.
@ -14,6 +16,8 @@ const (
Port = "53"
// TLSPort is the default port for DNS-over-TLS.
TLSPort = "853"
// QUICPort is the default port for DNS-over-QUIC.
QUICPort = "853"
// GRPCPort is the default port for DNS-over-gRPC.
GRPCPort = "443"
// HTTPSPort is the default port for DNS-over-HTTPS.

4
vendor/github.com/coredns/coredns/plugin/plugin.go generated vendored
View File

@ -108,5 +108,9 @@ var TimeBuckets = prometheus.ExponentialBuckets(0.00025, 2, 16) // from 0.25ms t
// SlimTimeBuckets is low cardinality set of duration buckets.
var SlimTimeBuckets = prometheus.ExponentialBuckets(0.00025, 10, 5) // from 0.25ms to 2.5 seconds
// NativeHistogramBucketFactor controls the resolution of Prometheus native histogram buckets.
// See: https://pkg.go.dev/github.com/prometheus/client_golang@v1.19.0/prometheus#section-readme
var NativeHistogramBucketFactor = 1.05
// ErrOnce is returned when a plugin doesn't support multiple setups per server.
var ErrOnce = errors.New("this plugin can only be used once per Server Block")

23
vendor/github.com/coredns/coredns/plugin/test/file.go generated vendored
View File

@ -3,6 +3,7 @@ package test
import (
"os"
"path/filepath"
"testing"
)
// TempFile will create a temporary file on disk and returns the name and a cleanup function to remove it later.
@ -18,12 +19,9 @@ func TempFile(dir, content string) (string, func(), error) {
return f.Name(), rmFunc, nil
}
// WritePEMFiles creates a tmp dir with ca.pem, cert.pem, and key.pem and the func to remove it
func WritePEMFiles(dir string) (string, func(), error) {
tempDir, err := os.MkdirTemp(dir, "go-test-pemfiles")
if err != nil {
return "", nil, err
}
// WritePEMFiles creates a tmp dir with ca.pem, cert.pem, and key.pem
func WritePEMFiles(t *testing.T) (string, error) {
tempDir := t.TempDir()
data := `-----BEGIN CERTIFICATE-----
MIIC9zCCAd+gAwIBAgIJALGtqdMzpDemMA0GCSqGSIb3DQEBCwUAMBIxEDAOBgNV
@ -45,7 +43,7 @@ I1rs/VUGKzcJGVIWbHrgjP68CTStGAvKgbsTqw7aLXTSqtPw88N9XVSyRg==
-----END CERTIFICATE-----`
path := filepath.Join(tempDir, "ca.pem")
if err := os.WriteFile(path, []byte(data), 0644); err != nil {
return "", nil, err
return "", err
}
data = `-----BEGIN CERTIFICATE-----
MIICozCCAYsCCQCRlf5BrvPuqjANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdr
@ -65,8 +63,8 @@ zhDEPP4FhY+Sz+y1yWirphl7A1aZwhXVPcfWIGqpQ3jzNwUeocbH27kuLh+U4hQo
qeg10RdFnw==
-----END CERTIFICATE-----`
path = filepath.Join(tempDir, "cert.pem")
if err = os.WriteFile(path, []byte(data), 0644); err != nil {
return "", nil, err
if err := os.WriteFile(path, []byte(data), 0644); err != nil {
return "", err
}
data = `-----BEGIN RSA PRIVATE KEY-----
@ -97,10 +95,9 @@ E/WObVJXDnBdViu0L9abE9iaTToBVri4cmlDlZagLuKVR+TFTCN/DSlVZTDkqkLI
8chzqtkH6b2b2R73hyRysWjsomys34ma3mEEPTX/aXeAF2MSZ/EWT9yL
-----END RSA PRIVATE KEY-----`
path = filepath.Join(tempDir, "key.pem")
if err = os.WriteFile(path, []byte(data), 0644); err != nil {
return "", nil, err
if err := os.WriteFile(path, []byte(data), 0644); err != nil {
return "", err
}
rmFunc := func() { os.RemoveAll(tempDir) }
return tempDir, rmFunc, nil
return tempDir, nil
}

4
vendor/github.com/coredns/coredns/plugin/test/helpers.go generated vendored
View File

@ -33,7 +33,11 @@ type Case struct {
Qtype uint16
Rcode int
Do bool
CheckingDisabled bool
RecursionAvailable bool
AuthenticatedData bool
Authoritative bool
Truncated bool
Answer []dns.RR
Ns []dns.RR
Extra []dns.RR

3
vendor/github.com/coredns/coredns/plugin/test/scrape.go generated vendored
View File

@ -19,7 +19,6 @@
//
// result := Scrape("http://localhost:9153/metrics")
// v := MetricValue("coredns_cache_capacity", result)
//
package test
import (
@ -217,7 +216,7 @@ func makeBuckets(m *dto.Metric) map[string]string {
func fetchMetricFamilies(url string, ch chan<- *dto.MetricFamily) {
defer close(ch)
req, err := http.NewRequest("GET", url, nil)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return
}

25
vendor/github.com/gobwas/ws/.travis.yml generated vendored
View File

@ -1,25 +0,0 @@
sudo: required
language: go
services:
- docker
os:
- linux
- windows
go:
- 1.8.x
- 1.9.x
- 1.10.x
- 1.11.x
- 1.x
install:
- go get github.com/gobwas/pool
- go get github.com/gobwas/httphead
script:
- if [ "$TRAVIS_OS_NAME" = "windows" ]; then go test ./...; fi
- if [ "$TRAVIS_OS_NAME" = "linux" ]; then make test autobahn; fi

2
vendor/github.com/gobwas/ws/LICENSE generated vendored
View File

@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2017-2018 Sergey Kamardin <gobwas@gmail.com>
Copyright (c) 2017-2021 Sergey Kamardin <gobwas@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

11
vendor/github.com/gobwas/ws/Makefile generated vendored
View File

@ -13,15 +13,22 @@ bin/gocovmerge:
.PHONY: autobahn
autobahn: clean bin/reporter
./autobahn/script/test.sh --build
./autobahn/script/test.sh --build --follow-logs
bin/reporter $(PWD)/autobahn/report/index.json
.PHONY: autobahn/report
autobahn/report: bin/reporter
./bin/reporter -http localhost:5555 ./autobahn/report/index.json
test:
go test -coverprofile=ws.coverage .
go test -coverprofile=wsutil.coverage ./wsutil
go test -coverprofile=wsfalte.coverage ./wsflate
# No statemenets to cover in ./tests (there are only tests).
go test ./tests
cover: bin/gocovmerge test autobahn
bin/gocovmerge ws.coverage wsutil.coverage autobahn/report/server.coverage > total.coverage
bin/gocovmerge ws.coverage wsutil.coverage wsflate.coverage autobahn/report/server.coverage > total.coverage
benchcmp: BENCH_BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
benchcmp: BENCH_OLD:=$(shell mktemp -t old.XXXX)

187
vendor/github.com/gobwas/ws/README.md generated vendored
View File

@ -1,7 +1,7 @@
# ws
[![GoDoc][godoc-image]][godoc-url]
[![Travis][travis-image]][travis-url]
[![CI][ci-badge]][ci-url]
> [RFC6455][rfc-url] WebSocket implementation in Go.
@ -351,10 +351,191 @@ func main() {
}
```
# Compression
There is a `ws/wsflate` package to support [Permessage-Deflate Compression
Extension][rfc-pmce].
It provides minimalistic I/O wrappers to be used in conjunction with any
deflate implementation (for example, the standard library's
[compress/flate][compress/flate]).
It is also compatible with `wsutil`'s reader and writer by providing
`wsflate.MessageState` type, which implements `wsutil.SendExtension` and
`wsutil.RecvExtension` interfaces.
```go
package main
import (
"bytes"
"log"
"net"
"github.com/gobwas/ws"
"github.com/gobwas/ws/wsflate"
)
func main() {
ln, err := net.Listen("tcp", "localhost:8080")
if err != nil {
// handle error
}
e := wsflate.Extension{
// We are using default parameters here since we use
// wsflate.{Compress,Decompress}Frame helpers below in the code.
// This assumes that we use standard compress/flate package as flate
// implementation.
Parameters: wsflate.DefaultParameters,
}
u := ws.Upgrader{
Negotiate: e.Negotiate,
}
for {
conn, err := ln.Accept()
if err != nil {
log.Fatal(err)
}
// Reset extension after previous upgrades.
e.Reset()
_, err = u.Upgrade(conn)
if err != nil {
log.Printf("upgrade error: %s", err)
continue
}
if _, ok := e.Accepted(); !ok {
log.Printf("didn't negotiate compression for %s", conn.RemoteAddr())
conn.Close()
continue
}
go func() {
defer conn.Close()
for {
frame, err := ws.ReadFrame(conn)
if err != nil {
// Handle error.
return
}
frame = ws.UnmaskFrameInPlace(frame)
if wsflate.IsCompressed(frame.Header) {
// Note that even after successful negotiation of
// compression extension, both sides are able to send
// non-compressed messages.
frame, err = wsflate.DecompressFrame(frame)
if err != nil {
// Handle error.
return
}
}
// Do something with frame...
ack := ws.NewTextFrame([]byte("this is an acknowledgement"))
// Compress response unconditionally.
ack, err = wsflate.CompressFrame(ack)
if err != nil {
// Handle error.
return
}
if err = ws.WriteFrame(conn, ack); err != nil {
// Handle error.
return
}
}
}()
}
}
```
You can use compression with `wsutil` package this way:
```go
// Upgrade somehow and negotiate compression to get the conn...
// Initialize flate reader. We are using nil as a source io.Reader because
// we will Reset() it in the message i/o loop below.
fr := wsflate.NewReader(nil, func(r io.Reader) wsflate.Decompressor {
return flate.NewReader(r)
})
// Initialize flate writer. We are using nil as a destination io.Writer
// because we will Reset() it in the message i/o loop below.
fw := wsflate.NewWriter(nil, func(w io.Writer) wsflate.Compressor {
f, _ := flate.NewWriter(w, 9)
return f
})
// Declare compression message state variable.
//
// It has two goals:
// - Allow users to check whether received message is compressed or not.
// - Help wsutil.Reader and wsutil.Writer to set/unset appropriate
// WebSocket header bits while writing next frame to the wire (it
// implements wsutil.RecvExtension and wsutil.SendExtension).
var msg wsflate.MessageState
// Initialize WebSocket reader as previously.
// Please note the use of Reader.Extensions field as well as
// of ws.StateExtended flag.
rd := &wsutil.Reader{
Source: conn,
State: ws.StateServerSide | ws.StateExtended,
Extensions: []wsutil.RecvExtension{
&msg,
},
}
// Initialize WebSocket writer with ws.StateExtended flag as well.
wr := wsutil.NewWriter(conn, ws.StateServerSide|ws.StateExtended, 0)
// Use the message state as wsutil.SendExtension.
wr.SetExtensions(&msg)
for {
h, err := rd.NextFrame()
if err != nil {
// handle error.
}
if h.OpCode.IsControl() {
// handle control frame.
}
if !msg.IsCompressed() {
// handle uncompressed frame (skipped for the sake of example
// simplicity).
}
// Reset the writer to echo same op code.
wr.Reset(h.OpCode)
// Reset both flate reader and writer to start the new round of i/o.
fr.Reset(rd)
fw.Reset(wr)
// Copy whole message from reader to writer decompressing it and
// compressing again.
if _, err := io.Copy(fw, fr); err != nil {
// handle error.
}
// Flush any remaining buffers from flate writer to WebSocket writer.
if err := fw.Close(); err != nil {
// handle error.
}
// Flush the whole WebSocket message to the wire.
if err := wr.Flush(); err != nil {
// handle error.
}
}
```
[rfc-url]: https://tools.ietf.org/html/rfc6455
[rfc-pmce]: https://tools.ietf.org/html/rfc7692#section-7
[godoc-image]: https://godoc.org/github.com/gobwas/ws?status.svg
[godoc-url]: https://godoc.org/github.com/gobwas/ws
[travis-image]: https://travis-ci.org/gobwas/ws.svg?branch=master
[travis-url]: https://travis-ci.org/gobwas/ws
[compress/flate]: https://golang.org/pkg/compress/flate/
[ci-badge]: https://github.com/gobwas/ws/workflows/CI/badge.svg
[ci-url]: https://github.com/gobwas/ws/actions?query=workflow%3ACI

2
vendor/github.com/gobwas/ws/cipher.go generated vendored
View File

@ -36,7 +36,7 @@ func Cipher(payload []byte, mask [4]byte, offset int) {
}
// NOTE: we use here binary.LittleEndian regardless of what is real
// endianess on machine is. To do so, we have to use binary.LittleEndian in
// endianness on machine is. To do so, we have to use binary.LittleEndian in
// the masking loop below as well.
var (
m = binary.LittleEndian.Uint32(mask[:])

58
vendor/github.com/gobwas/ws/dialer.go generated vendored
View File

@ -8,6 +8,7 @@ import (
"fmt"
"io"
"net"
"net/http"
"net/url"
"strconv"
"strings"
@ -145,7 +146,7 @@ type Dialer struct {
func (d Dialer) Dial(ctx context.Context, urlstr string) (conn net.Conn, br *bufio.Reader, hs Handshake, err error) {
u, err := url.ParseRequestURI(urlstr)
if err != nil {
return
return nil, nil, hs, err
}
// Prepare context to dial with. Initially it is the same as original, but
@ -163,7 +164,7 @@ func (d Dialer) Dial(ctx context.Context, urlstr string) (conn net.Conn, br *buf
}
}
if conn, err = d.dial(dialctx, u); err != nil {
return
return conn, nil, hs, err
}
defer func() {
if err != nil {
@ -189,7 +190,7 @@ func (d Dialer) Dial(ctx context.Context, urlstr string) (conn net.Conn, br *buf
br, hs, err = d.Upgrade(conn, u)
return
return conn, br, hs, err
}
var (
@ -204,7 +205,7 @@ func tlsDefaultConfig() *tls.Config {
return &tlsEmptyConfig
}
func hostport(host string, defaultPort string) (hostname, addr string) {
func hostport(host, defaultPort string) (hostname, addr string) {
var (
colon = strings.LastIndexByte(host, ':')
bracket = strings.IndexByte(host, ']')
@ -228,7 +229,7 @@ func (d Dialer) dial(ctx context.Context, u *url.URL) (conn net.Conn, err error)
hostname, addr := hostport(u.Host, ":443")
conn, err = dial(ctx, "tcp", addr)
if err != nil {
return
return nil, err
}
tlsClient := d.TLSClient
if tlsClient == nil {
@ -241,7 +242,7 @@ func (d Dialer) dial(ctx context.Context, u *url.URL) (conn net.Conn, err error)
if wrap := d.WrapConn; wrap != nil {
conn = wrap(conn)
}
return
return conn, err
}
func (d Dialer) tlsClient(conn net.Conn, hostname string) net.Conn {
@ -310,29 +311,29 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
initNonce(nonce)
httpWriteUpgradeRequest(bw, u, nonce, d.Protocols, d.Extensions, d.Header)
if err = bw.Flush(); err != nil {
return
if err := bw.Flush(); err != nil {
return br, hs, err
}
// Read HTTP status line like "HTTP/1.1 101 Switching Protocols".
sl, err := readLine(br)
if err != nil {
return
return br, hs, err
}
// Begin validation of the response.
// See https://tools.ietf.org/html/rfc6455#section-4.2.2
// Parse request line data like HTTP version, uri and method.
resp, err := httpParseResponseLine(sl)
if err != nil {
return
return br, hs, err
}
// Even if RFC says "1.1 or higher" without mentioning the part of the
// version, we apply it only to minor part.
if resp.major != 1 || resp.minor < 1 {
err = ErrHandshakeBadProtocol
return
return br, hs, err
}
if resp.status != 101 {
if resp.status != http.StatusSwitchingProtocols {
err = StatusError(resp.status)
if onStatusError := d.OnStatusError; onStatusError != nil {
// Invoke callback with multireader of status-line bytes br.
@ -344,7 +345,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
),
)
}
return
return br, hs, err
}
// If response status is 101 then we expect all technical headers to be
// valid. If not, then we stop processing response without giving user
@ -355,7 +356,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
line, e := readLine(br)
if e != nil {
err = e
return
return br, hs, err
}
if len(line) == 0 {
// Blank line, no more lines to read.
@ -365,7 +366,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
k, v, ok := httpParseHeaderLine(line)
if !ok {
err = ErrMalformedResponse
return
return br, hs, err
}
switch btsToString(k) {
@ -373,7 +374,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
headerSeen |= headerSeenUpgrade
if !bytes.Equal(v, specHeaderValueUpgrade) && !bytes.EqualFold(v, specHeaderValueUpgrade) {
err = ErrHandshakeBadUpgrade
return
return br, hs, err
}
case headerConnectionCanonical:
@ -384,14 +385,14 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
// multiple token. But in response it must contains exactly one.
if !bytes.Equal(v, specHeaderValueConnection) && !bytes.EqualFold(v, specHeaderValueConnection) {
err = ErrHandshakeBadConnection
return
return br, hs, err
}
case headerSecAcceptCanonical:
headerSeen |= headerSeenSecAccept
if !checkAcceptFromNonce(v, nonce) {
err = ErrHandshakeBadSecAccept
return
return br, hs, err
}
case headerSecProtocolCanonical:
@ -409,20 +410,20 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
// Server echoed subprotocol that is not present in client
// requested protocols.
err = ErrHandshakeBadSubProtocol
return
return br, hs, err
}
case headerSecExtensionsCanonical:
hs.Extensions, err = matchSelectedExtensions(v, d.Extensions, hs.Extensions)
if err != nil {
return
return br, hs, err
}
default:
if onHeader := d.OnHeader; onHeader != nil {
if e := onHeader(k, v); e != nil {
err = e
return
return br, hs, err
}
}
}
@ -439,7 +440,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha
panic("unknown headers state")
}
}
return
return br, hs, err
}
// PutReader returns bufio.Reader instance to the inner reuse pool.
@ -474,10 +475,19 @@ func matchSelectedExtensions(selected []byte, wanted, received []httphead.Option
index = -1
match := func() (ok bool) {
for _, want := range wanted {
if option.Equal(want) {
// A server accepts one or more extensions by including a
// |Sec-WebSocket-Extensions| header field containing one or more
// extensions that were requested by the client.
//
// The interpretation of any extension parameters, and what
// constitutes a valid response by a server to a requested set of
// parameters by a client, will be defined by each such extension.
if bytes.Equal(option.Name, want.Name) {
// Check parsed extension to be present in client
// requested extensions. We move matched extension
// from client list to avoid allocation.
// from client list to avoid allocation of httphead.Option.Name,
// httphead.Option.Parameters have to be copied from the header
want.Parameters, _ = option.Parameters.Copy(make([]byte, option.Parameters.Size()))
received = append(received, want)
return true
}

1
vendor/github.com/gobwas/ws/dialer_tls_go18.go generated vendored
View File

@ -1,3 +1,4 @@
//go:build go1.8
// +build go1.8
package ws

29
vendor/github.com/gobwas/ws/errors.go generated vendored
View File

@ -2,12 +2,12 @@ package ws
// RejectOption represents an option used to control the way connection is
// rejected.
type RejectOption func(*rejectConnectionError)
type RejectOption func(*ConnectionRejectedError)
// RejectionReason returns an option that makes connection to be rejected with
// given reason.
func RejectionReason(reason string) RejectOption {
return func(err *rejectConnectionError) {
return func(err *ConnectionRejectedError) {
err.reason = reason
}
}
@ -15,7 +15,7 @@ func RejectionReason(reason string) RejectOption {
// RejectionStatus returns an option that makes connection to be rejected with
// given HTTP status code.
func RejectionStatus(code int) RejectOption {
return func(err *rejectConnectionError) {
return func(err *ConnectionRejectedError) {
err.code = code
}
}
@ -23,32 +23,37 @@ func RejectionStatus(code int) RejectOption {
// RejectionHeader returns an option that makes connection to be rejected with
// given HTTP headers.
func RejectionHeader(h HandshakeHeader) RejectOption {
return func(err *rejectConnectionError) {
return func(err *ConnectionRejectedError) {
err.header = h
}
}
// RejectConnectionError constructs an error that could be used to control the way
// handshake is rejected by Upgrader.
// RejectConnectionError constructs an error that could be used to control the
// way handshake is rejected by Upgrader.
func RejectConnectionError(options ...RejectOption) error {
err := new(rejectConnectionError)
err := new(ConnectionRejectedError)
for _, opt := range options {
opt(err)
}
return err
}
// rejectConnectionError represents a rejection of upgrade error.
// ConnectionRejectedError represents a rejection of connection during
// WebSocket handshake error.
//
// It can be returned by Upgrader's On* hooks to control the way WebSocket
// handshake is rejected.
type rejectConnectionError struct {
// It can be returned by Upgrader's On* hooks to indicate that WebSocket
// handshake should be rejected.
type ConnectionRejectedError struct {
reason string
code int
header HandshakeHeader
}
// Error implements error interface.
func (r *rejectConnectionError) Error() string {
func (r *ConnectionRejectedError) Error() string {
return r.reason
}
func (r *ConnectionRejectedError) StatusCode() int {
return r.code
}

Some files were not shown because too many files have changed in this diff Show More