diff --git a/connection/errors.go b/connection/errors.go index 17cf58a3..1bb34d6d 100644 --- a/connection/errors.go +++ b/connection/errors.go @@ -2,7 +2,6 @@ package connection import ( "github.com/cloudflare/cloudflared/edgediscovery" - "github.com/cloudflare/cloudflared/h2mux" tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs" ) @@ -71,8 +70,6 @@ func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) b switch err.(type) { case edgediscovery.DialError: log.Error().Msg("Connection unable to dial edge") - case h2mux.MuxerHandshakeError: - log.Error().Msg("Connection handshake with edge server failed") default: log.Error().Msg("Connection failed") return false diff --git a/connection/h2mux.go b/connection/h2mux.go deleted file mode 100644 index 4de983bc..00000000 --- a/connection/h2mux.go +++ /dev/null @@ -1,32 +0,0 @@ -package connection - -import ( - "time" - - "github.com/rs/zerolog" - - "github.com/cloudflare/cloudflared/h2mux" -) - -const ( - muxerTimeout = 5 * time.Second -) - -type MuxerConfig struct { - HeartbeatInterval time.Duration - MaxHeartbeats uint64 - CompressionSetting h2mux.CompressionSetting - MetricsUpdateFreq time.Duration -} - -func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Logger) *h2mux.MuxerConfig { - return &h2mux.MuxerConfig{ - Timeout: muxerTimeout, - Handler: h, - IsClient: true, - HeartbeatInterval: mc.HeartbeatInterval, - MaxHeartbeats: mc.MaxHeartbeats, - Log: log, - CompressionQuality: mc.CompressionSetting, - } -} diff --git a/connection/h2mux_header.go b/connection/h2mux_header.go deleted file mode 100644 index 3987f0db..00000000 --- a/connection/h2mux_header.go +++ /dev/null @@ -1,128 +0,0 @@ -package connection - -import ( - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/cloudflare/cloudflared/h2mux" -) - -// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld -// to an HTTP/1 Request object destined for the local origin web service. -// This operation includes conversion of the pseudo-headers into their closest -// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3 -func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error { - for _, header := range h2 { - name := strings.ToLower(header.Name) - if !IsH2muxControlRequestHeader(name) { - continue - } - - switch name { - case ":method": - h1.Method = header.Value - case ":scheme": - // noop - use the preexisting scheme from h1.URL - case ":authority": - // Otherwise the host header will be based on the origin URL - h1.Host = header.Value - case ":path": - // We don't want to be an "opinionated" proxy, so ideally we would use :path as-is. - // However, this HTTP/1 Request object belongs to the Go standard library, - // whose URL package makes some opinionated decisions about the encoding of - // URL characters: see the docs of https://godoc.org/net/url#URL, - // in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath, - // which is always used when computing url.URL.String(), whether we'd like it or not. - // - // Well, not *always*. We could circumvent this by using url.URL.Opaque. But - // that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque - // is treated differently when HTTP_PROXY is set! - // See https://github.com/golang/go/issues/5684#issuecomment-66080888 - // - // This means we are subject to the behavior of net/url's function `shouldEscape` - // (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101 - - if header.Value == "*" { - h1.URL.Path = "*" - continue - } - // Due to the behavior of validation.ValidateUrl, h1.URL may - // already have a partial value, with or without a trailing slash. - base := h1.URL.String() - base = strings.TrimRight(base, "/") - // But we know :path begins with '/', because we handled '*' above - see RFC7540 - requestURL, err := url.Parse(base + header.Value) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value)) - } - h1.URL = requestURL - case "content-length": - contentLength, err := strconv.ParseInt(header.Value, 10, 64) - if err != nil { - return fmt.Errorf("unparseable content length") - } - h1.ContentLength = contentLength - case RequestUserHeaders: - // Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version - // Find and parse user headers serialized into a single one - userHeaders, err := DeserializeHeaders(header.Value) - if err != nil { - return errors.Wrap(err, "Unable to parse user headers") - } - for _, userHeader := range userHeaders { - h1.Header.Add(userHeader.Name, userHeader.Value) - } - default: - // All other control headers shall just be proxied transparently - h1.Header.Add(header.Name, header.Value) - } - } - - return nil -} - -func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) { - h2 = []h2mux.Header{ - {Name: ":status", Value: strconv.Itoa(status)}, - } - userHeaders := make(http.Header, len(h1)) - for header, values := range h1 { - h2name := strings.ToLower(header) - if h2name == "content-length" { - // This header has meaning in HTTP/2 and will be used by the edge, - // so it should be sent as an HTTP/2 response header. - - // Since these are http2 headers, they're required to be lowercase - h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]}) - } else if !IsH2muxControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) { - // User headers, on the other hand, must all be serialized so that - // HTTP/2 header validation won't be applied to HTTP/1 header values - userHeaders[header] = values - } - } - - // Perform user header serialization and set them in the single header - h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)}) - return h2 -} - -// IsH2muxControlRequestHeader is called in the direction of eyeball -> origin. -func IsH2muxControlRequestHeader(headerName string) bool { - return headerName == "content-length" || - headerName == "connection" || headerName == "upgrade" || // Websocket request headers - strings.HasPrefix(headerName, ":") || - strings.HasPrefix(headerName, "cf-") -} - -// IsH2muxControlResponseHeader is called in the direction of eyeball <- origin. -func IsH2muxControlResponseHeader(headerName string) bool { - return headerName == "content-length" || - strings.HasPrefix(headerName, ":") || - strings.HasPrefix(headerName, "cf-int-") || - strings.HasPrefix(headerName, "cf-cloudflared-") -} diff --git a/connection/h2mux_header_test.go b/connection/h2mux_header_test.go deleted file mode 100644 index a78e02f4..00000000 --- a/connection/h2mux_header_test.go +++ /dev/null @@ -1,642 +0,0 @@ -package connection - -import ( - "fmt" - "math/rand" - "net/http" - "net/url" - "reflect" - "regexp" - "strings" - "testing" - "testing/quick" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/cloudflare/cloudflared/h2mux" -) - -type ByName []h2mux.Header - -func (a ByName) Len() int { return len(a) } -func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByName) Less(i, j int) bool { - if a[i].Name == a[j].Name { - return a[i].Value < a[j].Value - } - - return a[i].Name < a[j].Name -} - -func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) { - request, err := http.NewRequest(http.MethodGet, "http://example.com", nil) - assert.NoError(t, err) - - mockHeaders := http.Header{ - "Mock header 1": {"Mock value 1"}, - "Mock header 2": {"Mock value 2"}, - } - - headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request) - - assert.True(t, reflect.DeepEqual(mockHeaders, request.Header)) - assert.NoError(t, headersConversionErr) -} - -func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header { - return []h2mux.Header{{ - Name: headersField, - Value: SerializeHeaders(headers), - }} -} - -func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) { - request, err := http.NewRequest(http.MethodGet, "http://example.com", nil) - assert.NoError(t, err) - - emptyHeaders := make(http.Header) - headersConversionErr := H2RequestHeadersToH1Request( - []h2mux.Header{{ - Name: RequestUserHeaders, - Value: SerializeHeaders(emptyHeaders), - }}, - request, - ) - - assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header)) - assert.NoError(t, headersConversionErr) -} - -func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) { - request, err := http.NewRequest(http.MethodGet, "http://example.com", nil) - assert.NoError(t, err) - - mockRequestHeaders := []h2mux.Header{ - {Name: ":path", Value: "//bad_path/"}, - {Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})}, - } - - headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request) - - assert.Equal(t, http.Header{ - "Mock header": []string{"Mock value"}, - }, request.Header) - - assert.Equal(t, "http://example.com//bad_path/", request.URL.String()) - - assert.NoError(t, headersConversionErr) -} - -func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) { - request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) - assert.NoError(t, err) - - mockRequestHeaders := []h2mux.Header{ - {Name: ":path", Value: "/?query=mock%20value"}, - {Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})}, - } - - headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request) - - assert.Equal(t, http.Header{ - "Mock header": []string{"Mock value"}, - }, request.Header) - - assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String()) - - assert.NoError(t, headersConversionErr) -} - -func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) { - request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) - assert.NoError(t, err) - - mockRequestHeaders := []h2mux.Header{ - {Name: ":path", Value: "/mock%20path"}, - {Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})}, - } - - headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request) - - assert.Equal(t, http.Header{ - "Mock header": []string{"Mock value"}, - }, request.Header) - - assert.Equal(t, "http://example.com/mock%20path", request.URL.String()) - - assert.NoError(t, headersConversionErr) -} - -func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) { - type testCase struct { - path string - want string - } - testCases := []testCase{ - { - path: "", - want: "", - }, - { - path: "/", - want: "/", - }, - { - path: "//", - want: "//", - }, - { - path: "/test", - want: "/test", - }, - { - path: "//test", - want: "//test", - }, - { - // https://github.com/cloudflare/cloudflared/issues/81 - path: "//test/", - want: "//test/", - }, - { - path: "/%2Ftest", - want: "/%2Ftest", - }, - { - path: "//%20test", - want: "//%20test", - }, - { - // https://github.com/cloudflare/cloudflared/issues/124 - path: "/test?get=somthing%20a", - want: "/test?get=somthing%20a", - }, - { - path: "/%20", - want: "/%20", - }, - { - // stdlib's EscapedPath() will always percent-encode ' ' - path: "/ ", - want: "/%20", - }, - { - path: "/ a ", - want: "/%20a%20", - }, - { - path: "/a%20b", - want: "/a%20b", - }, - { - path: "/foo/bar;param?query#frag", - want: "/foo/bar;param?query#frag", - }, - { - // stdlib's EscapedPath() will always percent-encode non-ASCII chars - path: "/a␠b", - want: "/a%E2%90%A0b", - }, - { - path: "/a-umlaut-ä", - want: "/a-umlaut-%C3%A4", - }, - { - path: "/a-umlaut-%C3%A4", - want: "/a-umlaut-%C3%A4", - }, - { - path: "/a-umlaut-%c3%a4", - want: "/a-umlaut-%c3%a4", - }, - { - // here the second '#' is treated as part of the fragment - path: "/a#b#c", - want: "/a#b%23c", - }, - { - path: "/a#b␠c", - want: "/a#b%E2%90%A0c", - }, - { - path: "/a#b%20c", - want: "/a#b%20c", - }, - { - path: "/a#b c", - want: "/a#b%20c", - }, - { - // stdlib's EscapedPath() will always percent-encode '\' - path: "/\\", - want: "/%5C", - }, - { - path: "/a\\", - want: "/a%5C", - }, - { - path: "/a,b.c.", - want: "/a,b.c.", - }, - { - path: "/.", - want: "/.", - }, - { - // stdlib's EscapedPath() will always percent-encode '`' - path: "/a`", - want: "/a%60", - }, - { - path: "/a[0]", - want: "/a[0]", - }, - { - path: "/?a[0]=5 &b[]=", - want: "/?a[0]=5 &b[]=", - }, - { - path: "/?a=%22b%20%22", - want: "/?a=%22b%20%22", - }, - } - - for index, testCase := range testCases { - requestURL := "https://example.com" - - request, err := http.NewRequest(http.MethodGet, requestURL, nil) - assert.NoError(t, err) - - mockRequestHeaders := []h2mux.Header{ - {Name: ":path", Value: testCase.path}, - {Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})}, - } - - headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request) - assert.NoError(t, headersConversionErr) - - assert.Equal(t, - http.Header{ - "Mock header": []string{"Mock value"}, - }, - request.Header) - - assert.Equal(t, - "https://example.com"+testCase.want, - request.URL.String(), - "Failed URL index: %v %#v", index, testCase) - } -} - -func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) { - config := &quick.Config{ - Values: func(args []reflect.Value, rand *rand.Rand) { - args[0] = reflect.ValueOf(randomHTTP2Path(t, rand)) - }, - } - - type testOrigin struct { - url string - - expectedScheme string - expectedBasePath string - } - testOrigins := []testOrigin{ - { - url: "http://origin.hostname.example.com:8080", - expectedScheme: "http", - expectedBasePath: "http://origin.hostname.example.com:8080", - }, - { - url: "http://origin.hostname.example.com:8080/", - expectedScheme: "http", - expectedBasePath: "http://origin.hostname.example.com:8080", - }, - { - url: "http://origin.hostname.example.com:8080/api", - expectedScheme: "http", - expectedBasePath: "http://origin.hostname.example.com:8080/api", - }, - { - url: "http://origin.hostname.example.com:8080/api/", - expectedScheme: "http", - expectedBasePath: "http://origin.hostname.example.com:8080/api", - }, - { - url: "https://origin.hostname.example.com:8080/api", - expectedScheme: "https", - expectedBasePath: "https://origin.hostname.example.com:8080/api", - }, - } - - // use multiple schemes to demonstrate that the URL is based on the - // origin's scheme, not the :scheme header - for _, testScheme := range []string{"http", "https"} { - for _, testOrigin := range testOrigins { - assertion := func(testPath string) bool { - const expectedMethod = "POST" - const expectedHostname = "request.hostname.example.com" - - h2 := []h2mux.Header{ - {Name: ":method", Value: expectedMethod}, - {Name: ":scheme", Value: testScheme}, - {Name: ":authority", Value: expectedHostname}, - {Name: ":path", Value: testPath}, - {Name: RequestUserHeaders, Value: ""}, - } - h1, err := http.NewRequest("GET", testOrigin.url, nil) - require.NoError(t, err) - - err = H2RequestHeadersToH1Request(h2, h1) - return assert.NoError(t, err) && - assert.Equal(t, expectedMethod, h1.Method) && - assert.Equal(t, expectedHostname, h1.Host) && - assert.Equal(t, testOrigin.expectedScheme, h1.URL.Scheme) && - assert.Equal(t, testOrigin.expectedBasePath+testPath, h1.URL.String()) - } - err := quick.Check(assertion, config) - assert.NoError(t, err) - } - } -} - -func randomASCIIPrintableChar(rand *rand.Rand) int { - // smallest printable ASCII char is 32, largest is 126 - const startPrintable = 32 - const endPrintable = 127 - return startPrintable + rand.Intn(endPrintable-startPrintable) -} - -// randomASCIIText generates an ASCII string, some of whose characters may be -// percent-encoded. Its "logical length" (ignoring percent-encoding) is -// between 1 and `maxLength`. -func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string { - length := minLength + rand.Intn(maxLength) - var result strings.Builder - for i := 0; i < length; i++ { - c := randomASCIIPrintableChar(rand) - - // 1/4 chance of using percent encoding when not necessary - if c == '%' || rand.Intn(4) == 0 { - result.WriteString(fmt.Sprintf("%%%02X", c)) - } else { - result.WriteByte(byte(c)) - } - } - return result.String() -} - -// Calls `randomASCIIText` and ensures the result is a valid URL path, -// i.e. one that can pass unchanged through url.URL.String() -func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string { - text := randomASCIIText(rand, minLength, maxLength) - re, err := regexp.Compile("[^/;,]*") - require.NoError(t, err) - return "/" + re.ReplaceAllStringFunc(text, url.PathEscape) -} - -// Calls `randomASCIIText` and ensures the result is a valid URL query, -// i.e. one that can pass unchanged through url.URL.String() -func randomHTTP1Query(rand *rand.Rand, minLength int, maxLength int) string { - text := randomASCIIText(rand, minLength, maxLength) - return "?" + strings.ReplaceAll(text, "#", "%23") -} - -// Calls `randomASCIIText` and ensures the result is a valid URL fragment, -// i.e. one that can pass unchanged through url.URL.String() -func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string { - text := randomASCIIText(rand, minLength, maxLength) - u, err := url.Parse("#" + text) - require.NoError(t, err) - return u.String() -} - -// Assemble a random :path pseudoheader that is legal by Go stdlib standards -// (i.e. all characters will satisfy "net/url".shouldEscape for their respective locations) -func randomHTTP2Path(t *testing.T, rand *rand.Rand) string { - result := randomHTTP1Path(t, rand, 1, 64) - if rand.Intn(2) == 1 { - result += randomHTTP1Query(rand, 1, 32) - } - if rand.Intn(2) == 1 { - result += randomHTTP1Fragment(t, rand, 1, 16) - } - return result -} - -func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) { - for name, values := range headers { - for _, value := range values { - h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value}) - } - } - - return h2muxHeaders -} - -func TestParseRequestHeaders(t *testing.T) { - mockUserHeadersToSerialize := http.Header{ - "Mock-Header-One": {"1", "1.5"}, - "Mock-Header-Two": {"2"}, - "Mock-Header-Three": {"3"}, - } - - mockHeaders := []h2mux.Header{ - {Name: "One", Value: "1"}, // will be dropped - {Name: "Cf-Two", Value: "cf-value-1"}, - {Name: "Cf-Two", Value: "cf-value-2"}, - {Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)}, - } - - expectedHeaders := []h2mux.Header{ - {Name: "Cf-Two", Value: "cf-value-1"}, - {Name: "Cf-Two", Value: "cf-value-2"}, - {Name: "Mock-Header-One", Value: "1"}, - {Name: "Mock-Header-One", Value: "1.5"}, - {Name: "Mock-Header-Two", Value: "2"}, - {Name: "Mock-Header-Three", Value: "3"}, - } - h1 := &http.Request{ - Header: make(http.Header), - } - err := H2RequestHeadersToH1Request(mockHeaders, h1) - assert.NoError(t, err) - assert.ElementsMatch(t, expectedHeaders, stdlibHeaderToH2muxHeader(h1.Header)) -} - -func TestIsH2muxControlRequestHeader(t *testing.T) { - controlRequestHeaders := []string{ - // Anything that begins with cf- - "cf-sample-header", - - // Any http2 pseudoheader - ":sample-pseudo-header", - - // content-length is a special case, it has to be there - // for some requests to work (per the HTTP2 spec) - "content-length", - - // Websocket request headers - "connection", - "upgrade", - } - - for _, header := range controlRequestHeaders { - assert.True(t, IsH2muxControlRequestHeader(header)) - } -} - -func TestIsH2muxControlResponseHeader(t *testing.T) { - controlResponseHeaders := []string{ - // Anything that begins with cf-int- or cf-cloudflared- - "cf-int-sample-header", - "cf-cloudflared-sample-header", - - // Any http2 pseudoheader - ":sample-pseudo-header", - - // content-length is a special case, it has to be there - // for some requests to work (per the HTTP2 spec) - "content-length", - } - - for _, header := range controlResponseHeaders { - assert.True(t, IsH2muxControlResponseHeader(header)) - } -} - -func TestIsNotH2muxControlRequestHeader(t *testing.T) { - notControlRequestHeaders := []string{ - "mock-header", - "another-sample-header", - } - - for _, header := range notControlRequestHeaders { - assert.False(t, IsH2muxControlRequestHeader(header)) - } -} - -func TestIsNotH2muxControlResponseHeader(t *testing.T) { - notControlResponseHeaders := []string{ - "mock-header", - "another-sample-header", - "upgrade", - "connection", - "cf-whatever", // On the response path, we only want to filter cf-int- and cf-cloudflared- - } - - for _, header := range notControlResponseHeaders { - assert.False(t, IsH2muxControlResponseHeader(header)) - } -} - -func TestH1ResponseToH2ResponseHeaders(t *testing.T) { - mockHeaders := http.Header{ - "User-header-one": {""}, - "User-header-two": {"1", "2"}, - "cf-header": {"cf-value"}, - "cf-int-header": {"cf-int-value"}, - "cf-cloudflared-header": {"cf-cloudflared-value"}, - "Content-Length": {"123"}, - } - mockResponse := http.Response{ - StatusCode: 200, - Header: mockHeaders, - } - - headers := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header) - - serializedHeadersIndex := -1 - for i, header := range headers { - if header.Name == ResponseUserHeaders { - serializedHeadersIndex = i - break - } - } - assert.NotEqual(t, -1, serializedHeadersIndex) - actualControlHeaders := append( - headers[:serializedHeadersIndex], - headers[serializedHeadersIndex+1:]..., - ) - expectedControlHeaders := []h2mux.Header{ - {Name: ":status", Value: "200"}, - {Name: "content-length", Value: "123"}, - } - - assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders) - - actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value) - expectedUserHeaders := []h2mux.Header{ - {Name: "User-header-one", Value: ""}, - {Name: "User-header-two", Value: "1"}, - {Name: "User-header-two", Value: "2"}, - {Name: "cf-header", Value: "cf-value"}, - } - assert.NoError(t, err) - assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders) -} - -// The purpose of this test is to check that our code and the http.Header -// implementation don't throw validation errors about header size -func TestHeaderSize(t *testing.T) { - largeValue := randSeq(5 * 1024 * 1024) // 5Mb - largeHeaders := http.Header{ - "User-header": {largeValue}, - } - mockResponse := http.Response{ - StatusCode: 200, - Header: largeHeaders, - } - - serializedHeaders := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header) - request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil) - assert.NoError(t, err) - for _, header := range serializedHeaders { - request.Header.Set(header.Name, header.Value) - } - - for _, header := range serializedHeaders { - if header.Name != ResponseUserHeaders { - continue - } - - deserializedHeaders, err := DeserializeHeaders(header.Value) - assert.NoError(t, err) - assert.Equal(t, largeValue, deserializedHeaders[0].Value) - } -} - -func randSeq(n int) string { - randomizer := rand.New(rand.NewSource(17)) - var letters = []rune(":;,+/=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - b := make([]rune, n) - for i := range b { - b[i] = letters[randomizer.Intn(len(letters))] - } - return string(b) -} - -func BenchmarkH1ResponseToH2ResponseHeaders(b *testing.B) { - ser := "eC1mb3J3YXJkZWQtcHJvdG8:aHR0cHM;dXBncmFkZS1pbnNlY3VyZS1yZXF1ZXN0cw:MQ;YWNjZXB0LWxhbmd1YWdl:ZW4tVVMsZW47cT0wLjkscnU7cT0wLjg;YWNjZXB0LWVuY29kaW5n:Z3ppcA;eC1mb3J3YXJkZWQtZm9y:MTczLjI0NS42MC42;dXNlci1hZ2VudA:TW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTBfMTRfNikgQXBwbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzg0LjAuNDE0Ny44OSBTYWZhcmkvNTM3LjM2;c2VjLWZldGNoLW1vZGU:bmF2aWdhdGU;Y2RuLWxvb3A:Y2xvdWRmbGFyZQ;c2VjLWZldGNoLWRlc3Q:ZG9jdW1lbnQ;c2VjLWZldGNoLXVzZXI:PzE;c2VjLWZldGNoLXNpdGU:bm9uZQ;Y29va2ll:X19jZmR1aWQ9ZGNkOWZjOGNjNWMxMzE0NTMyYTFkMjhlZDEyOWRhOTYwMTU2OTk1MTYzNDsgX19jZl9ibT1mYzY2MzMzYzAzZmM0MWFiZTZmOWEyYzI2ZDUwOTA0YzIxYzZhMTQ2LTE1OTU2MjIzNDEtMTgwMC1BZTVzS2pIU2NiWGVFM05mMUhrTlNQMG1tMHBLc2pQWkloVnM1Z2g1SkNHQkFhS1UxVDB2b003alBGN3FjMHVSR2NjZGcrWHdhL1EzbTJhQzdDVU4xZ2M9;YWNjZXB0:dGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksaW1hZ2Uvd2VicCxpbWFnZS9hcG5nLCovKjtxPTAuOCxhcHBsaWNhdGlvbi9zaWduZWQtZXhjaGFuZ2U7dj1iMztxPTAuOQ" - h2, _ := DeserializeHeaders(ser) - h1 := make(http.Header) - for _, header := range h2 { - h1.Add(header.Name, header.Value) - } - h1.Add("Content-Length", "200") - h1.Add("Cf-Something", "Else") - h1.Add("Upgrade", "websocket") - - h1resp := &http.Response{ - StatusCode: 200, - Header: h1, - } - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = H1ResponseToH2ResponseHeaders(h1resp.StatusCode, h1resp.Header) - } -} diff --git a/connection/header.go b/connection/header.go index d1544263..516c5df6 100644 --- a/connection/header.go +++ b/connection/header.go @@ -7,17 +7,15 @@ import ( "strings" "github.com/pkg/errors" - - "github.com/cloudflare/cloudflared/h2mux" ) var ( - // h2mux-style special headers + // internal special headers RequestUserHeaders = "cf-cloudflared-request-headers" ResponseUserHeaders = "cf-cloudflared-response-headers" ResponseMetaHeader = "cf-cloudflared-response-meta" - // h2mux-style special headers + // internal special headers CanonicalResponseUserHeaders = http.CanonicalHeaderKey(ResponseUserHeaders) CanonicalResponseMetaHeader = http.CanonicalHeaderKey(ResponseMetaHeader) ) @@ -28,6 +26,13 @@ var ( responseMetaHeaderOrigin = mustInitRespMetaHeader("origin") ) +// HTTPHeader is a custom header struct that expects only ever one value for the header. +// This structure is used to serialize the headers and attach them to the HTTP2 request when proxying. +type HTTPHeader struct { + Name string + Value string +} + type responseMetaHeader struct { Source string `json:"src"` } @@ -104,10 +109,10 @@ func SerializeHeaders(h1Headers http.Header) string { } // Deserialize headers serialized by `SerializeHeader` -func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) { +func DeserializeHeaders(serializedHeaders string) ([]HTTPHeader, error) { const unableToDeserializeErr = "Unable to deserialize headers" - var deserialized []h2mux.Header + var deserialized []HTTPHeader for _, serializedPair := range strings.Split(serializedHeaders, ";") { if len(serializedPair) == 0 { continue @@ -130,7 +135,7 @@ func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) { return nil, errors.Wrap(err, unableToDeserializeErr) } - deserialized = append(deserialized, h2mux.Header{ + deserialized = append(deserialized, HTTPHeader{ Name: string(deserializedName), Value: string(deserializedValue), }) diff --git a/connection/header_test.go b/connection/header_test.go index 88add316..1ca4b31b 100644 --- a/connection/header_test.go +++ b/connection/header_test.go @@ -46,18 +46,40 @@ func TestSerializeHeaders(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 13, len(deserializedHeaders)) - h2muxExpectedHeaders := stdlibHeaderToH2muxHeader(mockHeaders) + expectedHeaders := headerToReqHeader(mockHeaders) sort.Sort(ByName(deserializedHeaders)) - sort.Sort(ByName(h2muxExpectedHeaders)) + sort.Sort(ByName(expectedHeaders)) assert.True( t, - reflect.DeepEqual(h2muxExpectedHeaders, deserializedHeaders), - fmt.Sprintf("got = %#v, want = %#v\n", deserializedHeaders, h2muxExpectedHeaders), + reflect.DeepEqual(expectedHeaders, deserializedHeaders), + fmt.Sprintf("got = %#v, want = %#v\n", deserializedHeaders, expectedHeaders), ) } +type ByName []HTTPHeader + +func (a ByName) Len() int { return len(a) } +func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByName) Less(i, j int) bool { + if a[i].Name == a[j].Name { + return a[i].Value < a[j].Value + } + + return a[i].Name < a[j].Name +} + +func headerToReqHeader(headers http.Header) (reqHeaders []HTTPHeader) { + for name, values := range headers { + for _, value := range values { + reqHeaders = append(reqHeaders, HTTPHeader{Name: name, Value: value}) + } + } + + return reqHeaders +} + func TestSerializeNoHeaders(t *testing.T) { request, err := http.NewRequest(http.MethodGet, "http://example.com", nil) assert.NoError(t, err) diff --git a/connection/http2.go b/connection/http2.go index f5e4d873..aee9d9da 100644 --- a/connection/http2.go +++ b/connection/http2.go @@ -385,8 +385,7 @@ func determineHTTP2Type(r *http.Request) Type { func handleMissingRequestParts(connType Type, r *http.Request) { if connType == TypeHTTP { // http library has no guarantees that we receive a filled URL. If not, then we fill it, as we reuse the request - // for proxying. We use the same values as we used to in h2mux. For proxying they should not matter since we - // control the dialer on every egress proxied. + // for proxying. For proxying they should not matter since we control the dialer on every egress proxied. if len(r.URL.Scheme) == 0 { r.URL.Scheme = "http" } diff --git a/connection/metrics.go b/connection/metrics.go index c80bf46a..0801ebbc 100644 --- a/connection/metrics.go +++ b/connection/metrics.go @@ -2,11 +2,8 @@ package connection import ( "sync" - "time" "github.com/prometheus/client_golang/prometheus" - - "github.com/cloudflare/cloudflared/h2mux" ) const ( @@ -16,27 +13,6 @@ const ( configSubsystem = "config" ) -type muxerMetrics struct { - rtt *prometheus.GaugeVec - rttMin *prometheus.GaugeVec - rttMax *prometheus.GaugeVec - receiveWindowAve *prometheus.GaugeVec - sendWindowAve *prometheus.GaugeVec - receiveWindowMin *prometheus.GaugeVec - receiveWindowMax *prometheus.GaugeVec - sendWindowMin *prometheus.GaugeVec - sendWindowMax *prometheus.GaugeVec - inBoundRateCurr *prometheus.GaugeVec - inBoundRateMin *prometheus.GaugeVec - inBoundRateMax *prometheus.GaugeVec - outBoundRateCurr *prometheus.GaugeVec - outBoundRateMin *prometheus.GaugeVec - outBoundRateMax *prometheus.GaugeVec - compBytesBefore *prometheus.GaugeVec - compBytesAfter *prometheus.GaugeVec - compRateAve *prometheus.GaugeVec -} - type localConfigMetrics struct { pushes prometheus.Counter pushesErrors prometheus.Counter @@ -53,7 +29,6 @@ type tunnelMetrics struct { regFail *prometheus.CounterVec rpcFail *prometheus.CounterVec - muxerMetrics *muxerMetrics tunnelsHA tunnelsForHA userHostnamesCounts *prometheus.CounterVec @@ -91,252 +66,6 @@ func newLocalConfigMetrics() *localConfigMetrics { } } -func newMuxerMetrics() *muxerMetrics { - rtt := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "rtt", - Help: "Round-trip time in millisecond", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(rtt) - - rttMin := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "rtt_min", - Help: "Shortest round-trip time in millisecond", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(rttMin) - - rttMax := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "rtt_max", - Help: "Longest round-trip time in millisecond", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(rttMax) - - receiveWindowAve := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "receive_window_ave", - Help: "Average receive window size in bytes", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(receiveWindowAve) - - sendWindowAve := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "send_window_ave", - Help: "Average send window size in bytes", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(sendWindowAve) - - receiveWindowMin := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "receive_window_min", - Help: "Smallest receive window size in bytes", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(receiveWindowMin) - - receiveWindowMax := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "receive_window_max", - Help: "Largest receive window size in bytes", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(receiveWindowMax) - - sendWindowMin := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "send_window_min", - Help: "Smallest send window size in bytes", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(sendWindowMin) - - sendWindowMax := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "send_window_max", - Help: "Largest send window size in bytes", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(sendWindowMax) - - inBoundRateCurr := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "inbound_bytes_per_sec_curr", - Help: "Current inbounding bytes per second, 0 if there is no incoming connection", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(inBoundRateCurr) - - inBoundRateMin := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "inbound_bytes_per_sec_min", - Help: "Minimum non-zero inbounding bytes per second", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(inBoundRateMin) - - inBoundRateMax := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "inbound_bytes_per_sec_max", - Help: "Maximum inbounding bytes per second", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(inBoundRateMax) - - outBoundRateCurr := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "outbound_bytes_per_sec_curr", - Help: "Current outbounding bytes per second, 0 if there is no outgoing traffic", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(outBoundRateCurr) - - outBoundRateMin := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "outbound_bytes_per_sec_min", - Help: "Minimum non-zero outbounding bytes per second", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(outBoundRateMin) - - outBoundRateMax := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "outbound_bytes_per_sec_max", - Help: "Maximum outbounding bytes per second", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(outBoundRateMax) - - compBytesBefore := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "comp_bytes_before", - Help: "Bytes sent via cross-stream compression, pre compression", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(compBytesBefore) - - compBytesAfter := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "comp_bytes_after", - Help: "Bytes sent via cross-stream compression, post compression", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(compBytesAfter) - - compRateAve := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: muxerSubsystem, - Name: "comp_rate_ave", - Help: "Average outbound cross-stream compression ratio", - }, - []string{"connection_id"}, - ) - prometheus.MustRegister(compRateAve) - - return &muxerMetrics{ - rtt: rtt, - rttMin: rttMin, - rttMax: rttMax, - receiveWindowAve: receiveWindowAve, - sendWindowAve: sendWindowAve, - receiveWindowMin: receiveWindowMin, - receiveWindowMax: receiveWindowMax, - sendWindowMin: sendWindowMin, - sendWindowMax: sendWindowMax, - inBoundRateCurr: inBoundRateCurr, - inBoundRateMin: inBoundRateMin, - inBoundRateMax: inBoundRateMax, - outBoundRateCurr: outBoundRateCurr, - outBoundRateMin: outBoundRateMin, - outBoundRateMax: outBoundRateMax, - compBytesBefore: compBytesBefore, - compBytesAfter: compBytesAfter, - compRateAve: compRateAve, - } -} - -func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) { - m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT)) - m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin)) - m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax)) - m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve) - m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve) - m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin)) - m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax)) - m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin)) - m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax)) - m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr)) - m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin)) - m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax)) - m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr)) - m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin)) - m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax)) - m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value())) - m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value())) - m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve())) -} - -func convertRTTMilliSec(t time.Duration) float64 { - return float64(t / time.Millisecond) -} - // Metrics that can be collected without asking the edge func initTunnelMetrics() *tunnelMetrics { maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec( @@ -408,7 +137,6 @@ func initTunnelMetrics() *tunnelMetrics { return &tunnelMetrics{ serverLocations: serverLocations, oldServerLocations: make(map[string]string), - muxerMetrics: newMuxerMetrics(), tunnelsHA: newTunnelsForHA(), regSuccess: registerSuccess, regFail: registerFail, @@ -418,10 +146,6 @@ func initTunnelMetrics() *tunnelMetrics { } } -func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) { - t.muxerMetrics.update(connectionID, metrics) -} - func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) { t.locationLock.Lock() defer t.locationLock.Unlock() diff --git a/connection/protocol.go b/connection/protocol.go index ecc367b4..417c8b72 100644 --- a/connection/protocol.go +++ b/connection/protocol.go @@ -13,7 +13,7 @@ import ( const ( AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge" - // edgeH2muxTLSServerName is the server name to establish h2mux connection with edge + // edgeH2muxTLSServerName is the server name to establish h2mux connection with edge (unused, but kept for legacy reference). edgeH2muxTLSServerName = "cftunnel.com" // edgeH2TLSServerName is the server name to establish http2 connection with edge edgeH2TLSServerName = "h2.cftunnel.com" diff --git a/edgediscovery/dial.go b/edgediscovery/dial.go index 675e5dc5..1bbf59c3 100644 --- a/edgediscovery/dial.go +++ b/edgediscovery/dial.go @@ -9,7 +9,7 @@ import ( "github.com/pkg/errors" ) -// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node +// DialEdge makes a TLS connection to a Cloudflare edge node func DialEdge( ctx context.Context, timeout time.Duration, @@ -36,7 +36,7 @@ func DialEdge( if err = tlsEdgeConn.Handshake(); err != nil { return nil, newDialError(err, "TLS handshake with edge error") } - // clear the deadline on the conn; h2mux has its own timeouts + // clear the deadline on the conn; http2 has its own timeouts tlsEdgeConn.SetDeadline(time.Time{}) return tlsEdgeConn, nil } diff --git a/h2mux/activestreammap.go b/h2mux/activestreammap.go deleted file mode 100644 index 02386db3..00000000 --- a/h2mux/activestreammap.go +++ /dev/null @@ -1,195 +0,0 @@ -package h2mux - -import ( - "sync" - - "github.com/prometheus/client_golang/prometheus" - "golang.org/x/net/http2" -) - -var ( - ActiveStreams = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "cloudflared", - Subsystem: "tunnel", - Name: "active_streams", - Help: "Number of active streams created by all muxers.", - }) -) - -func init() { - prometheus.MustRegister(ActiveStreams) -} - -// activeStreamMap is used to moderate access to active streams between the read and write -// threads, and deny access to new peer streams while shutting down. -type activeStreamMap struct { - sync.RWMutex - // streams tracks open streams. - streams map[uint32]*MuxedStream - // nextStreamID is the next ID to use on our side of the connection. - // This is odd for clients, even for servers. - nextStreamID uint32 - // maxPeerStreamID is the ID of the most recent stream opened by the peer. - maxPeerStreamID uint32 - // activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams - activeStreams prometheus.Gauge - - // ignoreNewStreams is true when the connection is being shut down. New streams - // cannot be registered. - ignoreNewStreams bool - // streamsEmpty is a chan that will be closed when no more streams are open. - streamsEmptyChan chan struct{} - closeOnce sync.Once -} - -func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Gauge) *activeStreamMap { - m := &activeStreamMap{ - streams: make(map[uint32]*MuxedStream), - streamsEmptyChan: make(chan struct{}), - nextStreamID: 1, - activeStreams: activeStreams, - } - // Client initiated stream uses odd stream ID, server initiated stream uses even stream ID - if !useClientStreamNumbers { - m.nextStreamID = 2 - } - return m -} - -// This function should be called while `m` is locked. -func (m *activeStreamMap) notifyStreamsEmpty() { - m.closeOnce.Do(func() { - close(m.streamsEmptyChan) - }) -} - -// Len returns the number of active streams. -func (m *activeStreamMap) Len() int { - m.RLock() - defer m.RUnlock() - return len(m.streams) -} - -func (m *activeStreamMap) Get(streamID uint32) (*MuxedStream, bool) { - m.RLock() - defer m.RUnlock() - stream, ok := m.streams[streamID] - return stream, ok -} - -// Set returns true if the stream was assigned successfully. If a stream -// already existed with that ID or we are shutting down, return false. -func (m *activeStreamMap) Set(newStream *MuxedStream) bool { - m.Lock() - defer m.Unlock() - if _, ok := m.streams[newStream.streamID]; ok { - return false - } - if m.ignoreNewStreams { - return false - } - m.streams[newStream.streamID] = newStream - m.activeStreams.Inc() - return true -} - -// Delete stops tracking the stream. It should be called only after it is closed and reset. -func (m *activeStreamMap) Delete(streamID uint32) { - m.Lock() - defer m.Unlock() - if _, ok := m.streams[streamID]; ok { - delete(m.streams, streamID) - m.activeStreams.Dec() - } - - // shutting down, and now the map is empty - if m.ignoreNewStreams && len(m.streams) == 0 { - m.notifyStreamsEmpty() - } -} - -// Shutdown blocks new streams from being created. -// It returns `done`, a channel that is closed once the last stream has closed -// and `progress`, whether a shutdown was already in progress -func (m *activeStreamMap) Shutdown() (done <-chan struct{}, alreadyInProgress bool) { - m.Lock() - defer m.Unlock() - if m.ignoreNewStreams { - // already shutting down - return m.streamsEmptyChan, true - } - m.ignoreNewStreams = true - if len(m.streams) == 0 { - // there are no streams to wait for - m.notifyStreamsEmpty() - } - return m.streamsEmptyChan, false -} - -// AcquireLocalID acquires a new stream ID for a stream you're opening. -func (m *activeStreamMap) AcquireLocalID() uint32 { - m.Lock() - defer m.Unlock() - x := m.nextStreamID - m.nextStreamID += 2 - return x -} - -// ObservePeerID observes the ID of a stream opened by the peer. It returns true if we should accept -// the new stream, or false to reject it. The ErrCode gives the reason why. -func (m *activeStreamMap) AcquirePeerID(streamID uint32) (bool, http2.ErrCode) { - m.Lock() - defer m.Unlock() - switch { - case m.ignoreNewStreams: - return false, http2.ErrCodeStreamClosed - case streamID > m.maxPeerStreamID: - m.maxPeerStreamID = streamID - return true, http2.ErrCodeNo - default: - return false, http2.ErrCodeStreamClosed - } -} - -// IsPeerStreamID is true if the stream ID belongs to the peer. -func (m *activeStreamMap) IsPeerStreamID(streamID uint32) bool { - m.RLock() - defer m.RUnlock() - return (streamID % 2) != (m.nextStreamID % 2) -} - -// IsLocalStreamID is true if it is a stream we have opened, even if it is now closed. -func (m *activeStreamMap) IsLocalStreamID(streamID uint32) bool { - m.RLock() - defer m.RUnlock() - return (streamID%2) == (m.nextStreamID%2) && streamID < m.nextStreamID -} - -// LastPeerStreamID returns the most recently opened peer stream ID. -func (m *activeStreamMap) LastPeerStreamID() uint32 { - m.RLock() - defer m.RUnlock() - return m.maxPeerStreamID -} - -// LastLocalStreamID returns the most recently opened local stream ID. -func (m *activeStreamMap) LastLocalStreamID() uint32 { - m.RLock() - defer m.RUnlock() - if m.nextStreamID > 1 { - return m.nextStreamID - 2 - } - return 0 -} - -// Abort closes every active stream and prevents new ones being created. This should be used to -// return errors in pending read/writes when the underlying connection goes away. -func (m *activeStreamMap) Abort() { - m.Lock() - defer m.Unlock() - for _, stream := range m.streams { - stream.Close() - } - m.ignoreNewStreams = true - m.notifyStreamsEmpty() -} diff --git a/h2mux/activestreammap_test.go b/h2mux/activestreammap_test.go deleted file mode 100644 index 0395b79b..00000000 --- a/h2mux/activestreammap_test.go +++ /dev/null @@ -1,195 +0,0 @@ -package h2mux - -import ( - "sync" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestShutdown(t *testing.T) { - const numStreams = 1000 - m := newActiveStreamMap(true, ActiveStreams) - - // Add all the streams - { - var wg sync.WaitGroup - wg.Add(numStreams) - for i := 0; i < numStreams; i++ { - go func(streamID int) { - defer wg.Done() - stream := &MuxedStream{streamID: uint32(streamID)} - ok := m.Set(stream) - assert.True(t, ok) - }(i) - } - wg.Wait() - } - assert.Equal(t, numStreams, m.Len(), "All the streams should have been added") - - shutdownChan, alreadyInProgress := m.Shutdown() - select { - case <-shutdownChan: - assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed") - default: - } - assert.False(t, alreadyInProgress) - - shutdownChan2, alreadyInProgress2 := m.Shutdown() - assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel") - assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'") - - // Delete all the streams - { - var wg sync.WaitGroup - wg.Add(numStreams) - for i := 0; i < numStreams; i++ { - go func(streamID int) { - defer wg.Done() - m.Delete(uint32(streamID)) - }(i) - } - wg.Wait() - } - assert.Equal(t, 0, m.Len(), "All the streams should have been deleted") - - select { - case <-shutdownChan: - default: - assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed") - } -} - -func TestEmptyBeforeShutdown(t *testing.T) { - const numStreams = 1000 - m := newActiveStreamMap(true, ActiveStreams) - - // Add all the streams - { - var wg sync.WaitGroup - wg.Add(numStreams) - for i := 0; i < numStreams; i++ { - go func(streamID int) { - defer wg.Done() - stream := &MuxedStream{streamID: uint32(streamID)} - ok := m.Set(stream) - assert.True(t, ok) - }(i) - } - wg.Wait() - } - assert.Equal(t, numStreams, m.Len(), "All the streams should have been added") - - // Delete all the streams, bringing m to size 0 - { - var wg sync.WaitGroup - wg.Add(numStreams) - for i := 0; i < numStreams; i++ { - go func(streamID int) { - defer wg.Done() - m.Delete(uint32(streamID)) - }(i) - } - wg.Wait() - } - assert.Equal(t, 0, m.Len(), "All the streams should have been deleted") - - // Add one stream back - const soloStreamID = uint32(0) - ok := m.Set(&MuxedStream{streamID: soloStreamID}) - assert.True(t, ok) - - shutdownChan, alreadyInProgress := m.Shutdown() - select { - case <-shutdownChan: - assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed") - default: - } - assert.False(t, alreadyInProgress) - - shutdownChan2, alreadyInProgress2 := m.Shutdown() - assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel") - assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'") - - // Remove the remaining stream - m.Delete(soloStreamID) - - select { - case <-shutdownChan: - default: - assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed") - } -} - -type noopBuffer struct { - isClosed bool -} - -func (t *noopBuffer) Read(p []byte) (n int, err error) { return len(p), nil } -func (t *noopBuffer) Write(p []byte) (n int, err error) { return len(p), nil } -func (t *noopBuffer) Reset() {} -func (t *noopBuffer) Len() int { return 0 } -func (t *noopBuffer) Close() error { t.isClosed = true; return nil } -func (t *noopBuffer) Closed() bool { return t.isClosed } - -type noopReadyList struct{} - -func (_ *noopReadyList) Signal(streamID uint32) {} - -func TestAbort(t *testing.T) { - const numStreams = 1000 - m := newActiveStreamMap(true, ActiveStreams) - - var openedStreams sync.Map - - // Add all the streams - { - var wg sync.WaitGroup - wg.Add(numStreams) - for i := 0; i < numStreams; i++ { - go func(streamID int) { - defer wg.Done() - stream := &MuxedStream{ - streamID: uint32(streamID), - readBuffer: &noopBuffer{}, - writeBuffer: &noopBuffer{}, - readyList: &noopReadyList{}, - } - ok := m.Set(stream) - assert.True(t, ok) - - openedStreams.Store(stream.streamID, stream) - }(i) - } - wg.Wait() - } - assert.Equal(t, numStreams, m.Len(), "All the streams should have been added") - - shutdownChan, alreadyInProgress := m.Shutdown() - select { - case <-shutdownChan: - assert.Fail(t, "before Abort(), shutdownChan shouldn't be closed") - default: - } - assert.False(t, alreadyInProgress) - - m.Abort() - assert.Equal(t, numStreams, m.Len(), "Abort() shouldn't delete any streams") - openedStreams.Range(func(key interface{}, value interface{}) bool { - stream := value.(*MuxedStream) - readBuffer := stream.readBuffer.(*noopBuffer) - writeBuffer := stream.writeBuffer.(*noopBuffer) - return assert.True(t, readBuffer.isClosed && writeBuffer.isClosed, "Abort() should have closed all the streams") - }) - - select { - case <-shutdownChan: - default: - assert.Fail(t, "after Abort(), shutdownChan should have been closed") - } - - // multiple aborts shouldn't cause any issues - m.Abort() - m.Abort() - m.Abort() -} diff --git a/h2mux/bytes_counter.go b/h2mux/bytes_counter.go deleted file mode 100644 index 7260f8bb..00000000 --- a/h2mux/bytes_counter.go +++ /dev/null @@ -1,27 +0,0 @@ -package h2mux - -import ( - "sync/atomic" -) - -type AtomicCounter struct { - count uint64 -} - -func NewAtomicCounter(initCount uint64) *AtomicCounter { - return &AtomicCounter{count: initCount} -} - -func (c *AtomicCounter) IncrementBy(number uint64) { - atomic.AddUint64(&c.count, number) -} - -// Count returns the current value of counter and reset it to 0 -func (c *AtomicCounter) Count() uint64 { - return atomic.SwapUint64(&c.count, 0) -} - -// Value returns the current value of counter -func (c *AtomicCounter) Value() uint64 { - return atomic.LoadUint64(&c.count) -} diff --git a/h2mux/bytes_counter_test.go b/h2mux/bytes_counter_test.go deleted file mode 100644 index da579aaf..00000000 --- a/h2mux/bytes_counter_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package h2mux - -import ( - "sync" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCounter(t *testing.T) { - var wg sync.WaitGroup - wg.Add(dataPoints) - c := AtomicCounter{} - for i := 0; i < dataPoints; i++ { - go func() { - defer wg.Done() - c.IncrementBy(uint64(1)) - }() - } - wg.Wait() - assert.Equal(t, uint64(dataPoints), c.Count()) - assert.Equal(t, uint64(0), c.Count()) -} diff --git a/h2mux/error.go b/h2mux/error.go deleted file mode 100644 index 923eb335..00000000 --- a/h2mux/error.go +++ /dev/null @@ -1,66 +0,0 @@ -package h2mux - -import ( - "fmt" - - "golang.org/x/net/http2" -) - -var ( - // HTTP2 error codes: https://http2.github.io/http2-spec/#ErrorCodes - ErrHandshakeTimeout = MuxerHandshakeError{"1000 handshake timeout"} - ErrBadHandshakeNotSettings = MuxerHandshakeError{"1001 unexpected response"} - ErrBadHandshakeUnexpectedAck = MuxerHandshakeError{"1002 unexpected response"} - ErrBadHandshakeNoMagic = MuxerHandshakeError{"1003 unexpected response"} - ErrBadHandshakeWrongMagic = MuxerHandshakeError{"1004 connected to endpoint of wrong type"} - ErrBadHandshakeNotSettingsAck = MuxerHandshakeError{"1005 unexpected response"} - ErrBadHandshakeUnexpectedSettings = MuxerHandshakeError{"1006 unexpected response"} - - ErrUnexpectedFrameType = MuxerProtocolError{"2001 unexpected frame type", http2.ErrCodeProtocol} - ErrUnknownStream = MuxerProtocolError{"2002 unknown stream", http2.ErrCodeProtocol} - ErrInvalidStream = MuxerProtocolError{"2003 invalid stream", http2.ErrCodeProtocol} - ErrNotRPCStream = MuxerProtocolError{"2004 not RPC stream", http2.ErrCodeProtocol} - - ErrStreamHeadersSent = MuxerApplicationError{"3000 headers already sent"} - ErrStreamRequestConnectionClosed = MuxerApplicationError{"3001 connection closed while opening stream"} - ErrConnectionDropped = MuxerApplicationError{"3002 connection dropped"} - ErrStreamRequestTimeout = MuxerApplicationError{"3003 open stream timeout"} - ErrResponseHeadersTimeout = MuxerApplicationError{"3004 timeout waiting for initial response headers"} - ErrResponseHeadersConnectionClosed = MuxerApplicationError{"3005 connection closed while waiting for initial response headers"} - - ErrClosedStream = MuxerStreamError{"4000 stream closed", http2.ErrCodeStreamClosed} -) - -type MuxerHandshakeError struct { - cause string -} - -func (e MuxerHandshakeError) Error() string { - return fmt.Sprintf("Handshake error: %s", e.cause) -} - -type MuxerProtocolError struct { - cause string - h2code http2.ErrCode -} - -func (e MuxerProtocolError) Error() string { - return fmt.Sprintf("Protocol error: %s", e.cause) -} - -type MuxerApplicationError struct { - cause string -} - -func (e MuxerApplicationError) Error() string { - return fmt.Sprintf("Application error: %s", e.cause) -} - -type MuxerStreamError struct { - cause string - h2code http2.ErrCode -} - -func (e MuxerStreamError) Error() string { - return fmt.Sprintf("Stream error: %s", e.cause) -} diff --git a/h2mux/h2_compressor.go b/h2mux/h2_compressor.go deleted file mode 100644 index 7d609305..00000000 --- a/h2mux/h2_compressor.go +++ /dev/null @@ -1,17 +0,0 @@ -package h2mux - -import ( - "io" -) - -func CompressionIsSupported() bool { - return false -} - -func newDecompressor(src io.Reader) decompressor { - return nil -} - -func newCompressor(dst io.Writer, quality, lgwin int) compressor { - return nil -} diff --git a/h2mux/h2_dictionaries.go b/h2mux/h2_dictionaries.go deleted file mode 100644 index 5d11bee7..00000000 --- a/h2mux/h2_dictionaries.go +++ /dev/null @@ -1,596 +0,0 @@ -package h2mux - -import ( - "bytes" - "io" - "strings" - "sync" - - "golang.org/x/net/http2" -) - -/* This is an implementation of https://github.com/vkrasnov/h2-compression-dictionaries -but modified for tunnels in a few key ways: -Since tunnels is a server-to-server service, some aspects of the spec would cause -unnecessary head-of-line blocking on the CPU and on the network, hence this implementation -allows for parallel compression on the "client", and buffering on the "server" to solve -this problem. */ - -// Assign temporary values -const SettingCompression http2.SettingID = 0xff20 - -const ( - FrameSetCompressionContext http2.FrameType = 0xf0 - FrameUseDictionary http2.FrameType = 0xf1 - FrameSetDictionary http2.FrameType = 0xf2 -) - -const ( - FlagSetDictionaryAppend http2.Flags = 0x1 - FlagSetDictionaryOffset http2.Flags = 0x2 -) - -const compressionVersion = uint8(1) -const compressionFormat = uint8(2) - -type CompressionSetting uint - -const ( - CompressionNone CompressionSetting = iota - CompressionLow - CompressionMedium - CompressionMax -) - -type CompressionPreset struct { - nDicts, dictSize, quality uint8 -} - -type compressor interface { - Write([]byte) (int, error) - Flush() error - SetDictionary([]byte) - Close() error -} - -type decompressor interface { - Read([]byte) (int, error) - SetDictionary([]byte) - Close() error -} - -var compressionPresets = map[CompressionSetting]CompressionPreset{ - CompressionNone: {0, 0, 0}, - CompressionLow: {32, 17, 5}, - CompressionMedium: {64, 18, 6}, - CompressionMax: {255, 19, 9}, -} - -func compressionSettingVal(version, fmt, sz, nd uint8) uint32 { - // Currently the compression settings are include: - // * version: only 1 is supported - // * fmt: only 2 for brotli is supported - // * sz: log2 of the maximal allowed dictionary size - // * nd: max allowed number of dictionaries - return uint32(version)<<24 + uint32(fmt)<<16 + uint32(sz)<<8 + uint32(nd) -} - -func parseCompressionSettingVal(setting uint32) (version, fmt, sz, nd uint8) { - version = uint8(setting >> 24) - fmt = uint8(setting >> 16) - sz = uint8(setting >> 8) - nd = uint8(setting) - return -} - -func (c CompressionSetting) toH2Setting() uint32 { - p, ok := compressionPresets[c] - if !ok { - return 0 - } - return compressionSettingVal(compressionVersion, compressionFormat, p.dictSize, p.nDicts) -} - -func (c CompressionSetting) getPreset() CompressionPreset { - return compressionPresets[c] -} - -type dictUpdate struct { - reader *h2DictionaryReader - dictionary *h2ReadDictionary - buff []byte - isReady bool - isUse bool - s setDictRequest -} - -type h2ReadDictionary struct { - dictionary []byte - queue []*dictUpdate - maxSize int -} - -type h2ReadDictionaries struct { - d []h2ReadDictionary - maxSize int -} - -type h2DictionaryReader struct { - *SharedBuffer // Propagate the decompressed output into the original buffer - decompBuffer *bytes.Buffer // Intermediate buffer for the brotli compressor - dictionary []byte // The content of the dictionary being used by this reader - internalBuffer []byte - s, e int // Start and end of the buffer - decomp decompressor // The brotli compressor - isClosed bool // Indicates that Close was called for this reader - queue []*dictUpdate // List of dictionaries to update, when the data is available -} - -type h2WriteDictionary []byte - -type setDictRequest struct { - streamID uint32 - dictID uint8 - dictSZ uint64 - truncate, offset uint64 - P, E, D bool -} - -type useDictRequest struct { - dictID uint8 - streamID uint32 - setDict []setDictRequest -} - -type h2WriteDictionaries struct { - dictLock sync.Mutex - dictChan chan useDictRequest - dictionaries []h2WriteDictionary - nextAvail int // next unused dictionary slot - maxAvail int // max ID, defined by SETTINGS - maxSize int // max size, defined by SETTINGS - typeToDict map[string]uint8 // map from content type to dictionary that encodes it - pathToDict map[string]uint8 // map from path to dictionary that encodes it - quality int - window int - compIn, compOut *AtomicCounter -} - -type h2DictWriter struct { - *bytes.Buffer - comp compressor - dicts *h2WriteDictionaries - writerLock sync.Mutex - - streamID uint32 - path string - contentType string -} - -type h2Dictionaries struct { - write *h2WriteDictionaries - read *h2ReadDictionaries -} - -func (o *dictUpdate) update(buff []byte) { - o.buff = make([]byte, len(buff)) - copy(o.buff, buff) - o.isReady = true -} - -func (d *h2ReadDictionary) update() { - for len(d.queue) > 0 { - o := d.queue[0] - if !o.isReady { - break - } - if o.isUse { - reader := o.reader - reader.dictionary = make([]byte, len(d.dictionary)) - copy(reader.dictionary, d.dictionary) - reader.decomp = newDecompressor(reader.decompBuffer) - if len(reader.dictionary) > 0 { - reader.decomp.SetDictionary(reader.dictionary) - } - reader.Write([]byte{}) - } else { - d.dictionary = adjustDictionary(d.dictionary, o.buff, o.s, d.maxSize) - } - d.queue = d.queue[1:] - } -} - -func newH2ReadDictionaries(nd, sz uint8) h2ReadDictionaries { - d := make([]h2ReadDictionary, int(nd)) - for i := range d { - d[i].maxSize = 1 << uint(sz) - } - return h2ReadDictionaries{d: d, maxSize: 1 << uint(sz)} -} - -func (dicts *h2ReadDictionaries) getDictByID(dictID uint8) (*h2ReadDictionary, error) { - if int(dictID) > len(dicts.d) { - return nil, MuxerStreamError{"dictID too big", http2.ErrCodeProtocol} - } - - return &dicts.d[dictID], nil -} - -func (dicts *h2ReadDictionaries) newReader(b *SharedBuffer, dictID uint8) *h2DictionaryReader { - if int(dictID) > len(dicts.d) { - return nil - } - - dictionary := &dicts.d[dictID] - reader := &h2DictionaryReader{SharedBuffer: b, decompBuffer: &bytes.Buffer{}, internalBuffer: make([]byte, dicts.maxSize)} - - if len(dictionary.queue) == 0 { - reader.dictionary = make([]byte, len(dictionary.dictionary)) - copy(reader.dictionary, dictionary.dictionary) - reader.decomp = newDecompressor(reader.decompBuffer) - if len(reader.dictionary) > 0 { - reader.decomp.SetDictionary(reader.dictionary) - } - } else { - dictionary.queue = append(dictionary.queue, &dictUpdate{isUse: true, isReady: true, reader: reader}) - } - return reader -} - -func (r *h2DictionaryReader) updateWaitingDictionaries() { - // Update all the waiting dictionaries - for _, o := range r.queue { - if o.isReady { - continue - } - if r.isClosed || uint64(r.e) >= o.s.dictSZ { - o.update(r.internalBuffer[:r.e]) - if o == o.dictionary.queue[0] { - defer o.dictionary.update() - } - } - } -} - -// Write actually happens when reading from network, this is therefore the stage where we decompress the buffer -func (r *h2DictionaryReader) Write(p []byte) (n int, err error) { - // Every write goes into brotli buffer first - n, err = r.decompBuffer.Write(p) - if err != nil { - return - } - - if r.decomp == nil { - return - } - - for { - m, err := r.decomp.Read(r.internalBuffer[r.e:]) - if err != nil && err != io.EOF { - r.SharedBuffer.Close() - r.decomp.Close() - return n, err - } - - r.SharedBuffer.Write(r.internalBuffer[r.e : r.e+m]) - r.e += m - - if m == 0 { - break - } - - if r.e == len(r.internalBuffer) { - r.updateWaitingDictionaries() - r.e = 0 - } - } - - r.updateWaitingDictionaries() - - if r.isClosed { - r.SharedBuffer.Close() - r.decomp.Close() - } - - return -} - -func (r *h2DictionaryReader) Close() error { - if r.isClosed { - return nil - } - r.isClosed = true - r.Write([]byte{}) - return nil -} - -var compressibleTypes = map[string]bool{ - "application/atom+xml": true, - "application/javascript": true, - "application/json": true, - "application/ld+json": true, - "application/manifest+json": true, - "application/rss+xml": true, - "application/vnd.geo+json": true, - "application/vnd.ms-fontobject": true, - "application/x-font-ttf": true, - "application/x-yaml": true, - "application/x-web-app-manifest+json": true, - "application/xhtml+xml": true, - "application/xml": true, - "font/opentype": true, - "image/bmp": true, - "image/svg+xml": true, - "image/x-icon": true, - "text/cache-manifest": true, - "text/css": true, - "text/html": true, - "text/plain": true, - "text/vcard": true, - "text/vnd.rim.location.xloc": true, - "text/vtt": true, - "text/x-component": true, - "text/x-cross-domain-policy": true, - "text/x-yaml": true, -} - -func getContentType(headers []Header) string { - for _, h := range headers { - if strings.ToLower(h.Name) == "content-type" { - val := strings.ToLower(h.Value) - sep := strings.IndexRune(val, ';') - if sep != -1 { - return val[:sep] - } - return val - } - } - - return "" -} - -func newH2WriteDictionaries(nd, sz, quality uint8, compIn, compOut *AtomicCounter) (*h2WriteDictionaries, chan useDictRequest) { - useDictChan := make(chan useDictRequest) - return &h2WriteDictionaries{ - dictionaries: make([]h2WriteDictionary, nd), - nextAvail: 0, - maxAvail: int(nd), - maxSize: 1 << uint(sz), - dictChan: useDictChan, - typeToDict: make(map[string]uint8), - pathToDict: make(map[string]uint8), - quality: int(quality), - window: 1 << uint(sz+1), - compIn: compIn, - compOut: compOut, - }, useDictChan -} - -func adjustDictionary(currentDictionary, newData []byte, set setDictRequest, maxSize int) []byte { - currentDictionary = append(currentDictionary, newData[:set.dictSZ]...) - - if len(currentDictionary) > maxSize { - currentDictionary = currentDictionary[len(currentDictionary)-maxSize:] - } - - return currentDictionary -} - -func (h2d *h2WriteDictionaries) getNextDictID() (dictID uint8, ok bool) { - if h2d.nextAvail < h2d.maxAvail { - dictID, ok = uint8(h2d.nextAvail), true - h2d.nextAvail++ - return - } - - return 0, false -} - -func (h2d *h2WriteDictionaries) getGenericDictID() (dictID uint8, ok bool) { - if h2d.maxAvail == 0 { - return 0, false - } - return uint8(h2d.maxAvail - 1), true -} - -func (h2d *h2WriteDictionaries) getDictWriter(s *MuxedStream, headers []Header) *h2DictWriter { - w := s.writeBuffer - - if w == nil { - return nil - } - - if s.method != "GET" && s.method != "POST" { - return nil - } - - s.contentType = getContentType(headers) - if _, ok := compressibleTypes[s.contentType]; !ok && !strings.HasPrefix(s.contentType, "text") { - return nil - } - - return &h2DictWriter{ - Buffer: w.(*bytes.Buffer), - path: s.path, - contentType: s.contentType, - streamID: s.streamID, - dicts: h2d, - } -} - -func assignDictToStream(s *MuxedStream, p []byte) bool { - - // On first write to stream: - // * assign the right dictionary - // * update relevant dictionaries - // * send the required USE_DICT and SET_DICT frames - - h2d := s.dictionaries.write - if h2d == nil { - return false - } - - w, ok := s.writeBuffer.(*h2DictWriter) - if !ok || w.comp != nil { - return false - } - - h2d.dictLock.Lock() - - if w.comp != nil { - // Check again with lock, in therory the interface allows for unordered writes - h2d.dictLock.Unlock() - return false - } - - // The logic of dictionary generation is below - - // Is there a dictionary for the exact path or content-type? - var useID uint8 - pathID, pathFound := h2d.pathToDict[w.path] - typeID, typeFound := h2d.typeToDict[w.contentType] - - if pathFound { - // Use dictionary for path as top priority - useID = pathID - if !typeFound { // Shouldn't really happen, unless type changes between requests - typeID, typeFound = h2d.getNextDictID() - if typeFound { - h2d.typeToDict[w.contentType] = typeID - } - } - } else if typeFound { - // Use dictionary for same content type as second priority - useID = typeID - pathID, pathFound = h2d.getNextDictID() - if pathFound { // If a slot is available, generate new dictionary for path - h2d.pathToDict[w.path] = pathID - } - } else { - // Use the overflow dictionary as last resort - // If slots are available generate new dictionaries for path and content-type - useID, _ = h2d.getGenericDictID() - pathID, pathFound = h2d.getNextDictID() - if pathFound { - h2d.pathToDict[w.path] = pathID - } - typeID, typeFound = h2d.getNextDictID() - if typeFound { - h2d.typeToDict[w.contentType] = typeID - } - } - - useLen := h2d.maxSize - if len(p) < useLen { - useLen = len(p) - } - - // Update all the dictionaries using the new data - setDicts := make([]setDictRequest, 0, 3) - setDict := setDictRequest{ - streamID: w.streamID, - dictID: useID, - dictSZ: uint64(useLen), - } - setDicts = append(setDicts, setDict) - if pathID != useID { - setDict.dictID = pathID - setDicts = append(setDicts, setDict) - } - if typeID != useID { - setDict.dictID = typeID - setDicts = append(setDicts, setDict) - } - - h2d.dictChan <- useDictRequest{streamID: w.streamID, dictID: uint8(useID), setDict: setDicts} - - dict := h2d.dictionaries[useID] - - // Brolti requires the dictionary to be immutable - copyDict := make([]byte, len(dict)) - copy(copyDict, dict) - - for _, set := range setDicts { - h2d.dictionaries[set.dictID] = adjustDictionary(h2d.dictionaries[set.dictID], p, set, h2d.maxSize) - } - - w.comp = newCompressor(w.Buffer, h2d.quality, h2d.window) - - s.writeLock.Lock() - h2d.dictLock.Unlock() - - if len(copyDict) > 0 { - w.comp.SetDictionary(copyDict) - } - - return true -} - -func (w *h2DictWriter) Write(p []byte) (n int, err error) { - bufLen := w.Buffer.Len() - if w.comp != nil { - n, err = w.comp.Write(p) - if err != nil { - return - } - err = w.comp.Flush() - w.dicts.compIn.IncrementBy(uint64(n)) - w.dicts.compOut.IncrementBy(uint64(w.Buffer.Len() - bufLen)) - return - } - return w.Buffer.Write(p) -} - -func (w *h2DictWriter) Close() error { - if w.comp != nil { - return w.comp.Close() - } - return nil -} - -// From http2/hpack -func http2ReadVarInt(n byte, p []byte) (remain []byte, v uint64, err error) { - if n < 1 || n > 8 { - panic("bad n") - } - if len(p) == 0 { - return nil, 0, MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol} - } - v = uint64(p[0]) - if n < 8 { - v &= (1 << uint64(n)) - 1 - } - if v < (1< 0 { - b := p[0] - p = p[1:] - v += uint64(b&127) << m - if b&128 == 0 { - return p, v, nil - } - m += 7 - if m >= 63 { - return origP, 0, MuxerStreamError{"invalid integer", http2.ErrCodeProtocol} - } - } - return nil, 0, MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol} -} - -func appendVarInt(dst []byte, n byte, i uint64) []byte { - k := uint64((1 << n) - 1) - if i < k { - return append(dst, byte(i)) - } - dst = append(dst, byte(k)) - i -= k - for ; i >= 128; i >>= 7 { - dst = append(dst, byte(0x80|(i&0x7f))) - } - return append(dst, byte(i)) -} diff --git a/h2mux/h2mux.go b/h2mux/h2mux.go deleted file mode 100644 index c7c75f3b..00000000 --- a/h2mux/h2mux.go +++ /dev/null @@ -1,506 +0,0 @@ -package h2mux - -import ( - "context" - "io" - "strings" - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/rs/zerolog" - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" - "golang.org/x/sync/errgroup" -) - -const ( - defaultFrameSize uint32 = 1 << 14 // Minimum frame size in http2 spec - defaultWindowSize uint32 = (1 << 16) - 1 // Minimum window size in http2 spec - maxWindowSize uint32 = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size in http2 spec - defaultTimeout time.Duration = 5 * time.Second - defaultRetries uint64 = 5 - defaultWriteBufferMaxLen int = 1024 * 1024 // 1mb - writeBufferInitialSize int = 16 * 1024 // 16KB - - SettingMuxerMagic http2.SettingID = 0x42db - MuxerMagicOrigin uint32 = 0xa2e43c8b - MuxerMagicEdge uint32 = 0x1088ebf9 -) - -type MuxedStreamHandler interface { - ServeStream(*MuxedStream) error -} - -type MuxedStreamFunc func(stream *MuxedStream) error - -func (f MuxedStreamFunc) ServeStream(stream *MuxedStream) error { - return f(stream) -} - -type MuxerConfig struct { - Timeout time.Duration - Handler MuxedStreamHandler - IsClient bool - // Name is used to identify this muxer instance when logging. - Name string - // The minimum time this connection can be idle before sending a heartbeat. - HeartbeatInterval time.Duration - // The minimum number of heartbeats to send before terminating the connection. - MaxHeartbeats uint64 - // Logger to use - Log *zerolog.Logger - CompressionQuality CompressionSetting - // Initial size for HTTP2 flow control windows - DefaultWindowSize uint32 - // Largest allowable size for HTTP2 flow control windows - MaxWindowSize uint32 - // Largest allowable capacity for the buffer of data to be sent - StreamWriteBufferMaxLen int -} - -type Muxer struct { - // f is used to read and write HTTP2 frames on the wire. - f *http2.Framer - // config is the MuxerConfig given in Handshake. - config MuxerConfig - // w, r are references to the underlying connection used. - w io.WriteCloser - r io.ReadCloser - // muxReader is the read process. - muxReader *MuxReader - // muxWriter is the write process. - muxWriter *MuxWriter - // muxMetricsUpdater is the process to update metrics - muxMetricsUpdater muxMetricsUpdater - // newStreamChan is used to create new streams on the writer thread. - // The writer will assign the next available stream ID. - newStreamChan chan MuxedStreamRequest - // abortChan is used to abort the writer event loop. - abortChan chan struct{} - // abortOnce is used to ensure abortChan is closed once only. - abortOnce sync.Once - // readyList is used to signal writable streams. - readyList *ReadyList - // streams tracks currently-open streams. - streams *activeStreamMap - // explicitShutdown records whether the Muxer is closing because Shutdown was called, or due to another - // error. - explicitShutdown *BooleanFuse - - compressionQuality CompressionPreset -} - -func RPCHeaders() []Header { - return []Header{ - {Name: ":method", Value: "RPC"}, - {Name: ":scheme", Value: "capnp"}, - {Name: ":path", Value: "*"}, - } -} - -// Handshake establishes a muxed connection with the peer. -// After the handshake completes, it is possible to open and accept streams. -func Handshake( - w io.WriteCloser, - r io.ReadCloser, - config MuxerConfig, - activeStreamsMetrics prometheus.Gauge, -) (*Muxer, error) { - // Set default config values - if config.Timeout == 0 { - config.Timeout = defaultTimeout - } - if config.DefaultWindowSize == 0 { - config.DefaultWindowSize = defaultWindowSize - } - if config.MaxWindowSize == 0 { - config.MaxWindowSize = maxWindowSize - } - if config.StreamWriteBufferMaxLen == 0 { - config.StreamWriteBufferMaxLen = defaultWriteBufferMaxLen - } - // Initialise connection state fields - m := &Muxer{ - f: http2.NewFramer(w, r), // A framer that writes to w and reads from r - config: config, - w: w, - r: r, - newStreamChan: make(chan MuxedStreamRequest), - abortChan: make(chan struct{}), - readyList: NewReadyList(), - streams: newActiveStreamMap(config.IsClient, activeStreamsMetrics), - } - - m.f.ReadMetaHeaders = hpack.NewDecoder(4096, func(hpack.HeaderField) {}) - // Initialise the settings to identify this connection and confirm the other end is sane. - handshakeSetting := http2.Setting{ID: SettingMuxerMagic, Val: MuxerMagicEdge} - compressionSetting := http2.Setting{ID: SettingCompression, Val: 0} - - expectedMagic := MuxerMagicOrigin - if config.IsClient { - handshakeSetting.Val = MuxerMagicOrigin - expectedMagic = MuxerMagicEdge - } - errChan := make(chan error, 2) - // Simultaneously send our settings and verify the peer's settings. - go func() { errChan <- m.f.WriteSettings(handshakeSetting, compressionSetting) }() - go func() { errChan <- m.readPeerSettings(expectedMagic) }() - err := joinErrorsWithTimeout(errChan, 2, config.Timeout, ErrHandshakeTimeout) - if err != nil { - return nil, err - } - // Confirm sanity by ACKing the frame and expecting an ACK for our frame. - // Not strictly necessary, but let's pretend to be H2-like. - go func() { errChan <- m.f.WriteSettingsAck() }() - go func() { errChan <- m.readPeerSettingsAck() }() - err = joinErrorsWithTimeout(errChan, 2, config.Timeout, ErrHandshakeTimeout) - if err != nil { - return nil, err - } - - // set up reader/writer pair ready for serve - streamErrors := NewStreamErrorMap() - goAwayChan := make(chan http2.ErrCode, 1) - inBoundCounter := NewAtomicCounter(0) - outBoundCounter := NewAtomicCounter(0) - pingTimestamp := NewPingTimestamp() - connActive := NewSignal() - idleDuration := config.HeartbeatInterval - // Sanity check to ensure idelDuration is sane - if idleDuration == 0 || idleDuration < defaultTimeout { - idleDuration = defaultTimeout - config.Log.Info().Msgf("muxer: Minimum idle time has been adjusted to %d", defaultTimeout) - } - maxRetries := config.MaxHeartbeats - if maxRetries == 0 { - maxRetries = defaultRetries - config.Log.Info().Msgf("muxer: Minimum number of unacked heartbeats to send before closing the connection has been adjusted to %d", maxRetries) - } - - compBytesBefore, compBytesAfter := NewAtomicCounter(0), NewAtomicCounter(0) - - m.muxMetricsUpdater = newMuxMetricsUpdater( - m.abortChan, - compBytesBefore, - compBytesAfter, - ) - - m.explicitShutdown = NewBooleanFuse() - m.muxReader = &MuxReader{ - f: m.f, - handler: m.config.Handler, - streams: m.streams, - readyList: m.readyList, - streamErrors: streamErrors, - goAwayChan: goAwayChan, - abortChan: m.abortChan, - pingTimestamp: pingTimestamp, - connActive: connActive, - initialStreamWindow: m.config.DefaultWindowSize, - streamWindowMax: m.config.MaxWindowSize, - streamWriteBufferMaxLen: m.config.StreamWriteBufferMaxLen, - r: m.r, - metricsUpdater: m.muxMetricsUpdater, - bytesRead: inBoundCounter, - } - m.muxWriter = &MuxWriter{ - f: m.f, - streams: m.streams, - streamErrors: streamErrors, - readyStreamChan: m.readyList.ReadyChannel(), - newStreamChan: m.newStreamChan, - goAwayChan: goAwayChan, - abortChan: m.abortChan, - pingTimestamp: pingTimestamp, - idleTimer: NewIdleTimer(idleDuration, maxRetries), - connActiveChan: connActive.WaitChannel(), - maxFrameSize: defaultFrameSize, - metricsUpdater: m.muxMetricsUpdater, - bytesWrote: outBoundCounter, - } - m.muxWriter.headerEncoder = hpack.NewEncoder(&m.muxWriter.headerBuffer) - - if m.compressionQuality.dictSize > 0 && m.compressionQuality.nDicts > 0 { - nd, sz := m.compressionQuality.nDicts, m.compressionQuality.dictSize - writeDicts, dictChan := newH2WriteDictionaries( - nd, - sz, - m.compressionQuality.quality, - compBytesBefore, - compBytesAfter, - ) - readDicts := newH2ReadDictionaries(nd, sz) - m.muxReader.dictionaries = h2Dictionaries{read: &readDicts, write: writeDicts} - m.muxWriter.useDictChan = dictChan - } - - return m, nil -} - -func (m *Muxer) readPeerSettings(magic uint32) error { - frame, err := m.f.ReadFrame() - if err != nil { - return err - } - settingsFrame, ok := frame.(*http2.SettingsFrame) - if !ok { - return ErrBadHandshakeNotSettings - } - if settingsFrame.Header().Flags != 0 { - return ErrBadHandshakeUnexpectedAck - } - peerMagic, ok := settingsFrame.Value(SettingMuxerMagic) - if !ok { - return ErrBadHandshakeNoMagic - } - if magic != peerMagic { - return ErrBadHandshakeWrongMagic - } - peerCompression, ok := settingsFrame.Value(SettingCompression) - if !ok { - m.compressionQuality = compressionPresets[CompressionNone] - return nil - } - ver, fmt, sz, nd := parseCompressionSettingVal(peerCompression) - if ver != compressionVersion || fmt != compressionFormat || sz == 0 || nd == 0 { - m.compressionQuality = compressionPresets[CompressionNone] - return nil - } - // Values used for compression are the minimum between the two peers - if sz < m.compressionQuality.dictSize { - m.compressionQuality.dictSize = sz - } - if nd < m.compressionQuality.nDicts { - m.compressionQuality.nDicts = nd - } - return nil -} - -func (m *Muxer) readPeerSettingsAck() error { - frame, err := m.f.ReadFrame() - if err != nil { - return err - } - settingsFrame, ok := frame.(*http2.SettingsFrame) - if !ok { - return ErrBadHandshakeNotSettingsAck - } - if settingsFrame.Header().Flags != http2.FlagSettingsAck { - return ErrBadHandshakeUnexpectedSettings - } - return nil -} - -func joinErrorsWithTimeout(errChan <-chan error, receiveCount int, timeout time.Duration, timeoutError error) error { - for i := 0; i < receiveCount; i++ { - select { - case err := <-errChan: - if err != nil { - return err - } - case <-time.After(timeout): - return timeoutError - } - } - return nil -} - -// Serve runs the event loops that comprise h2mux: -// - MuxReader.run() -// - MuxWriter.run() -// - muxMetricsUpdater.run() -// In the normal case, Shutdown() is called concurrently with Serve() to stop -// these loops. -func (m *Muxer) Serve(ctx context.Context) error { - errGroup, _ := errgroup.WithContext(ctx) - errGroup.Go(func() error { - ch := make(chan error) - go func() { - err := m.muxReader.run(m.config.Log) - m.explicitShutdown.Fuse(false) - m.r.Close() - m.abort() - // don't block if parent goroutine quit early - select { - case ch <- err: - default: - } - }() - select { - case err := <-ch: - return err - case <-ctx.Done(): - return ctx.Err() - } - }) - - errGroup.Go(func() error { - ch := make(chan error) - go func() { - err := m.muxWriter.run(m.config.Log) - m.explicitShutdown.Fuse(false) - m.w.Close() - m.abort() - // don't block if parent goroutine quit early - select { - case ch <- err: - default: - } - }() - select { - case err := <-ch: - return err - case <-ctx.Done(): - return ctx.Err() - } - }) - - errGroup.Go(func() error { - ch := make(chan error) - go func() { - err := m.muxMetricsUpdater.run(m.config.Log) - // don't block if parent goroutine quit early - select { - case ch <- err: - default: - } - }() - select { - case err := <-ch: - return err - case <-ctx.Done(): - return ctx.Err() - } - }) - - err := errGroup.Wait() - if isUnexpectedTunnelError(err, m.explicitShutdown.Value()) { - return err - } - return nil -} - -// Shutdown is called to initiate the "happy path" of muxer termination. -// It blocks new streams from being created. -// It returns a channel that is closed when the last stream has been closed. -func (m *Muxer) Shutdown() <-chan struct{} { - m.explicitShutdown.Fuse(true) - return m.muxReader.Shutdown() -} - -// IsUnexpectedTunnelError identifies errors that are expected when shutting down the h2mux tunnel. -// The set of expected errors change depending on whether we initiated shutdown or not. -func isUnexpectedTunnelError(err error, expectedShutdown bool) bool { - if err == nil { - return false - } - if !expectedShutdown { - return true - } - return !isConnectionClosedError(err) -} - -func isConnectionClosedError(err error) bool { - if err == io.EOF { - return true - } - if err == io.ErrClosedPipe { - return true - } - if err.Error() == "tls: use of closed connection" { - return true - } - if strings.HasSuffix(err.Error(), "use of closed network connection") { - return true - } - return false -} - -// OpenStream opens a new data stream with the given headers. -// Called by proxy server and tunnel -func (m *Muxer) OpenStream(ctx context.Context, headers []Header, body io.Reader) (*MuxedStream, error) { - stream := m.NewStream(headers) - if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, body)); err != nil { - return nil, err - } - if err := m.AwaitResponseHeaders(ctx, stream); err != nil { - return nil, err - } - return stream, nil -} - -func (m *Muxer) OpenRPCStream(ctx context.Context) (*MuxedStream, error) { - stream := m.NewStream(RPCHeaders()) - if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, nil)); err != nil { - stream.Close() - return nil, err - } - if err := m.AwaitResponseHeaders(ctx, stream); err != nil { - stream.Close() - return nil, err - } - if !IsRPCStreamResponse(stream) { - stream.Close() - return nil, ErrNotRPCStream - } - return stream, nil -} - -func (m *Muxer) NewStream(headers []Header) *MuxedStream { - return NewStream(m.config, headers, m.readyList, m.muxReader.dictionaries) -} - -func (m *Muxer) MakeMuxedStreamRequest(ctx context.Context, request MuxedStreamRequest) error { - select { - case <-ctx.Done(): - return ErrStreamRequestTimeout - case <-m.abortChan: - return ErrStreamRequestConnectionClosed - // Will be received by mux writer - case m.newStreamChan <- request: - return nil - } -} - -func (m *Muxer) CloseStreamRead(stream *MuxedStream) { - stream.CloseRead() - if stream.WriteClosed() { - m.streams.Delete(stream.streamID) - } -} - -func (m *Muxer) AwaitResponseHeaders(ctx context.Context, stream *MuxedStream) error { - select { - case <-ctx.Done(): - return ErrResponseHeadersTimeout - case <-m.abortChan: - return ErrResponseHeadersConnectionClosed - case <-stream.responseHeadersReceived: - return nil - } -} - -func (m *Muxer) Metrics() *MuxerMetrics { - return m.muxMetricsUpdater.metrics() -} - -func (m *Muxer) abort() { - m.abortOnce.Do(func() { - close(m.abortChan) - m.readyList.Close() - m.streams.Abort() - }) -} - -// Return how many retries/ticks since the connection was last marked active -func (m *Muxer) TimerRetries() uint64 { - return m.muxWriter.idleTimer.RetryCount() -} - -func IsRPCStreamResponse(stream *MuxedStream) bool { - headers := stream.Headers - return len(headers) == 1 && - headers[0].Name == ":status" && - headers[0].Value == "200" -} diff --git a/h2mux/h2mux_test.go b/h2mux/h2mux_test.go deleted file mode 100644 index de79068e..00000000 --- a/h2mux/h2mux_test.go +++ /dev/null @@ -1,909 +0,0 @@ -package h2mux - -import ( - "bytes" - "context" - "fmt" - "io" - "math/rand" - "net" - "os" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "golang.org/x/sync/errgroup" -) - -const ( - testOpenStreamTimeout = time.Millisecond * 5000 - testHandshakeTimeout = time.Millisecond * 1000 -) - -var log = zerolog.Nop() - -func TestMain(m *testing.M) { - if os.Getenv("VERBOSE") == "1" { - //TODO: set log level - } - os.Exit(m.Run()) -} - -type DefaultMuxerPair struct { - OriginMuxConfig MuxerConfig - OriginMux *Muxer - OriginConn net.Conn - EdgeMuxConfig MuxerConfig - EdgeMux *Muxer - EdgeConn net.Conn - doneC chan struct{} -} - -func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc) *DefaultMuxerPair { - origin, edge := net.Pipe() - p := &DefaultMuxerPair{ - OriginMuxConfig: MuxerConfig{ - Timeout: testHandshakeTimeout, - Handler: f, - IsClient: true, - Name: "origin", - Log: &log, - DefaultWindowSize: (1 << 8) - 1, - MaxWindowSize: (1 << 15) - 1, - StreamWriteBufferMaxLen: 1024, - HeartbeatInterval: defaultTimeout, - MaxHeartbeats: defaultRetries, - }, - OriginConn: origin, - EdgeMuxConfig: MuxerConfig{ - Timeout: testHandshakeTimeout, - IsClient: false, - Name: "edge", - Log: &log, - DefaultWindowSize: (1 << 8) - 1, - MaxWindowSize: (1 << 15) - 1, - StreamWriteBufferMaxLen: 1024, - HeartbeatInterval: defaultTimeout, - MaxHeartbeats: defaultRetries, - }, - EdgeConn: edge, - doneC: make(chan struct{}), - } - assert.NoError(t, p.Handshake(testName)) - return p -} - -func NewCompressedMuxerPair(t assert.TestingT, testName string, quality CompressionSetting, f MuxedStreamFunc) *DefaultMuxerPair { - origin, edge := net.Pipe() - p := &DefaultMuxerPair{ - OriginMuxConfig: MuxerConfig{ - Timeout: time.Second, - Handler: f, - IsClient: true, - Name: "origin", - CompressionQuality: quality, - Log: &log, - HeartbeatInterval: defaultTimeout, - MaxHeartbeats: defaultRetries, - }, - OriginConn: origin, - EdgeMuxConfig: MuxerConfig{ - Timeout: time.Second, - IsClient: false, - Name: "edge", - CompressionQuality: quality, - Log: &log, - HeartbeatInterval: defaultTimeout, - MaxHeartbeats: defaultRetries, - }, - EdgeConn: edge, - doneC: make(chan struct{}), - } - assert.NoError(t, p.Handshake(testName)) - return p -} - -func (p *DefaultMuxerPair) Handshake(testName string) error { - ctx, cancel := context.WithTimeout(context.Background(), testHandshakeTimeout) - defer cancel() - errGroup, _ := errgroup.WithContext(ctx) - errGroup.Go(func() (err error) { - p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, ActiveStreams) - return errors.Wrap(err, "edge handshake failure") - }) - errGroup.Go(func() (err error) { - p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, ActiveStreams) - return errors.Wrap(err, "origin handshake failure") - }) - - return errGroup.Wait() -} - -func (p *DefaultMuxerPair) Serve(t assert.TestingT) { - ctx := context.Background() - var wg sync.WaitGroup - wg.Add(2) - go func() { - err := p.EdgeMux.Serve(ctx) - if err != nil && err != io.EOF && err != io.ErrClosedPipe { - t.Errorf("error in edge muxer Serve(): %s", err) - } - p.OriginMux.Shutdown() - wg.Done() - }() - go func() { - err := p.OriginMux.Serve(ctx) - if err != nil && err != io.EOF && err != io.ErrClosedPipe { - t.Errorf("error in origin muxer Serve(): %s", err) - } - p.EdgeMux.Shutdown() - wg.Done() - }() - go func() { - // notify when both muxes have stopped serving - wg.Wait() - close(p.doneC) - }() -} - -func (p *DefaultMuxerPair) Wait(t *testing.T) { - select { - case <-p.doneC: - return - case <-time.After(5 * time.Second): - t.Fatal("timeout waiting for shutdown") - } -} - -func (p *DefaultMuxerPair) OpenEdgeMuxStream(headers []Header, body io.Reader) (*MuxedStream, error) { - ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout) - defer cancel() - return p.EdgeMux.OpenStream(ctx, headers, body) -} - -func TestHandshake(t *testing.T) { - f := func(stream *MuxedStream) error { - return nil - } - muxPair := NewDefaultMuxerPair(t, t.Name(), f) - AssertIfPipeReadable(t, muxPair.OriginConn) - AssertIfPipeReadable(t, muxPair.EdgeConn) -} - -func TestSingleStream(t *testing.T) { - f := MuxedStreamFunc(func(stream *MuxedStream) error { - if len(stream.Headers) != 1 { - t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "test-header" { - t.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "headerValue" { - t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value) - } - _ = stream.WriteHeaders([]Header{ - {Name: "response-header", Value: "responseValue"}, - }) - buf := []byte("Hello world") - _, _ = stream.Write(buf) - n, err := io.ReadFull(stream, buf) - if n > 0 { - t.Fatalf("read %d bytes after EOF", n) - } - if err != io.EOF { - t.Fatalf("expected EOF, got %s", err) - } - return nil - }) - muxPair := NewDefaultMuxerPair(t, t.Name(), f) - muxPair.Serve(t) - - stream, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - if err != nil { - t.Fatalf("error in OpenStream: %s", err) - } - if len(stream.Headers) != 1 { - t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "response-header" { - t.Fatalf("expected header name %s, got %s", "response-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "responseValue" { - t.Fatalf("expected header value %s, got %s", "responseValue", stream.Headers[0].Value) - } - responseBody := make([]byte, 11) - n, err := io.ReadFull(stream, responseBody) - if err != nil { - t.Fatalf("error from (*MuxedStream).Read: %s", err) - } - if n != len(responseBody) { - t.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n) - } - if string(responseBody) != "Hello world" { - t.Fatalf("expected response body %s, got %s", "Hello world", responseBody) - } - _ = stream.Close() - n, err = stream.Write([]byte("aaaaa")) - if n > 0 { - t.Fatalf("wrote %d bytes after EOF", n) - } - if err != io.EOF { - t.Fatalf("expected EOF, got %s", err) - } -} - -func TestSingleStreamLargeResponseBody(t *testing.T) { - bodySize := 1 << 24 - f := MuxedStreamFunc(func(stream *MuxedStream) error { - if len(stream.Headers) != 1 { - t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "test-header" { - t.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "headerValue" { - t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value) - } - _ = stream.WriteHeaders([]Header{ - {Name: "response-header", Value: "responseValue"}, - }) - payload := make([]byte, bodySize) - for i := range payload { - payload[i] = byte(i % 256) - } - t.Log("Writing payload...") - n, err := stream.Write(payload) - t.Logf("Wrote %d bytes into the stream", n) - if err != nil { - t.Fatalf("origin write error: %s", err) - } - if n != len(payload) { - t.Fatalf("origin short write: %d/%d bytes", n, len(payload)) - } - - return nil - }) - muxPair := NewDefaultMuxerPair(t, t.Name(), f) - muxPair.Serve(t) - - stream, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - if err != nil { - t.Fatalf("error in OpenStream: %s", err) - } - if len(stream.Headers) != 1 { - t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "response-header" { - t.Fatalf("expected header name %s, got %s", "response-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "responseValue" { - t.Fatalf("expected header value %s, got %s", "responseValue", stream.Headers[0].Value) - } - responseBody := make([]byte, bodySize) - - n, err := io.ReadFull(stream, responseBody) - if err != nil { - t.Fatalf("error from (*MuxedStream).Read: %s", err) - } - if n != len(responseBody) { - t.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n) - } -} - -func TestMultipleStreams(t *testing.T) { - f := MuxedStreamFunc(func(stream *MuxedStream) error { - if len(stream.Headers) != 1 { - t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "client-token" { - t.Fatalf("expected header name %s, got %s", "client-token", stream.Headers[0].Name) - } - log.Debug().Msgf("Got request for stream %s", stream.Headers[0].Value) - _ = stream.WriteHeaders([]Header{ - {Name: "response-token", Value: stream.Headers[0].Value}, - }) - log.Debug().Msgf("Wrote headers for stream %s", stream.Headers[0].Value) - _, _ = stream.Write([]byte("OK")) - log.Debug().Msgf("Wrote body for stream %s", stream.Headers[0].Value) - return nil - }) - muxPair := NewDefaultMuxerPair(t, t.Name(), f) - muxPair.Serve(t) - - maxStreams := 64 - errorsC := make(chan error, maxStreams) - var wg sync.WaitGroup - wg.Add(maxStreams) - for i := 0; i < maxStreams; i++ { - go func(tokenId int) { - defer wg.Done() - tokenString := fmt.Sprintf("%d", tokenId) - stream, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "client-token", Value: tokenString}}, - nil, - ) - log.Debug().Msgf("Got headers for stream %d", tokenId) - if err != nil { - errorsC <- err - return - } - if len(stream.Headers) != 1 { - errorsC <- fmt.Errorf("stream %d has error: expected %d headers, got %d", stream.streamID, 1, len(stream.Headers)) - return - } - if stream.Headers[0].Name != "response-token" { - errorsC <- fmt.Errorf("stream %d has error: expected header name %s, got %s", stream.streamID, "response-token", stream.Headers[0].Name) - return - } - if stream.Headers[0].Value != tokenString { - errorsC <- fmt.Errorf("stream %d has error: expected header value %s, got %s", stream.streamID, tokenString, stream.Headers[0].Value) - return - } - responseBody := make([]byte, 2) - n, err := io.ReadFull(stream, responseBody) - if err != nil { - errorsC <- fmt.Errorf("stream %d has error: error from (*MuxedStream).Read: %s", stream.streamID, err) - return - } - if n != len(responseBody) { - errorsC <- fmt.Errorf("stream %d has error: expected response body to have %d bytes, got %d", stream.streamID, len(responseBody), n) - return - } - if string(responseBody) != "OK" { - errorsC <- fmt.Errorf("stream %d has error: expected response body %s, got %s", stream.streamID, "OK", responseBody) - return - } - }(i) - } - wg.Wait() - close(errorsC) - testFail := false - for err := range errorsC { - testFail = true - log.Error().Msgf("%s", err) - } - if testFail { - t.Fatalf("TestMultipleStreams failed") - } -} - -func TestMultipleStreamsFlowControl(t *testing.T) { - maxStreams := 32 - responseSizes := make([]int32, maxStreams) - for i := 0; i < maxStreams; i++ { - responseSizes[i] = rand.Int31n(int32(defaultWindowSize << 4)) - } - - f := MuxedStreamFunc(func(stream *MuxedStream) error { - if len(stream.Headers) != 1 { - t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "test-header" { - t.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "headerValue" { - t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value) - } - _ = stream.WriteHeaders([]Header{ - {Name: "response-header", Value: "responseValue"}, - }) - payload := make([]byte, responseSizes[(stream.streamID-2)/2]) - for i := range payload { - payload[i] = byte(i % 256) - } - n, err := stream.Write(payload) - if err != nil { - t.Fatalf("origin write error: %s", err) - } - if n != len(payload) { - t.Fatalf("origin short write: %d/%d bytes", n, len(payload)) - } - return nil - }) - muxPair := NewDefaultMuxerPair(t, t.Name(), f) - muxPair.Serve(t) - - errGroup, _ := errgroup.WithContext(context.Background()) - for i := 0; i < maxStreams; i++ { - errGroup.Go(func() error { - stream, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - if err != nil { - return fmt.Errorf("error in OpenStream: %d %s", stream.streamID, err) - } - if len(stream.Headers) != 1 { - return fmt.Errorf("stream %d expected %d headers, got %d", stream.streamID, 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "response-header" { - return fmt.Errorf("stream %d expected header name %s, got %s", stream.streamID, "response-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "responseValue" { - return fmt.Errorf("stream %d expected header value %s, got %s", stream.streamID, "responseValue", stream.Headers[0].Value) - } - - responseBody := make([]byte, responseSizes[(stream.streamID-2)/2]) - n, err := io.ReadFull(stream, responseBody) - if err != nil { - return fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err) - } - if n != len(responseBody) { - return fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(responseBody), n) - } - return nil - }) - } - assert.NoError(t, errGroup.Wait()) -} - -func TestGracefulShutdown(t *testing.T) { - sendC := make(chan struct{}) - responseBuf := bytes.Repeat([]byte("Hello world"), 65536) - - f := MuxedStreamFunc(func(stream *MuxedStream) error { - _ = stream.WriteHeaders([]Header{ - {Name: "response-header", Value: "responseValue"}, - }) - <-sendC - log.Debug().Msgf("Writing %d bytes", len(responseBuf)) - _, _ = stream.Write(responseBuf) - _ = stream.CloseWrite() - log.Debug().Msgf("Wrote %d bytes", len(responseBuf)) - // Reading from the stream will block until the edge closes its end of the stream. - // Otherwise, we'll close the whole connection before receiving the 'stream closed' - // message from the edge. - // Graceful shutdown works if you omit this, it just gives spurious errors for now - - // TODO ignore errors when writing 'stream closed' and we're shutting down. - _, _ = stream.Read([]byte{0}) - log.Debug().Msgf("Handler ends") - return nil - }) - muxPair := NewDefaultMuxerPair(t, t.Name(), f) - muxPair.Serve(t) - - stream, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - if err != nil { - t.Fatalf("error in OpenStream: %s", err) - } - // Start graceful shutdown of the edge mux - this should also close the origin mux when done - muxPair.EdgeMux.Shutdown() - close(sendC) - responseBody := make([]byte, len(responseBuf)) - log.Debug().Msgf("Waiting for %d bytes", len(responseBuf)) - n, err := io.ReadFull(stream, responseBody) - if err != nil { - t.Fatalf("error from (*MuxedStream).Read with %d bytes read: %s", n, err) - } - if n != len(responseBody) { - t.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n) - } - if !bytes.Equal(responseBuf, responseBody) { - t.Fatalf("response body mismatch") - } - _ = stream.Close() - muxPair.Wait(t) -} - -func TestUnexpectedShutdown(t *testing.T) { - sendC := make(chan struct{}) - handlerFinishC := make(chan struct{}) - responseBuf := bytes.Repeat([]byte("Hello world"), 65536) - - f := MuxedStreamFunc(func(stream *MuxedStream) error { - defer close(handlerFinishC) - _ = stream.WriteHeaders([]Header{ - {Name: "response-header", Value: "responseValue"}, - }) - <-sendC - n, err := stream.Read([]byte{0}) - if err != io.EOF { - t.Fatalf("unexpected error from (*MuxedStream).Read: %s", err) - } - if n != 0 { - t.Fatalf("expected empty read, got %d bytes", n) - } - // Write comes after read, because write buffers data before it is flushed. It wouldn't know about EOF - // until some time later. Calling read first forces it to know about EOF now. - _, err = stream.Write(responseBuf) - if err != io.EOF { - t.Fatalf("unexpected error from (*MuxedStream).Write: %s", err) - } - return nil - }) - muxPair := NewDefaultMuxerPair(t, t.Name(), f) - muxPair.Serve(t) - - stream, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - // Close the underlying connection before telling the origin to write. - _ = muxPair.EdgeConn.Close() - close(sendC) - if err != nil { - t.Fatalf("error in OpenStream: %s", err) - } - responseBody := make([]byte, len(responseBuf)) - n, err := io.ReadFull(stream, responseBody) - if err != io.EOF { - t.Fatalf("unexpected error from (*MuxedStream).Read: %s", err) - } - if n != 0 { - t.Fatalf("expected response body to have %d bytes, got %d", 0, n) - } - // The write ordering requirement explained in the origin handler applies here too. - _, err = stream.Write(responseBuf) - if err != io.EOF { - t.Fatalf("unexpected error from (*MuxedStream).Write: %s", err) - } - <-handlerFinishC -} - -func EchoHandler(stream *MuxedStream) error { - var buf bytes.Buffer - _, _ = fmt.Fprintf(&buf, "Hello, world!\n\n# REQUEST HEADERS:\n\n") - for _, header := range stream.Headers { - _, _ = fmt.Fprintf(&buf, "[%s] = %s\n", header.Name, header.Value) - } - _ = stream.WriteHeaders([]Header{ - {Name: ":status", Value: "200"}, - {Name: "server", Value: "Echo-server/1.0"}, - {Name: "date", Value: time.Now().Format(time.RFC850)}, - {Name: "content-type", Value: "text/html; charset=utf-8"}, - {Name: "content-length", Value: strconv.Itoa(buf.Len())}, - }) - _, _ = buf.WriteTo(stream) - return nil -} - -func TestOpenAfterDisconnect(t *testing.T) { - for i := 0; i < 3; i++ { - muxPair := NewDefaultMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), i), EchoHandler) - muxPair.Serve(t) - - switch i { - case 0: - // Close both directions of the connection to cause EOF on both peers. - _ = muxPair.OriginConn.Close() - _ = muxPair.EdgeConn.Close() - case 1: - // Close origin conn to cause EOF on origin first. - _ = muxPair.OriginConn.Close() - case 2: - // Close edge conn to cause EOF on edge first. - _ = muxPair.EdgeConn.Close() - } - - _, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - if err != ErrStreamRequestConnectionClosed && err != ErrResponseHeadersConnectionClosed { - t.Fatalf("case %v: unexpected error in OpenStream: %v", i, err) - } - } -} - -func TestHPACK(t *testing.T) { - muxPair := NewDefaultMuxerPair(t, t.Name(), EchoHandler) - muxPair.Serve(t) - - stream, err := muxPair.OpenEdgeMuxStream( - []Header{ - {Name: ":method", Value: "RPC"}, - {Name: ":scheme", Value: "capnp"}, - {Name: ":path", Value: "*"}, - }, - nil, - ) - if err != nil { - t.Fatalf("error in OpenStream: %s", err) - } - _ = stream.Close() - - for i := 0; i < 3; i++ { - stream, err := muxPair.OpenEdgeMuxStream( - []Header{ - {Name: ":method", Value: "GET"}, - {Name: ":scheme", Value: "https"}, - {Name: ":authority", Value: "tunnel.otterlyadorable.co.uk"}, - {Name: ":path", Value: "/get"}, - {Name: "accept-encoding", Value: "gzip"}, - {Name: "cf-ray", Value: "378948953f044408-SFO-DOG"}, - {Name: "cf-visitor", Value: "{\"scheme\":\"https\"}"}, - {Name: "cf-connecting-ip", Value: "2400:cb00:0025:010d:0000:0000:0000:0001"}, - {Name: "x-forwarded-for", Value: "2400:cb00:0025:010d:0000:0000:0000:0001"}, - {Name: "x-forwarded-proto", Value: "https"}, - {Name: "accept-language", Value: "en-gb"}, - {Name: "referer", Value: "https://tunnel.otterlyadorable.co.uk/"}, - {Name: "cookie", Value: "__cfduid=d4555095065f92daedc059490771967d81493032162"}, - {Name: "connection", Value: "Keep-Alive"}, - {Name: "cf-ipcountry", Value: "US"}, - {Name: "accept", Value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - {Name: "user-agent", Value: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4"}, - }, - nil, - ) - if err != nil { - t.Fatalf("error in OpenStream: %s", err) - } - if len(stream.Headers) == 0 { - t.Fatal("response has no headers") - } - if stream.Headers[0].Name != ":status" { - t.Fatalf("first header should be status, found %s instead", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "200" { - t.Fatalf("expected status 200, got %s", stream.Headers[0].Value) - } - _, _ = io.ReadAll(stream) - _ = stream.Close() - } -} - -func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) { - errC := make(chan error) - go func() { - b := []byte{0} - n, err := pipe.Read(b) - if n > 0 { - t.Errorf("read pipe was not empty") - return - } - errC <- err - }() - select { - case err := <-errC: - if err != nil { - t.Fatalf("read error: %s", err) - } - case <-time.After(100 * time.Millisecond): - // nothing to read - } -} - -func sampleSiteHandler(files map[string][]byte) MuxedStreamFunc { - return func(stream *MuxedStream) error { - var contentType string - var pathHeader Header - - for _, h := range stream.Headers { - if h.Name == ":path" { - pathHeader = h - break - } - } - - if pathHeader.Name != ":path" { - return fmt.Errorf("Couldn't find :path header in test") - } - - if strings.Contains(pathHeader.Value, "html") { - contentType = "text/html; charset=utf-8" - } else if strings.Contains(pathHeader.Value, "js") { - contentType = "application/javascript" - } else if strings.Contains(pathHeader.Value, "css") { - contentType = "text/css" - } else { - contentType = "img/gif" - } - _ = stream.WriteHeaders([]Header{ - {Name: "content-type", Value: contentType}, - }) - log.Debug().Msgf("Wrote headers for stream %s", pathHeader.Value) - file, ok := files[pathHeader.Value] - if !ok { - return fmt.Errorf("%s content is not preloaded", pathHeader.Value) - } - _, _ = stream.Write(file) - log.Debug().Msgf("Wrote body for stream %s", pathHeader.Value) - return nil - } -} - -func sampleSiteTest(muxPair *DefaultMuxerPair, path string, files map[string][]byte) error { - stream, err := muxPair.OpenEdgeMuxStream( - []Header{ - {Name: ":method", Value: "GET"}, - {Name: ":scheme", Value: "https"}, - {Name: ":authority", Value: "tunnel.otterlyadorable.co.uk"}, - {Name: ":path", Value: path}, - {Name: "accept-encoding", Value: "br, gzip"}, - {Name: "cf-ray", Value: "378948953f044408-SFO-DOG"}, - }, - nil, - ) - if err != nil { - return fmt.Errorf("error in OpenStream: %v", err) - } - file, ok := files[path] - if !ok { - return fmt.Errorf("%s content is not preloaded", path) - } - responseBody := make([]byte, len(file)) - n, err := io.ReadFull(stream, responseBody) - if err != nil { - return fmt.Errorf("error from (*MuxedStream).Read: %v", err) - } - if n != len(file) { - return fmt.Errorf("expected response body to have %d bytes, got %d", len(file), n) - } - if string(responseBody[:n]) != string(file) { - return fmt.Errorf("expected response body %s, got %s", file, responseBody[:n]) - } - return nil -} - -func loadSampleFiles(paths []string) (map[string][]byte, error) { - files := make(map[string][]byte) - for _, path := range paths { - if _, ok := files[path]; !ok { - expectBody, err := os.ReadFile(path) - if err != nil { - return nil, err - } - files[path] = expectBody - } - } - return files, nil -} - -func BenchmarkOpenStream(b *testing.B) { - const streams = 5000 - for i := 0; i < b.N; i++ { - b.StopTimer() - f := MuxedStreamFunc(func(stream *MuxedStream) error { - if len(stream.Headers) != 1 { - b.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "test-header" { - b.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "headerValue" { - b.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value) - } - _ = stream.WriteHeaders([]Header{ - {Name: "response-header", Value: "responseValue"}, - }) - return nil - }) - muxPair := NewDefaultMuxerPair(b, fmt.Sprintf("%s_%d", b.Name(), i), f) - muxPair.Serve(b) - b.StartTimer() - openStreams(b, muxPair, streams) - } -} - -func openStreams(b *testing.B, muxPair *DefaultMuxerPair, n int) { - errGroup, _ := errgroup.WithContext(context.Background()) - for i := 0; i < n; i++ { - errGroup.Go(func() error { - _, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - return err - }) - } - assert.NoError(b, errGroup.Wait()) -} - -func BenchmarkSingleStreamLargeResponseBody(b *testing.B) { - const bodySize = 1 << 24 - - const writeBufferSize = 16 << 10 - const writeN = bodySize / writeBufferSize - payload := make([]byte, writeBufferSize) - for i := range payload { - payload[i] = byte(i % 256) - } - - const readBufferSize = 16 << 10 - const readN = bodySize / readBufferSize - responseBody := make([]byte, readBufferSize) - - f := MuxedStreamFunc(func(stream *MuxedStream) error { - if len(stream.Headers) != 1 { - b.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "test-header" { - b.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "headerValue" { - b.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value) - } - _ = stream.WriteHeaders([]Header{ - {Name: "response-header", Value: "responseValue"}, - }) - for i := 0; i < writeN; i++ { - n, err := stream.Write(payload) - if err != nil { - b.Fatalf("origin write error: %s", err) - } - if n != len(payload) { - b.Fatalf("origin short write: %d/%d bytes", n, len(payload)) - } - } - - return nil - }) - - name := fmt.Sprintf("%s_%d", b.Name(), rand.Int()) - origin, edge := net.Pipe() - - muxPair := &DefaultMuxerPair{ - OriginMuxConfig: MuxerConfig{ - Timeout: testHandshakeTimeout, - Handler: f, - IsClient: true, - Name: "origin", - Log: &log, - DefaultWindowSize: defaultWindowSize, - MaxWindowSize: maxWindowSize, - StreamWriteBufferMaxLen: defaultWriteBufferMaxLen, - HeartbeatInterval: defaultTimeout, - MaxHeartbeats: defaultRetries, - }, - OriginConn: origin, - EdgeMuxConfig: MuxerConfig{ - Timeout: testHandshakeTimeout, - IsClient: false, - Name: "edge", - Log: &log, - DefaultWindowSize: defaultWindowSize, - MaxWindowSize: maxWindowSize, - StreamWriteBufferMaxLen: defaultWriteBufferMaxLen, - HeartbeatInterval: defaultTimeout, - MaxHeartbeats: defaultRetries, - }, - EdgeConn: edge, - doneC: make(chan struct{}), - } - assert.NoError(b, muxPair.Handshake(name)) - muxPair.Serve(b) - - b.ReportAllocs() - for i := 0; i < b.N; i++ { - stream, err := muxPair.OpenEdgeMuxStream( - []Header{{Name: "test-header", Value: "headerValue"}}, - nil, - ) - if err != nil { - b.Fatalf("error in OpenStream: %s", err) - } - if len(stream.Headers) != 1 { - b.Fatalf("expected %d headers, got %d", 1, len(stream.Headers)) - } - if stream.Headers[0].Name != "response-header" { - b.Fatalf("expected header name %s, got %s", "response-header", stream.Headers[0].Name) - } - if stream.Headers[0].Value != "responseValue" { - b.Fatalf("expected header value %s, got %s", "responseValue", stream.Headers[0].Value) - } - - for k := 0; k < readN; k++ { - n, err := io.ReadFull(stream, responseBody) - if err != nil { - b.Fatalf("error from (*MuxedStream).Read: %s", err) - } - if n != len(responseBody) { - b.Fatalf("expected response body to have %d bytes, got %d", len(responseBody), n) - } - } - } -} diff --git a/h2mux/idletimer.go b/h2mux/idletimer.go deleted file mode 100644 index 6e171801..00000000 --- a/h2mux/idletimer.go +++ /dev/null @@ -1,81 +0,0 @@ -package h2mux - -import ( - "math/rand" - "sync" - "time" -) - -// IdleTimer is a type of Timer designed for managing heartbeats on an idle connection. -// The timer ticks on an interval with added jitter to avoid accidental synchronisation -// between two endpoints. It tracks the number of retries/ticks since the connection was -// last marked active. -// -// The methods of IdleTimer must not be called while a goroutine is reading from C. -type IdleTimer struct { - // The channel on which ticks are delivered. - C <-chan time.Time - - // A timer used to measure idle connection time. Reset after sending data. - idleTimer *time.Timer - // The maximum length of time a connection is idle before sending a ping. - idleDuration time.Duration - // A pseudorandom source used to add jitter to the idle duration. - randomSource *rand.Rand - // The maximum number of retries allowed. - maxRetries uint64 - // The number of retries since the connection was last marked active. - retries uint64 - // A lock to prevent race condition while checking retries - stateLock sync.RWMutex -} - -func NewIdleTimer(idleDuration time.Duration, maxRetries uint64) *IdleTimer { - t := &IdleTimer{ - idleTimer: time.NewTimer(idleDuration), - idleDuration: idleDuration, - randomSource: rand.New(rand.NewSource(time.Now().Unix())), - maxRetries: maxRetries, - } - t.C = t.idleTimer.C - return t -} - -// Retry should be called when retrying the idle timeout. If the maximum number of retries -// has been met, returns false. -// After calling this function and sending a heartbeat, call ResetTimer. Since sending the -// heartbeat could be a blocking operation, we resetting the timer after the write completes -// to avoid it expiring during the write. -func (t *IdleTimer) Retry() bool { - t.stateLock.Lock() - defer t.stateLock.Unlock() - if t.retries >= t.maxRetries { - return false - } - t.retries++ - return true -} - -func (t *IdleTimer) RetryCount() uint64 { - t.stateLock.RLock() - defer t.stateLock.RUnlock() - return t.retries -} - -// MarkActive resets the idle connection timer and suppresses any outstanding idle events. -func (t *IdleTimer) MarkActive() { - if !t.idleTimer.Stop() { - // eat the timer event to prevent spurious pings - <-t.idleTimer.C - } - t.stateLock.Lock() - t.retries = 0 - t.stateLock.Unlock() - t.ResetTimer() -} - -// Reset the idle timer according to the configured duration, with some added jitter. -func (t *IdleTimer) ResetTimer() { - jitter := time.Duration(t.randomSource.Int63n(int64(t.idleDuration))) - t.idleTimer.Reset(t.idleDuration + jitter) -} diff --git a/h2mux/idletimer_test.go b/h2mux/idletimer_test.go deleted file mode 100644 index 92f2b2a3..00000000 --- a/h2mux/idletimer_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package h2mux - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestRetry(t *testing.T) { - timer := NewIdleTimer(time.Second, 2) - assert.Equal(t, uint64(0), timer.RetryCount()) - ok := timer.Retry() - assert.True(t, ok) - assert.Equal(t, uint64(1), timer.RetryCount()) - ok = timer.Retry() - assert.True(t, ok) - assert.Equal(t, uint64(2), timer.RetryCount()) - ok = timer.Retry() - assert.False(t, ok) -} - -func TestMarkActive(t *testing.T) { - timer := NewIdleTimer(time.Second, 2) - assert.Equal(t, uint64(0), timer.RetryCount()) - ok := timer.Retry() - assert.True(t, ok) - assert.Equal(t, uint64(1), timer.RetryCount()) - timer.MarkActive() - assert.Equal(t, uint64(0), timer.RetryCount()) -} diff --git a/h2mux/muxedstream.go b/h2mux/muxedstream.go deleted file mode 100644 index 2e75735f..00000000 --- a/h2mux/muxedstream.go +++ /dev/null @@ -1,457 +0,0 @@ -package h2mux - -import ( - "bytes" - "io" - "sync" -) - -type ReadWriteLengther interface { - io.ReadWriter - Reset() - Len() int -} - -type ReadWriteClosedCloser interface { - io.ReadWriteCloser - Closed() bool -} - -// MuxedStreamDataSignaller is a write-only *ReadyList -type MuxedStreamDataSignaller interface { - // Non-blocking: call this when data is ready to be sent for the given stream ID. - Signal(ID uint32) -} - -type Header struct { - Name, Value string -} - -// MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data. -type MuxedStream struct { - streamID uint32 - - // The "Receive" end of the stream - readBufferLock sync.RWMutex - readBuffer ReadWriteClosedCloser - // This is the amount of bytes that are in our receive window - // (how much data we can receive into this stream). - receiveWindow uint32 - // current receive window size limit. Exponentially increase it when it's exhausted - receiveWindowCurrentMax uint32 - // hard limit set in http2 spec. 2^31-1 - receiveWindowMax uint32 - // The desired size increment for receiveWindow. - // If this is nonzero, a WINDOW_UPDATE frame needs to be sent. - windowUpdate uint32 - // The headers that were most recently received. - // Particularly: - // * for an eyeball-initiated stream (as passed to TunnelHandler::ServeStream), - // these are the request headers - // * for a cloudflared-initiated stream (as created by Register/UnregisterTunnel), - // these are the response headers. - // They are useful in both of these contexts; hence `Headers` is public. - Headers []Header - // For use in the context of a cloudflared-initiated stream. - responseHeadersReceived chan struct{} - - // The "Send" end of the stream - writeLock sync.Mutex - writeBuffer ReadWriteLengther - // The maximum capacity that the send buffer should grow to. - writeBufferMaxLen int - // A channel to be notified when the send buffer is not full. - writeBufferHasSpace chan struct{} - // This is the amount of bytes that are in the peer's receive window - // (how much data we can send from this stream). - sendWindow uint32 - // The muxer's readyList - readyList MuxedStreamDataSignaller - // The headers that should be sent, and a flag so we only send them once. - headersSent bool - writeHeaders []Header - - // EOF-related fields - // true if the write end of this stream has been closed - writeEOF bool - // true if we have sent EOF to the peer - sentEOF bool - // true if the peer sent us an EOF - receivedEOF bool - // Compression-related fields - receivedUseDict bool - method string - contentType string - path string - dictionaries h2Dictionaries -} - -type TunnelHostname string - -func (th TunnelHostname) String() string { - return string(th) -} - -func (th TunnelHostname) IsSet() bool { - return th != "" -} - -func NewStream(config MuxerConfig, writeHeaders []Header, readyList MuxedStreamDataSignaller, dictionaries h2Dictionaries) *MuxedStream { - return &MuxedStream{ - responseHeadersReceived: make(chan struct{}), - readBuffer: NewSharedBuffer(), - writeBuffer: &bytes.Buffer{}, - writeBufferMaxLen: config.StreamWriteBufferMaxLen, - writeBufferHasSpace: make(chan struct{}, 1), - receiveWindow: config.DefaultWindowSize, - receiveWindowCurrentMax: config.DefaultWindowSize, - receiveWindowMax: config.MaxWindowSize, - sendWindow: config.DefaultWindowSize, - readyList: readyList, - writeHeaders: writeHeaders, - dictionaries: dictionaries, - } -} - -func (s *MuxedStream) Read(p []byte) (n int, err error) { - var readBuffer ReadWriteClosedCloser - if s.dictionaries.read != nil { - s.readBufferLock.RLock() - readBuffer = s.readBuffer - s.readBufferLock.RUnlock() - } else { - readBuffer = s.readBuffer - } - n, err = readBuffer.Read(p) - s.replenishReceiveWindow(uint32(n)) - return -} - -// Blocks until len(p) bytes have been written to the buffer -func (s *MuxedStream) Write(p []byte) (int, error) { - // If assignDictToStream returns success, then it will have acquired the - // writeLock. Otherwise we must acquire it ourselves. - ok := assignDictToStream(s, p) - if !ok { - s.writeLock.Lock() - } - defer s.writeLock.Unlock() - - if s.writeEOF { - return 0, io.EOF - } - - // pre-allocate some space in the write buffer if possible - if buffer, ok := s.writeBuffer.(*bytes.Buffer); ok { - if buffer.Cap() == 0 { - buffer.Grow(writeBufferInitialSize) - } - } - - totalWritten := 0 - for totalWritten < len(p) { - // If the buffer is full, block till there is more room. - // Use a loop to recheck the buffer size after the lock is reacquired. - for s.writeBufferMaxLen <= s.writeBuffer.Len() { - s.awaitWriteBufferHasSpace() - if s.writeEOF { - return totalWritten, io.EOF - } - } - amountToWrite := len(p) - totalWritten - spaceAvailable := s.writeBufferMaxLen - s.writeBuffer.Len() - if spaceAvailable < amountToWrite { - amountToWrite = spaceAvailable - } - amountWritten, err := s.writeBuffer.Write(p[totalWritten : totalWritten+amountToWrite]) - totalWritten += amountWritten - if err != nil { - return totalWritten, err - } - s.writeNotify() - } - return totalWritten, nil -} - -func (s *MuxedStream) Close() error { - // TUN-115: Close the write buffer before the read buffer. - // In the case of shutdown, read will not get new data, but the write buffer can still receive - // new data. Closing read before write allows application to race between a failed read and a - // successful write, even though this close should appear to be atomic. - // This can't happen the other way because reads may succeed after a failed write; if we read - // past EOF the application will block until we close the buffer. - err := s.CloseWrite() - if err != nil { - if s.CloseRead() == nil { - // don't bother the caller with errors if at least one close succeeded - return nil - } - return err - } - return s.CloseRead() -} - -func (s *MuxedStream) CloseRead() error { - return s.readBuffer.Close() -} - -func (s *MuxedStream) CloseWrite() error { - s.writeLock.Lock() - defer s.writeLock.Unlock() - if s.writeEOF { - return io.EOF - } - s.writeEOF = true - if c, ok := s.writeBuffer.(io.Closer); ok { - c.Close() - } - // Allow MuxedStream::Write() to terminate its loop with err=io.EOF, if needed - s.notifyWriteBufferHasSpace() - // We need to send something over the wire, even if it's an END_STREAM with no data - s.writeNotify() - return nil -} - -func (s *MuxedStream) WriteClosed() bool { - s.writeLock.Lock() - defer s.writeLock.Unlock() - return s.writeEOF -} - -func (s *MuxedStream) WriteHeaders(headers []Header) error { - s.writeLock.Lock() - defer s.writeLock.Unlock() - if s.writeHeaders != nil { - return ErrStreamHeadersSent - } - - if s.dictionaries.write != nil { - dictWriter := s.dictionaries.write.getDictWriter(s, headers) - if dictWriter != nil { - s.writeBuffer = dictWriter - } - - } - - s.writeHeaders = headers - s.headersSent = false - s.writeNotify() - return nil -} - -// IsRPCStream returns if the stream is used to transport RPC. -func (s *MuxedStream) IsRPCStream() bool { - rpcHeaders := RPCHeaders() - if len(s.Headers) != len(rpcHeaders) { - return false - } - // The headers order matters, so RPC stream should be opened with OpenRPCStream method and let MuxWriter serializes the headers. - for i, rpcHeader := range rpcHeaders { - if s.Headers[i] != rpcHeader { - return false - } - } - return true -} - -// Block until a value is sent on writeBufferHasSpace. -// Must be called while holding writeLock -func (s *MuxedStream) awaitWriteBufferHasSpace() { - s.writeLock.Unlock() - <-s.writeBufferHasSpace - s.writeLock.Lock() -} - -// Send a value on writeBufferHasSpace without blocking. -// Must be called while holding writeLock -func (s *MuxedStream) notifyWriteBufferHasSpace() { - select { - case s.writeBufferHasSpace <- struct{}{}: - default: - } -} - -func (s *MuxedStream) getReceiveWindow() uint32 { - s.writeLock.Lock() - defer s.writeLock.Unlock() - return s.receiveWindow -} - -func (s *MuxedStream) getSendWindow() uint32 { - s.writeLock.Lock() - defer s.writeLock.Unlock() - return s.sendWindow -} - -// writeNotify must happen while holding writeLock. -func (s *MuxedStream) writeNotify() { - s.readyList.Signal(s.streamID) -} - -// Call by muxreader when it gets a WindowUpdateFrame. This is an update of the peer's -// receive window (how much data we can send). -func (s *MuxedStream) replenishSendWindow(bytes uint32) { - s.writeLock.Lock() - defer s.writeLock.Unlock() - s.sendWindow += bytes - s.writeNotify() -} - -// Call by muxreader when it receives a data frame -func (s *MuxedStream) consumeReceiveWindow(bytes uint32) bool { - s.writeLock.Lock() - defer s.writeLock.Unlock() - // received data size is greater than receive window/buffer - if s.receiveWindow < bytes { - return false - } - s.receiveWindow -= bytes - if s.receiveWindow < s.receiveWindowCurrentMax/2 && s.receiveWindowCurrentMax < s.receiveWindowMax { - // exhausting client send window (how much data client can send) - // and there is room to grow the receive window - newMax := s.receiveWindowCurrentMax << 1 - if newMax > s.receiveWindowMax { - newMax = s.receiveWindowMax - } - s.windowUpdate += newMax - s.receiveWindowCurrentMax - s.receiveWindowCurrentMax = newMax - // notify MuxWriter to write WINDOW_UPDATE frame - s.writeNotify() - } - return true -} - -// Arranges for the MuxWriter to send a WINDOW_UPDATE -// Called by MuxedStream::Read when data has left the read buffer. -func (s *MuxedStream) replenishReceiveWindow(bytes uint32) { - s.writeLock.Lock() - defer s.writeLock.Unlock() - s.windowUpdate += bytes - s.writeNotify() -} - -// receiveEOF should be called when the peer indicates no more data will be sent. -// Returns true if the socket is now closed (i.e. the write side is already closed). -func (s *MuxedStream) receiveEOF() (closed bool) { - s.writeLock.Lock() - defer s.writeLock.Unlock() - s.receivedEOF = true - s.CloseRead() - return s.writeEOF && s.writeBuffer.Len() == 0 -} - -func (s *MuxedStream) gotReceiveEOF() bool { - s.writeLock.Lock() - defer s.writeLock.Unlock() - return s.receivedEOF -} - -// MuxedStreamReader implements io.ReadCloser for the read end of the stream. -// This is useful for passing to functions that close the object after it is done reading, -// but you still want to be able to write data afterwards (e.g. http.Client). -type MuxedStreamReader struct { - *MuxedStream -} - -func (s MuxedStreamReader) Read(p []byte) (n int, err error) { - return s.MuxedStream.Read(p) -} - -func (s MuxedStreamReader) Close() error { - return s.MuxedStream.CloseRead() -} - -// streamChunk represents a chunk of data to be written. -type streamChunk struct { - streamID uint32 - // true if a HEADERS frame should be sent - sendHeaders bool - headers []Header - // nonzero if a WINDOW_UPDATE frame should be sent; - // in that case, it is the increment value to use - windowUpdate uint32 - // true if data frames should be sent - sendData bool - eof bool - - buffer []byte - offset int -} - -// getChunk atomically extracts a chunk of data to be written by MuxWriter. -// The data returned will not exceed the send window for this stream. -func (s *MuxedStream) getChunk() *streamChunk { - s.writeLock.Lock() - defer s.writeLock.Unlock() - - chunk := &streamChunk{ - streamID: s.streamID, - sendHeaders: !s.headersSent, - headers: s.writeHeaders, - windowUpdate: s.windowUpdate, - sendData: !s.sentEOF, - eof: s.writeEOF && uint32(s.writeBuffer.Len()) <= s.sendWindow, - } - // Copy at most s.sendWindow bytes, adjust the sendWindow accordingly - toCopy := int(s.sendWindow) - if toCopy > s.writeBuffer.Len() { - toCopy = s.writeBuffer.Len() - } - - if toCopy > 0 { - buf := make([]byte, toCopy) - writeLen, _ := s.writeBuffer.Read(buf) - chunk.buffer = buf[:writeLen] - s.sendWindow -= uint32(writeLen) - } - - // Allow MuxedStream::Write() to continue, if needed - if s.writeBuffer.Len() < s.writeBufferMaxLen { - s.notifyWriteBufferHasSpace() - } - - // When we write the chunk, we'll write the WINDOW_UPDATE frame if needed - s.receiveWindow += s.windowUpdate - s.windowUpdate = 0 - - // When we write the chunk, we'll write the headers if needed - s.headersSent = true - - // if this chunk contains the end of the stream, close the stream now - if chunk.sendData && chunk.eof { - s.sentEOF = true - } - - return chunk -} - -func (c *streamChunk) sendHeadersFrame() bool { - return c.sendHeaders -} - -func (c *streamChunk) sendWindowUpdateFrame() bool { - return c.windowUpdate > 0 -} - -func (c *streamChunk) sendDataFrame() bool { - return c.sendData -} - -func (c *streamChunk) nextDataFrame(frameSize int) (payload []byte, endStream bool) { - bytesLeft := len(c.buffer) - c.offset - if frameSize > bytesLeft { - frameSize = bytesLeft - } - nextOffset := c.offset + frameSize - payload = c.buffer[c.offset:nextOffset] - c.offset = nextOffset - - if c.offset == len(c.buffer) { - // this is the last data frame in this chunk - c.sendData = false - if c.eof { - endStream = true - } - } - return -} diff --git a/h2mux/muxedstream_test.go b/h2mux/muxedstream_test.go deleted file mode 100644 index b0e0ac13..00000000 --- a/h2mux/muxedstream_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package h2mux - -import ( - "bytes" - "io" - "testing" - - "github.com/stretchr/testify/assert" -) - -const testWindowSize uint32 = 65535 -const testMaxWindowSize uint32 = testWindowSize << 2 - -// Only sending WINDOW_UPDATE frame, so sendWindow should never change -func TestFlowControlSingleStream(t *testing.T) { - stream := &MuxedStream{ - responseHeadersReceived: make(chan struct{}), - readBuffer: NewSharedBuffer(), - writeBuffer: &bytes.Buffer{}, - receiveWindow: testWindowSize, - receiveWindowCurrentMax: testWindowSize, - receiveWindowMax: testMaxWindowSize, - sendWindow: testWindowSize, - readyList: NewReadyList(), - } - var tempWindowUpdate uint32 - var tempStreamChunk *streamChunk - - assert.True(t, stream.consumeReceiveWindow(testWindowSize/2)) - dataSent := testWindowSize / 2 - assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow) - assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax) - assert.Equal(t, testWindowSize, stream.sendWindow) - assert.Equal(t, uint32(0), stream.windowUpdate) - - tempStreamChunk = stream.getChunk() - assert.Equal(t, uint32(0), tempStreamChunk.windowUpdate) - assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow) - assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax) - assert.Equal(t, testWindowSize, stream.sendWindow) - assert.Equal(t, uint32(0), stream.windowUpdate) - - assert.True(t, stream.consumeReceiveWindow(2)) - dataSent += 2 - assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow) - assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax) - assert.Equal(t, testWindowSize, stream.sendWindow) - assert.Equal(t, testWindowSize, stream.windowUpdate) - tempWindowUpdate = stream.windowUpdate - - tempStreamChunk = stream.getChunk() - assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate) - assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow) - assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax) - assert.Equal(t, testWindowSize, stream.sendWindow) - assert.Equal(t, uint32(0), stream.windowUpdate) - - assert.True(t, stream.consumeReceiveWindow(testWindowSize+10)) - dataSent += testWindowSize + 10 - assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow) - assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax) - assert.Equal(t, testWindowSize, stream.sendWindow) - assert.Equal(t, testWindowSize<<1, stream.windowUpdate) - tempWindowUpdate = stream.windowUpdate - - tempStreamChunk = stream.getChunk() - assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate) - assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow) - assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax) - assert.Equal(t, testWindowSize, stream.sendWindow) - assert.Equal(t, uint32(0), stream.windowUpdate) - - assert.False(t, stream.consumeReceiveWindow(testMaxWindowSize+1)) - assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow) - assert.Equal(t, testMaxWindowSize, stream.receiveWindowCurrentMax) -} - -func TestMuxedStreamEOF(t *testing.T) { - for i := 0; i < 4096; i++ { - readyList := NewReadyList() - stream := &MuxedStream{ - streamID: 1, - readBuffer: NewSharedBuffer(), - receiveWindow: 65536, - receiveWindowMax: 65536, - sendWindow: 65536, - readyList: readyList, - } - - go func() { stream.Close() }() - n, err := stream.Read([]byte{0}) - assert.Equal(t, io.EOF, err) - assert.Equal(t, 0, n) - // Write comes after read, because write buffers data before it is flushed. It wouldn't know about EOF - // until some time later. Calling read first forces it to know about EOF now. - n, err = stream.Write([]byte{1}) - assert.Equal(t, io.EOF, err) - assert.Equal(t, 0, n) - } -} - -func TestIsRPCStream(t *testing.T) { - tests := []struct { - stream *MuxedStream - isRPCStream bool - }{ - { - stream: &MuxedStream{}, - isRPCStream: false, - }, - { - stream: &MuxedStream{Headers: RPCHeaders()}, - isRPCStream: true, - }, - { - stream: &MuxedStream{Headers: []Header{ - {Name: ":method", Value: "rpc"}, - {Name: ":scheme", Value: "Capnp"}, - {Name: ":path", Value: "/"}, - }}, - isRPCStream: false, - }, - } - for _, test := range tests { - assert.Equal(t, test.isRPCStream, test.stream.IsRPCStream()) - } -} diff --git a/h2mux/muxmetrics.go b/h2mux/muxmetrics.go deleted file mode 100644 index 3423bde0..00000000 --- a/h2mux/muxmetrics.go +++ /dev/null @@ -1,296 +0,0 @@ -package h2mux - -import ( - "sync" - "time" - - "github.com/golang-collections/collections/queue" - "github.com/rs/zerolog" -) - -// data points used to compute average receive window and send window size -const ( - // data points used to compute average receive window and send window size - dataPoints = 100 - // updateFreq is set to 1 sec so we can get inbound & outbound byes/sec - updateFreq = time.Second -) - -type muxMetricsUpdater interface { - // metrics returns the latest metrics - metrics() *MuxerMetrics - // run is a blocking call to start the event loop - run(log *zerolog.Logger) error - // updateRTTChan is called by muxReader to report new RTT measurements - updateRTT(rtt *roundTripMeasurement) - //updateReceiveWindowChan is called by muxReader and muxWriter when receiveWindow size is updated - updateReceiveWindow(receiveWindow uint32) - //updateSendWindowChan is called by muxReader and muxWriter when sendWindow size is updated - updateSendWindow(sendWindow uint32) - // updateInBoundBytesChan is called periodicallyby muxReader to report bytesRead - updateInBoundBytes(inBoundBytes uint64) - // updateOutBoundBytesChan is called periodically by muxWriter to report bytesWrote - updateOutBoundBytes(outBoundBytes uint64) -} - -type muxMetricsUpdaterImpl struct { - // rttData keeps record of rtt, rttMin, rttMax and last measured time - rttData *rttData - // receiveWindowData keeps record of receive window measurement - receiveWindowData *flowControlData - // sendWindowData keeps record of send window measurement - sendWindowData *flowControlData - // inBoundRate is incoming bytes/sec - inBoundRate *rate - // outBoundRate is outgoing bytes/sec - outBoundRate *rate - // updateRTTChan is the channel to receive new RTT measurement - updateRTTChan chan *roundTripMeasurement - //updateReceiveWindowChan is the channel to receive updated receiveWindow size - updateReceiveWindowChan chan uint32 - //updateSendWindowChan is the channel to receive updated sendWindow size - updateSendWindowChan chan uint32 - // updateInBoundBytesChan us the channel to receive bytesRead - updateInBoundBytesChan chan uint64 - // updateOutBoundBytesChan us the channel to receive bytesWrote - updateOutBoundBytesChan chan uint64 - // shutdownC is to signal the muxerMetricsUpdater to shutdown - abortChan <-chan struct{} - - compBytesBefore, compBytesAfter *AtomicCounter -} - -type MuxerMetrics struct { - RTT, RTTMin, RTTMax time.Duration - ReceiveWindowAve, SendWindowAve float64 - ReceiveWindowMin, ReceiveWindowMax, SendWindowMin, SendWindowMax uint32 - InBoundRateCurr, InBoundRateMin, InBoundRateMax uint64 - OutBoundRateCurr, OutBoundRateMin, OutBoundRateMax uint64 - CompBytesBefore, CompBytesAfter *AtomicCounter -} - -func (m *MuxerMetrics) CompRateAve() float64 { - if m.CompBytesBefore.Value() == 0 { - return 1. - } - return float64(m.CompBytesAfter.Value()) / float64(m.CompBytesBefore.Value()) -} - -type roundTripMeasurement struct { - receiveTime, sendTime time.Time -} - -type rttData struct { - rtt, rttMin, rttMax time.Duration - lastMeasurementTime time.Time - lock sync.RWMutex -} - -type flowControlData struct { - sum uint64 - min, max uint32 - queue *queue.Queue - lock sync.RWMutex -} - -type rate struct { - curr uint64 - min, max uint64 - lock sync.RWMutex -} - -func newMuxMetricsUpdater( - abortChan <-chan struct{}, - compBytesBefore, compBytesAfter *AtomicCounter, -) muxMetricsUpdater { - updateRTTChan := make(chan *roundTripMeasurement, 1) - updateReceiveWindowChan := make(chan uint32, 1) - updateSendWindowChan := make(chan uint32, 1) - updateInBoundBytesChan := make(chan uint64) - updateOutBoundBytesChan := make(chan uint64) - - return &muxMetricsUpdaterImpl{ - rttData: newRTTData(), - receiveWindowData: newFlowControlData(), - sendWindowData: newFlowControlData(), - inBoundRate: newRate(), - outBoundRate: newRate(), - updateRTTChan: updateRTTChan, - updateReceiveWindowChan: updateReceiveWindowChan, - updateSendWindowChan: updateSendWindowChan, - updateInBoundBytesChan: updateInBoundBytesChan, - updateOutBoundBytesChan: updateOutBoundBytesChan, - abortChan: abortChan, - compBytesBefore: compBytesBefore, - compBytesAfter: compBytesAfter, - } -} - -func (updater *muxMetricsUpdaterImpl) metrics() *MuxerMetrics { - m := &MuxerMetrics{} - m.RTT, m.RTTMin, m.RTTMax = updater.rttData.metrics() - m.ReceiveWindowAve, m.ReceiveWindowMin, m.ReceiveWindowMax = updater.receiveWindowData.metrics() - m.SendWindowAve, m.SendWindowMin, m.SendWindowMax = updater.sendWindowData.metrics() - m.InBoundRateCurr, m.InBoundRateMin, m.InBoundRateMax = updater.inBoundRate.get() - m.OutBoundRateCurr, m.OutBoundRateMin, m.OutBoundRateMax = updater.outBoundRate.get() - m.CompBytesBefore, m.CompBytesAfter = updater.compBytesBefore, updater.compBytesAfter - return m -} - -func (updater *muxMetricsUpdaterImpl) run(log *zerolog.Logger) error { - defer log.Debug().Msg("mux - metrics: event loop finished") - for { - select { - case <-updater.abortChan: - log.Debug().Msgf("mux - metrics: Stopping mux metrics updater") - return nil - case roundTripMeasurement := <-updater.updateRTTChan: - go updater.rttData.update(roundTripMeasurement) - log.Debug().Msg("mux - metrics: Update rtt") - case receiveWindow := <-updater.updateReceiveWindowChan: - go updater.receiveWindowData.update(receiveWindow) - log.Debug().Msg("mux - metrics: Update receive window") - case sendWindow := <-updater.updateSendWindowChan: - go updater.sendWindowData.update(sendWindow) - log.Debug().Msg("mux - metrics: Update send window") - case inBoundBytes := <-updater.updateInBoundBytesChan: - // inBoundBytes is bytes/sec because the update interval is 1 sec - go updater.inBoundRate.update(inBoundBytes) - log.Debug().Msgf("mux - metrics: Inbound bytes %d", inBoundBytes) - case outBoundBytes := <-updater.updateOutBoundBytesChan: - // outBoundBytes is bytes/sec because the update interval is 1 sec - go updater.outBoundRate.update(outBoundBytes) - log.Debug().Msgf("mux - metrics: Outbound bytes %d", outBoundBytes) - } - } -} - -func (updater *muxMetricsUpdaterImpl) updateRTT(rtt *roundTripMeasurement) { - select { - case updater.updateRTTChan <- rtt: - case <-updater.abortChan: - } - -} - -func (updater *muxMetricsUpdaterImpl) updateReceiveWindow(receiveWindow uint32) { - select { - case updater.updateReceiveWindowChan <- receiveWindow: - case <-updater.abortChan: - } -} - -func (updater *muxMetricsUpdaterImpl) updateSendWindow(sendWindow uint32) { - select { - case updater.updateSendWindowChan <- sendWindow: - case <-updater.abortChan: - } -} - -func (updater *muxMetricsUpdaterImpl) updateInBoundBytes(inBoundBytes uint64) { - select { - case updater.updateInBoundBytesChan <- inBoundBytes: - case <-updater.abortChan: - } - -} - -func (updater *muxMetricsUpdaterImpl) updateOutBoundBytes(outBoundBytes uint64) { - select { - case updater.updateOutBoundBytesChan <- outBoundBytes: - case <-updater.abortChan: - } -} - -func newRTTData() *rttData { - return &rttData{} -} - -func (r *rttData) update(measurement *roundTripMeasurement) { - r.lock.Lock() - defer r.lock.Unlock() - // discard pings before lastMeasurementTime - if r.lastMeasurementTime.After(measurement.sendTime) { - return - } - r.lastMeasurementTime = measurement.sendTime - r.rtt = measurement.receiveTime.Sub(measurement.sendTime) - if r.rttMax < r.rtt { - r.rttMax = r.rtt - } - if r.rttMin == 0 || r.rttMin > r.rtt { - r.rttMin = r.rtt - } -} - -func (r *rttData) metrics() (rtt, rttMin, rttMax time.Duration) { - r.lock.RLock() - defer r.lock.RUnlock() - return r.rtt, r.rttMin, r.rttMax -} - -func newFlowControlData() *flowControlData { - return &flowControlData{queue: queue.New()} -} - -func (f *flowControlData) update(measurement uint32) { - f.lock.Lock() - defer f.lock.Unlock() - var firstItem uint32 - // store new data into queue, remove oldest data if queue is full - f.queue.Enqueue(measurement) - if f.queue.Len() > dataPoints { - // data type should always be uint32 - firstItem = f.queue.Dequeue().(uint32) - } - // if (measurement - firstItem) < 0, uint64(measurement - firstItem) - // will overflow and become a large positive number - f.sum += uint64(measurement) - f.sum -= uint64(firstItem) - if measurement > f.max { - f.max = measurement - } - if f.min == 0 || measurement < f.min { - f.min = measurement - } -} - -// caller of ave() should acquire lock first -func (f *flowControlData) ave() float64 { - if f.queue.Len() == 0 { - return 0 - } - return float64(f.sum) / float64(f.queue.Len()) -} - -func (f *flowControlData) metrics() (ave float64, min, max uint32) { - f.lock.RLock() - defer f.lock.RUnlock() - return f.ave(), f.min, f.max -} - -func newRate() *rate { - return &rate{} -} - -func (r *rate) update(measurement uint64) { - r.lock.Lock() - defer r.lock.Unlock() - r.curr = measurement - // if measurement is 0, then there is no incoming/outgoing connection, don't update min/max - if r.curr == 0 { - return - } - if measurement > r.max { - r.max = measurement - } - if r.min == 0 || measurement < r.min { - r.min = measurement - } -} - -func (r *rate) get() (curr, min, max uint64) { - r.lock.RLock() - defer r.lock.RUnlock() - return r.curr, r.min, r.max -} diff --git a/h2mux/muxmetrics_test.go b/h2mux/muxmetrics_test.go deleted file mode 100644 index a9213a2c..00000000 --- a/h2mux/muxmetrics_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package h2mux - -import ( - "sync" - "testing" - "time" - - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" -) - -func ave(sum uint64, len int) float64 { - return float64(sum) / float64(len) -} - -func TestRTTUpdate(t *testing.T) { - r := newRTTData() - start := time.Now() - // send at 0 ms, receive at 2 ms, RTT = 2ms - m := &roundTripMeasurement{receiveTime: start.Add(2 * time.Millisecond), sendTime: start} - r.update(m) - assert.Equal(t, start, r.lastMeasurementTime) - assert.Equal(t, 2*time.Millisecond, r.rtt) - assert.Equal(t, 2*time.Millisecond, r.rttMin) - assert.Equal(t, 2*time.Millisecond, r.rttMax) - - // send at 3 ms, receive at 6 ms, RTT = 3ms - m = &roundTripMeasurement{receiveTime: start.Add(6 * time.Millisecond), sendTime: start.Add(3 * time.Millisecond)} - r.update(m) - assert.Equal(t, start.Add(3*time.Millisecond), r.lastMeasurementTime) - assert.Equal(t, 3*time.Millisecond, r.rtt) - assert.Equal(t, 2*time.Millisecond, r.rttMin) - assert.Equal(t, 3*time.Millisecond, r.rttMax) - - // send at 7 ms, receive at 8 ms, RTT = 1ms - m = &roundTripMeasurement{receiveTime: start.Add(8 * time.Millisecond), sendTime: start.Add(7 * time.Millisecond)} - r.update(m) - assert.Equal(t, start.Add(7*time.Millisecond), r.lastMeasurementTime) - assert.Equal(t, 1*time.Millisecond, r.rtt) - assert.Equal(t, 1*time.Millisecond, r.rttMin) - assert.Equal(t, 3*time.Millisecond, r.rttMax) - - // send at -4 ms, receive at 0 ms, RTT = 4ms, but this ping is before last measurement - // so it will be discarded - m = &roundTripMeasurement{receiveTime: start, sendTime: start.Add(-2 * time.Millisecond)} - r.update(m) - assert.Equal(t, start.Add(7*time.Millisecond), r.lastMeasurementTime) - assert.Equal(t, 1*time.Millisecond, r.rtt) - assert.Equal(t, 1*time.Millisecond, r.rttMin) - assert.Equal(t, 3*time.Millisecond, r.rttMax) -} - -func TestFlowControlDataUpdate(t *testing.T) { - f := newFlowControlData() - assert.Equal(t, 0, f.queue.Len()) - assert.Equal(t, float64(0), f.ave()) - - var sum uint64 - min := maxWindowSize - dataPoints - max := maxWindowSize - for i := 1; i <= dataPoints; i++ { - size := maxWindowSize - uint32(i) - f.update(size) - assert.Equal(t, max-uint32(1), f.max) - assert.Equal(t, size, f.min) - - assert.Equal(t, i, f.queue.Len()) - - sum += uint64(size) - assert.Equal(t, sum, f.sum) - assert.Equal(t, ave(sum, f.queue.Len()), f.ave()) - } - - // queue is full, should start to dequeue first element - for i := 1; i <= dataPoints; i++ { - f.update(max) - assert.Equal(t, max, f.max) - assert.Equal(t, min, f.min) - - assert.Equal(t, dataPoints, f.queue.Len()) - - sum += uint64(i) - assert.Equal(t, sum, f.sum) - assert.Equal(t, ave(sum, dataPoints), f.ave()) - } -} - -func TestMuxMetricsUpdater(t *testing.T) { - t.Skip("Inherently racy test due to muxMetricsUpdaterImpl.run()") - errChan := make(chan error) - abortChan := make(chan struct{}) - compBefore, compAfter := NewAtomicCounter(0), NewAtomicCounter(0) - m := newMuxMetricsUpdater(abortChan, compBefore, compAfter) - log := zerolog.Nop() - - go func() { - errChan <- m.run(&log) - }() - - var wg sync.WaitGroup - wg.Add(2) - - // mock muxReader - readerStart := time.Now() - rm := &roundTripMeasurement{receiveTime: readerStart, sendTime: readerStart} - m.updateRTT(rm) - go func() { - defer wg.Done() - assert.Equal(t, 0, dataPoints%4, - "dataPoints is not divisible by 4; this test should be adjusted accordingly") - readerSend := readerStart.Add(time.Millisecond) - for i := 1; i <= dataPoints/4; i++ { - readerReceive := readerSend.Add(time.Duration(i) * time.Millisecond) - rm := &roundTripMeasurement{receiveTime: readerReceive, sendTime: readerSend} - m.updateRTT(rm) - readerSend = readerReceive.Add(time.Millisecond) - m.updateReceiveWindow(uint32(i)) - m.updateSendWindow(uint32(i)) - - m.updateInBoundBytes(uint64(i)) - } - }() - - // mock muxWriter - go func() { - defer wg.Done() - assert.Equal(t, 0, dataPoints%4, - "dataPoints is not divisible by 4; this test should be adjusted accordingly") - for j := dataPoints/4 + 1; j <= dataPoints/2; j++ { - m.updateReceiveWindow(uint32(j)) - m.updateSendWindow(uint32(j)) - - // should always be discarded since the send time is before readerSend - rm := &roundTripMeasurement{receiveTime: readerStart, sendTime: readerStart.Add(-time.Duration(j*dataPoints) * time.Millisecond)} - m.updateRTT(rm) - - m.updateOutBoundBytes(uint64(j)) - } - - }() - wg.Wait() - - metrics := m.metrics() - points := dataPoints / 2 - assert.Equal(t, time.Millisecond, metrics.RTTMin) - assert.Equal(t, time.Duration(dataPoints/4)*time.Millisecond, metrics.RTTMax) - - // sum(1..i) = i*(i+1)/2, ave(1..i) = i*(i+1)/2/i = (i+1)/2 - assert.Equal(t, float64(points+1)/float64(2), metrics.ReceiveWindowAve) - assert.Equal(t, uint32(1), metrics.ReceiveWindowMin) - assert.Equal(t, uint32(points), metrics.ReceiveWindowMax) - - assert.Equal(t, float64(points+1)/float64(2), metrics.SendWindowAve) - assert.Equal(t, uint32(1), metrics.SendWindowMin) - assert.Equal(t, uint32(points), metrics.SendWindowMax) - - assert.Equal(t, uint64(dataPoints/4), metrics.InBoundRateCurr) - assert.Equal(t, uint64(1), metrics.InBoundRateMin) - assert.Equal(t, uint64(dataPoints/4), metrics.InBoundRateMax) - - assert.Equal(t, uint64(dataPoints/2), metrics.OutBoundRateCurr) - assert.Equal(t, uint64(dataPoints/4+1), metrics.OutBoundRateMin) - assert.Equal(t, uint64(dataPoints/2), metrics.OutBoundRateMax) - - close(abortChan) - assert.Nil(t, <-errChan) - close(errChan) - -} diff --git a/h2mux/muxreader.go b/h2mux/muxreader.go deleted file mode 100644 index cf8d98f1..00000000 --- a/h2mux/muxreader.go +++ /dev/null @@ -1,508 +0,0 @@ -package h2mux - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "net/url" - "time" - - "github.com/rs/zerolog" - "golang.org/x/net/http2" -) - -type MuxReader struct { - // f is used to read HTTP2 frames. - f *http2.Framer - // handler provides a callback to receive new streams. if nil, new streams cannot be accepted. - handler MuxedStreamHandler - // streams tracks currently-open streams. - streams *activeStreamMap - // readyList is used to signal writable streams. - readyList *ReadyList - // streamErrors lets us report stream errors to the MuxWriter. - streamErrors *StreamErrorMap - // goAwayChan is used to tell the writer to send a GOAWAY message. - goAwayChan chan<- http2.ErrCode - // abortChan is used when shutting down ungracefully. When this becomes readable, all activity should stop. - abortChan <-chan struct{} - // pingTimestamp is an atomic value containing the latest received ping timestamp. - pingTimestamp *PingTimestamp - // connActive is used to signal to the writer that something happened on the connection. - // This is used to clear idle timeout disconnection deadlines. - connActive Signal - // The initial value for the send and receive window of a new stream. - initialStreamWindow uint32 - // The max value for the send window of a stream. - streamWindowMax uint32 - // The max size for the write buffer of a stream - streamWriteBufferMaxLen int - // r is a reference to the underlying connection used when shutting down. - r io.Closer - // metricsUpdater is used to report metrics - metricsUpdater muxMetricsUpdater - // bytesRead is the amount of bytes read from data frames since the last time we called metricsUpdater.updateInBoundBytes() - bytesRead *AtomicCounter - // dictionaries holds the h2 cross-stream compression dictionaries - dictionaries h2Dictionaries -} - -// Shutdown blocks new streams from being created. -// It returns a channel that is closed once the last stream has closed. -func (r *MuxReader) Shutdown() <-chan struct{} { - done, alreadyInProgress := r.streams.Shutdown() - if alreadyInProgress { - return done - } - r.sendGoAway(http2.ErrCodeNo) - go func() { - // close reader side when last stream ends; this will cause the writer to abort - <-done - r.r.Close() - }() - return done -} - -func (r *MuxReader) run(log *zerolog.Logger) error { - defer log.Debug().Msg("mux - read: event loop finished") - - // routine to periodically update bytesRead - go func() { - ticker := time.NewTicker(updateFreq) - defer ticker.Stop() - for { - select { - case <-r.abortChan: - return - case <-ticker.C: - r.metricsUpdater.updateInBoundBytes(r.bytesRead.Count()) - } - } - }() - - for { - frame, err := r.f.ReadFrame() - if err != nil { - errorString := fmt.Sprintf("mux - read: %s", err) - if errorDetail := r.f.ErrorDetail(); errorDetail != nil { - errorString = fmt.Sprintf("%s: errorDetail: %s", errorString, errorDetail) - } - switch e := err.(type) { - case http2.StreamError: - log.Info().Msgf("%s: stream error", errorString) - // Ideally we wouldn't return here, since that aborts the muxer. - // We should communicate the error to the relevant MuxedStream - // data structure, so that callers of MuxedStream.Read() and - // MuxedStream.Write() would see it. Then we could `continue` - // and keep the muxer going. - return r.streamError(e.StreamID, e.Code) - case http2.ConnectionError: - log.Info().Msgf("%s: stream error", errorString) - return r.connectionError(err) - default: - if isConnectionClosedError(err) { - if r.streams.Len() == 0 { - // don't log the error here -- that would just be extra noise - log.Debug().Msg("mux - read: shutting down") - return nil - } - log.Info().Msgf("%s: connection closed unexpectedly", errorString) - return err - } else { - log.Info().Msgf("%s: frame read error", errorString) - return r.connectionError(err) - } - } - } - r.connActive.Signal() - log.Debug().Msgf("mux - read: read frame: data %v", frame) - switch f := frame.(type) { - case *http2.DataFrame: - err = r.receiveFrameData(f, log) - case *http2.MetaHeadersFrame: - err = r.receiveHeaderData(f) - case *http2.RSTStreamFrame: - streamID := f.Header().StreamID - if streamID == 0 { - return ErrInvalidStream - } - if stream, ok := r.streams.Get(streamID); ok { - stream.Close() - } - r.streams.Delete(streamID) - case *http2.PingFrame: - r.receivePingData(f) - case *http2.GoAwayFrame: - err = r.receiveGoAway(f) - // The receiver of a flow-controlled frame sends a WINDOW_UPDATE frame as it - // consumes data and frees up space in flow-control windows - case *http2.WindowUpdateFrame: - err = r.updateStreamWindow(f) - case *http2.UnknownFrame: - switch f.Header().Type { - case FrameUseDictionary: - err = r.receiveUseDictionary(f) - case FrameSetDictionary: - err = r.receiveSetDictionary(f) - default: - err = ErrUnexpectedFrameType - } - default: - err = ErrUnexpectedFrameType - } - if err != nil { - log.Debug().Msgf("mux - read: read error: data %v", frame) - return r.connectionError(err) - } - } -} - -func (r *MuxReader) newMuxedStream(streamID uint32) *MuxedStream { - return &MuxedStream{ - streamID: streamID, - readBuffer: NewSharedBuffer(), - writeBuffer: &bytes.Buffer{}, - writeBufferMaxLen: r.streamWriteBufferMaxLen, - writeBufferHasSpace: make(chan struct{}, 1), - receiveWindow: r.initialStreamWindow, - receiveWindowCurrentMax: r.initialStreamWindow, - receiveWindowMax: r.streamWindowMax, - sendWindow: r.initialStreamWindow, - readyList: r.readyList, - dictionaries: r.dictionaries, - } -} - -// getStreamForFrame returns a stream if valid, or an error describing why the stream could not be returned. -func (r *MuxReader) getStreamForFrame(frame http2.Frame) (*MuxedStream, error) { - sid := frame.Header().StreamID - if sid == 0 { - return nil, ErrUnexpectedFrameType - } - if stream, ok := r.streams.Get(sid); ok { - return stream, nil - } - if r.streams.IsLocalStreamID(sid) { - // no stream available, but no error - return nil, ErrClosedStream - } - if sid < r.streams.LastPeerStreamID() { - // no stream available, stream closed error - return nil, ErrClosedStream - } - return nil, ErrUnknownStream -} - -func (r *MuxReader) defaultStreamErrorHandler(err error, header http2.FrameHeader) error { - if header.Flags.Has(http2.FlagHeadersEndStream) { - return nil - } else if err == ErrUnknownStream || err == ErrClosedStream { - return r.streamError(header.StreamID, http2.ErrCodeStreamClosed) - } else { - return err - } -} - -// Receives header frames from a stream. A non-nil error is a connection error. -func (r *MuxReader) receiveHeaderData(frame *http2.MetaHeadersFrame) error { - var stream *MuxedStream - sid := frame.Header().StreamID - if sid == 0 { - return ErrUnexpectedFrameType - } - newStream := r.streams.IsPeerStreamID(sid) - if newStream { - // header request - // TODO support trailers (if stream exists) - ok, err := r.streams.AcquirePeerID(sid) - if !ok { - // ignore new streams while shutting down - return r.streamError(sid, err) - } - stream = r.newMuxedStream(sid) - // Set stream. Returns false if a stream already existed with that ID or we are shutting down, return false. - if !r.streams.Set(stream) { - // got HEADERS frame for an existing stream - // TODO support trailers - return r.streamError(sid, http2.ErrCodeInternal) - } - } else { - // header response - var err error - if stream, err = r.getStreamForFrame(frame); err != nil { - return r.defaultStreamErrorHandler(err, frame.Header()) - } - } - headers := make([]Header, 0, len(frame.Fields)) - for _, header := range frame.Fields { - switch header.Name { - case ":method": - stream.method = header.Value - case ":path": - u, err := url.Parse(header.Value) - if err == nil { - stream.path = u.Path - } - case "accept-encoding": - // remove accept-encoding if dictionaries are enabled - if r.dictionaries.write != nil { - continue - } - } - headers = append(headers, Header{Name: header.Name, Value: header.Value}) - } - stream.Headers = headers - if frame.Header().Flags.Has(http2.FlagHeadersEndStream) { - stream.receiveEOF() - return nil - } - if newStream { - go r.handleStream(stream) - } else { - close(stream.responseHeadersReceived) - } - return nil -} - -func (r *MuxReader) handleStream(stream *MuxedStream) { - defer stream.Close() - r.handler.ServeStream(stream) -} - -// Receives a data frame from a stream. A non-nil error is a connection error. -func (r *MuxReader) receiveFrameData(frame *http2.DataFrame, log *zerolog.Logger) error { - stream, err := r.getStreamForFrame(frame) - if err != nil { - return r.defaultStreamErrorHandler(err, frame.Header()) - } - data := frame.Data() - if len(data) > 0 { - n, err := stream.readBuffer.Write(data) - if err != nil { - return r.streamError(stream.streamID, http2.ErrCodeInternal) - } - r.bytesRead.IncrementBy(uint64(n)) - } - if frame.Header().Flags.Has(http2.FlagDataEndStream) { - if stream.receiveEOF() { - r.streams.Delete(stream.streamID) - log.Debug().Msgf("mux - read: stream closed: streamID: %d", frame.Header().StreamID) - } else { - log.Debug().Msgf("mux - read: shutdown receive side: streamID: %d", frame.Header().StreamID) - } - return nil - } - if !stream.consumeReceiveWindow(uint32(len(data))) { - return r.streamError(stream.streamID, http2.ErrCodeFlowControl) - } - r.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow()) - return nil -} - -// Receive a PING from the peer. Update RTT and send/receive window metrics if it's an ACK. -func (r *MuxReader) receivePingData(frame *http2.PingFrame) { - ts := int64(binary.LittleEndian.Uint64(frame.Data[:])) - if !frame.IsAck() { - r.pingTimestamp.Set(ts) - return - } - - // Update the computed RTT aggregations with a new measurement. - // `ts` is the time that the probe was sent. - // We assume that `time.Now()` is the time we received that probe. - r.metricsUpdater.updateRTT(&roundTripMeasurement{ - receiveTime: time.Now(), - sendTime: time.Unix(0, ts), - }) -} - -// Receive a GOAWAY from the peer. Gracefully shut down our connection. -func (r *MuxReader) receiveGoAway(frame *http2.GoAwayFrame) error { - r.Shutdown() - // Close all streams above the last processed stream - lastStream := r.streams.LastLocalStreamID() - for i := frame.LastStreamID + 2; i <= lastStream; i++ { - if stream, ok := r.streams.Get(i); ok { - stream.Close() - } - } - return nil -} - -// Receive a USE_DICTIONARY from the peer. Setup dictionary for stream. -func (r *MuxReader) receiveUseDictionary(frame *http2.UnknownFrame) error { - payload := frame.Payload() - streamID := frame.StreamID - - // Check frame is formatted properly - if len(payload) != 1 { - return r.streamError(streamID, http2.ErrCodeProtocol) - } - - stream, err := r.getStreamForFrame(frame) - if err != nil { - return err - } - - if stream.receivedUseDict == true || stream.dictionaries.read == nil { - return r.streamError(streamID, http2.ErrCodeInternal) - } - - stream.receivedUseDict = true - dictID := payload[0] - - dictReader := stream.dictionaries.read.newReader(stream.readBuffer.(*SharedBuffer), dictID) - if dictReader == nil { - return r.streamError(streamID, http2.ErrCodeInternal) - } - - stream.readBufferLock.Lock() - stream.readBuffer = dictReader - stream.readBufferLock.Unlock() - - return nil -} - -// Receive a SET_DICTIONARY from the peer. Update dictionaries accordingly. -func (r *MuxReader) receiveSetDictionary(frame *http2.UnknownFrame) (err error) { - - payload := frame.Payload() - flags := frame.Flags - - stream, err := r.getStreamForFrame(frame) - if err != nil && err != ErrClosedStream { - return err - } - reader, ok := stream.readBuffer.(*h2DictionaryReader) - if !ok { - return r.streamError(frame.StreamID, http2.ErrCodeProtocol) - } - - // A SetDictionary frame consists of several - // Dictionary-Entries that specify how existing dictionaries - // are to be updated using the current stream data - // +---------------+---------------+ - // | Dictionary-Entry (+) ... - // +---------------+---------------+ - - for { - // Each Dictionary-Entry is formatted as follows: - // +-------------------------------+ - // | Dictionary-ID (8) | - // +---+---------------------------+ - // | P | Size (7+) | - // +---+---------------------------+ - // | E?| D?| Truncate? (6+) | - // +---+---------------------------+ - // | Offset? (8+) | - // +-------------------------------+ - - var size, truncate, offset uint64 - var p, e, d bool - - // Parse a single Dictionary-Entry - if len(payload) < 2 { // Must have at least id and size - return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol} - } - - dictID := uint8(payload[0]) - p = (uint8(payload[1]) >> 7) == 1 - payload, size, err = http2ReadVarInt(7, payload[1:]) - if err != nil { - return - } - - if flags.Has(FlagSetDictionaryAppend) { - // Presence of FlagSetDictionaryAppend means we expect e, d and truncate - if len(payload) < 1 { - return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol} - } - e = (uint8(payload[0]) >> 7) == 1 - d = (uint8((payload[0])>>6) & 1) == 1 - payload, truncate, err = http2ReadVarInt(6, payload) - if err != nil { - return - } - } - - if flags.Has(FlagSetDictionaryOffset) { - // Presence of FlagSetDictionaryOffset means we expect offset - if len(payload) < 1 { - return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol} - } - payload, offset, err = http2ReadVarInt(8, payload) - if err != nil { - return - } - } - - setdict := setDictRequest{streamID: stream.streamID, - dictID: dictID, - dictSZ: size, - truncate: truncate, - offset: offset, - P: p, - E: e, - D: d} - - // Find the right dictionary - dict, err := r.dictionaries.read.getDictByID(dictID) - if err != nil { - return err - } - - // Register a dictionary update order for the dictionary and reader - updateEntry := &dictUpdate{reader: reader, dictionary: dict, s: setdict} - dict.queue = append(dict.queue, updateEntry) - reader.queue = append(reader.queue, updateEntry) - // End of frame - if len(payload) == 0 { - break - } - } - return nil -} - -// Receives header frames from a stream. A non-nil error is a connection error. -func (r *MuxReader) updateStreamWindow(frame *http2.WindowUpdateFrame) error { - stream, err := r.getStreamForFrame(frame) - if err != nil && err != ErrUnknownStream && err != ErrClosedStream { - return err - } - if stream == nil { - // ignore window updates on closed streams - return nil - } - stream.replenishSendWindow(frame.Increment) - r.metricsUpdater.updateSendWindow(stream.getSendWindow()) - return nil -} - -// Raise a stream processing error, closing the stream. Runs on the write thread. -func (r *MuxReader) streamError(streamID uint32, e http2.ErrCode) error { - r.streamErrors.RaiseError(streamID, e) - return nil -} - -func (r *MuxReader) connectionError(err error) error { - http2Code := http2.ErrCodeInternal - switch e := err.(type) { - case http2.ConnectionError: - http2Code = http2.ErrCode(e) - case MuxerProtocolError: - http2Code = e.h2code - } - r.sendGoAway(http2Code) - return err -} - -// Instruct the writer to send a GOAWAY message if possible. This may fail in -// the case where an existing GOAWAY message is in flight or the writer event -// loop already ended. -func (r *MuxReader) sendGoAway(errCode http2.ErrCode) { - select { - case r.goAwayChan <- errCode: - default: - } -} diff --git a/h2mux/muxreader_test.go b/h2mux/muxreader_test.go deleted file mode 100644 index 10ae7ff8..00000000 --- a/h2mux/muxreader_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package h2mux - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -var ( - methodHeader = Header{ - Name: ":method", - Value: "GET", - } - schemeHeader = Header{ - Name: ":scheme", - Value: "https", - } - pathHeader = Header{ - Name: ":path", - Value: "/api/tunnels", - } - respStatusHeader = Header{ - Name: ":status", - Value: "200", - } -) - -type mockOriginStreamHandler struct { - stream *MuxedStream -} - -func (mosh *mockOriginStreamHandler) ServeStream(stream *MuxedStream) error { - mosh.stream = stream - // Echo tunnel hostname in header - stream.WriteHeaders([]Header{respStatusHeader}) - return nil -} - -func assertOpenStreamSucceed(t *testing.T, stream *MuxedStream, err error) { - assert.NoError(t, err) - assert.Len(t, stream.Headers, 1) - assert.Equal(t, respStatusHeader, stream.Headers[0]) -} - -func TestMissingHeaders(t *testing.T) { - originHandler := &mockOriginStreamHandler{} - muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream) - muxPair.Serve(t) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - reqHeaders := []Header{ - { - Name: "content-type", - Value: "application/json", - }, - } - - stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil) - assertOpenStreamSucceed(t, stream, err) - - assert.Empty(t, originHandler.stream.method) - assert.Empty(t, originHandler.stream.path) -} - -func TestReceiveHeaderData(t *testing.T) { - originHandler := &mockOriginStreamHandler{} - muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream) - muxPair.Serve(t) - - reqHeaders := []Header{ - methodHeader, - schemeHeader, - pathHeader, - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil) - assertOpenStreamSucceed(t, stream, err) - - assert.Equal(t, methodHeader.Value, originHandler.stream.method) - assert.Equal(t, pathHeader.Value, originHandler.stream.path) -} diff --git a/h2mux/muxwriter.go b/h2mux/muxwriter.go deleted file mode 100644 index c4d8fded..00000000 --- a/h2mux/muxwriter.go +++ /dev/null @@ -1,311 +0,0 @@ -package h2mux - -import ( - "bytes" - "encoding/binary" - "io" - "time" - - "github.com/rs/zerolog" - - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" -) - -type MuxWriter struct { - // f is used to write HTTP2 frames. - f *http2.Framer - // streams tracks currently-open streams. - streams *activeStreamMap - // streamErrors receives stream errors raised by the MuxReader. - streamErrors *StreamErrorMap - // readyStreamChan is used to multiplex writable streams onto the single connection. - // When a stream becomes writable its ID is sent on this channel. - readyStreamChan <-chan uint32 - // newStreamChan is used to create new streams with a given set of headers. - newStreamChan <-chan MuxedStreamRequest - // goAwayChan is used to send a single GOAWAY message to the peer. The element received - // is the HTTP/2 error code to send. - goAwayChan <-chan http2.ErrCode - // abortChan is used when shutting down ungracefully. When this becomes readable, all activity should stop. - abortChan <-chan struct{} - // pingTimestamp is an atomic value containing the latest received ping timestamp. - pingTimestamp *PingTimestamp - // A timer used to measure idle connection time. Reset after sending data. - idleTimer *IdleTimer - // connActiveChan receives a signal that the connection received some (read) activity. - connActiveChan <-chan struct{} - // Maximum size of all frames that can be sent on this connection. - maxFrameSize uint32 - // headerEncoder is the stateful header encoder for this connection - headerEncoder *hpack.Encoder - // headerBuffer is the temporary buffer used by headerEncoder. - headerBuffer bytes.Buffer - - // metricsUpdater is used to report metrics - metricsUpdater muxMetricsUpdater - // bytesWrote is the amount of bytes written to data frames since the last time we called metricsUpdater.updateOutBoundBytes() - bytesWrote *AtomicCounter - - useDictChan <-chan useDictRequest -} - -type MuxedStreamRequest struct { - stream *MuxedStream - body io.Reader -} - -func NewMuxedStreamRequest(stream *MuxedStream, body io.Reader) MuxedStreamRequest { - return MuxedStreamRequest{ - stream: stream, - body: body, - } -} - -func (r *MuxedStreamRequest) flushBody() { - io.Copy(r.stream, r.body) - r.stream.CloseWrite() -} - -func tsToPingData(ts int64) [8]byte { - pingData := [8]byte{} - binary.LittleEndian.PutUint64(pingData[:], uint64(ts)) - return pingData -} - -func (w *MuxWriter) run(log *zerolog.Logger) error { - defer log.Debug().Msg("mux - write: event loop finished") - - // routine to periodically communicate bytesWrote - go func() { - ticker := time.NewTicker(updateFreq) - defer ticker.Stop() - for { - select { - case <-w.abortChan: - return - case <-ticker.C: - w.metricsUpdater.updateOutBoundBytes(w.bytesWrote.Count()) - } - } - }() - - for { - select { - case <-w.abortChan: - log.Debug().Msg("mux - write: aborting writer thread") - return nil - case errCode := <-w.goAwayChan: - log.Debug().Msgf("mux - write: sending GOAWAY code %v", errCode) - err := w.f.WriteGoAway(w.streams.LastPeerStreamID(), errCode, []byte{}) - if err != nil { - return err - } - w.idleTimer.MarkActive() - case <-w.pingTimestamp.GetUpdateChan(): - log.Debug().Msg("mux - write: sending PING ACK") - err := w.f.WritePing(true, tsToPingData(w.pingTimestamp.Get())) - if err != nil { - return err - } - w.idleTimer.MarkActive() - case <-w.idleTimer.C: - if !w.idleTimer.Retry() { - return ErrConnectionDropped - } - log.Debug().Msg("mux - write: sending PING") - err := w.f.WritePing(false, tsToPingData(time.Now().UnixNano())) - if err != nil { - return err - } - w.idleTimer.ResetTimer() - case <-w.connActiveChan: - w.idleTimer.MarkActive() - case <-w.streamErrors.GetSignalChan(): - for streamID, errCode := range w.streamErrors.GetErrors() { - log.Debug().Msgf("mux - write: resetting stream with code: %v streamID: %d", errCode, streamID) - err := w.f.WriteRSTStream(streamID, errCode) - if err != nil { - return err - } - } - w.idleTimer.MarkActive() - case streamRequest := <-w.newStreamChan: - streamID := w.streams.AcquireLocalID() - streamRequest.stream.streamID = streamID - if !w.streams.Set(streamRequest.stream) { - // Race between OpenStream and Shutdown, and Shutdown won. Let Shutdown (and the eventual abort) take - // care of this stream. Ideally we'd pass the error directly to the stream object somehow so the - // caller can be unblocked sooner, but the value of that optimisation is minimal for most of the - // reasons why you'd call Shutdown anyway. - continue - } - if streamRequest.body != nil { - go streamRequest.flushBody() - } - err := w.writeStreamData(streamRequest.stream, log) - if err != nil { - return err - } - w.idleTimer.MarkActive() - case streamID := <-w.readyStreamChan: - stream, ok := w.streams.Get(streamID) - if !ok { - continue - } - err := w.writeStreamData(stream, log) - if err != nil { - return err - } - w.idleTimer.MarkActive() - case useDict := <-w.useDictChan: - err := w.writeUseDictionary(useDict) - if err != nil { - log.Error().Msgf("mux - write: error writing use dictionary: %s", err) - return err - } - w.idleTimer.MarkActive() - } - } -} - -func (w *MuxWriter) writeStreamData(stream *MuxedStream, log *zerolog.Logger) error { - log.Debug().Msgf("mux - write: writable: streamID: %d", stream.streamID) - chunk := stream.getChunk() - w.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow()) - w.metricsUpdater.updateSendWindow(stream.getSendWindow()) - if chunk.sendHeadersFrame() { - err := w.writeHeaders(chunk.streamID, chunk.headers) - if err != nil { - log.Error().Msgf("mux - write: error writing headers: %s: streamID: %d", err, stream.streamID) - return err - } - log.Debug().Msgf("mux - write: output headers: streamID: %d", stream.streamID) - } - - if chunk.sendWindowUpdateFrame() { - // Send a WINDOW_UPDATE frame to update our receive window. - // If the Stream ID is zero, the window update applies to the connection as a whole - // RFC7540 section-6.9.1 "A receiver that receives a flow-controlled frame MUST - // always account for its contribution against the connection flow-control - // window, unless the receiver treats this as a connection error" - err := w.f.WriteWindowUpdate(chunk.streamID, chunk.windowUpdate) - if err != nil { - log.Error().Msgf("mux - write: error writing window update: %s: streamID: %d", err, stream.streamID) - return err - } - log.Debug().Msgf("mux - write: increment receive window by %d streamID: %d", chunk.windowUpdate, stream.streamID) - } - - for chunk.sendDataFrame() { - payload, sentEOF := chunk.nextDataFrame(int(w.maxFrameSize)) - err := w.f.WriteData(chunk.streamID, sentEOF, payload) - if err != nil { - log.Error().Msgf("mux - write: error writing data: %s: streamID: %d", err, stream.streamID) - return err - } - // update the amount of data wrote - w.bytesWrote.IncrementBy(uint64(len(payload))) - log.Debug().Msgf("mux - write: output data: %d: streamID: %d", len(payload), stream.streamID) - - if sentEOF { - if stream.readBuffer.Closed() { - // transition into closed state - if !stream.gotReceiveEOF() { - // the peer may send data that we no longer want to receive. Force them into the - // closed state. - log.Debug().Msgf("mux - write: resetting stream: streamID: %d", stream.streamID) - w.f.WriteRSTStream(chunk.streamID, http2.ErrCodeNo) - } else { - // Half-open stream transitioned into closed - log.Debug().Msgf("mux - write: closing stream: streamID: %d", stream.streamID) - } - w.streams.Delete(chunk.streamID) - } else { - log.Debug().Msgf("mux - write: closing stream write side: streamID: %d", stream.streamID) - } - } - } - return nil -} - -func (w *MuxWriter) encodeHeaders(headers []Header) ([]byte, error) { - w.headerBuffer.Reset() - for _, header := range headers { - err := w.headerEncoder.WriteField(hpack.HeaderField{ - Name: header.Name, - Value: header.Value, - }) - if err != nil { - return nil, err - } - } - return w.headerBuffer.Bytes(), nil -} - -// writeHeaders writes a block of encoded headers, splitting it into multiple frames if necessary. -func (w *MuxWriter) writeHeaders(streamID uint32, headers []Header) error { - encodedHeaders, err := w.encodeHeaders(headers) - if err != nil || len(encodedHeaders) == 0 { - return err - } - - blockSize := int(w.maxFrameSize) - // CONTINUATION is unnecessary; the headers fit within the blockSize - if len(encodedHeaders) < blockSize { - return w.f.WriteHeaders(http2.HeadersFrameParam{ - StreamID: streamID, - EndHeaders: true, - BlockFragment: encodedHeaders, - }) - } - - choppedHeaders := chopEncodedHeaders(encodedHeaders, blockSize) - // len(choppedHeaders) is at least 2 - if err := w.f.WriteHeaders(http2.HeadersFrameParam{StreamID: streamID, EndHeaders: false, BlockFragment: choppedHeaders[0]}); err != nil { - return err - } - for i := 1; i < len(choppedHeaders)-1; i++ { - if err := w.f.WriteContinuation(streamID, false, choppedHeaders[i]); err != nil { - return err - } - } - if err := w.f.WriteContinuation(streamID, true, choppedHeaders[len(choppedHeaders)-1]); err != nil { - return err - } - - return nil -} - -// Partition a slice of bytes into `len(slice) / blockSize` slices of length `blockSize` -func chopEncodedHeaders(headers []byte, chunkSize int) [][]byte { - var divided [][]byte - - for i := 0; i < len(headers); i += chunkSize { - end := i + chunkSize - - if end > len(headers) { - end = len(headers) - } - - divided = append(divided, headers[i:end]) - } - - return divided -} - -func (w *MuxWriter) writeUseDictionary(dictRequest useDictRequest) error { - err := w.f.WriteRawFrame(FrameUseDictionary, 0, dictRequest.streamID, []byte{byte(dictRequest.dictID)}) - if err != nil { - return err - } - payload := make([]byte, 0, 64) - for _, set := range dictRequest.setDict { - payload = append(payload, byte(set.dictID)) - payload = appendVarInt(payload, 7, uint64(set.dictSZ)) - payload = append(payload, 0x80) // E = 1, D = 0, Truncate = 0 - } - - err = w.f.WriteRawFrame(FrameSetDictionary, FlagSetDictionaryAppend, dictRequest.streamID, payload) - return err -} diff --git a/h2mux/muxwriter_test.go b/h2mux/muxwriter_test.go deleted file mode 100644 index 07e23bdc..00000000 --- a/h2mux/muxwriter_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package h2mux - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestChopEncodedHeaders(t *testing.T) { - mockEncodedHeaders := make([]byte, 5) - for i := range mockEncodedHeaders { - mockEncodedHeaders[i] = byte(i) - } - chopped := chopEncodedHeaders(mockEncodedHeaders, 4) - - assert.Equal(t, 2, len(chopped)) - assert.Equal(t, []byte{0, 1, 2, 3}, chopped[0]) - assert.Equal(t, []byte{4}, chopped[1]) -} - -func TestChopEncodedEmptyHeaders(t *testing.T) { - mockEncodedHeaders := make([]byte, 0) - chopped := chopEncodedHeaders(mockEncodedHeaders, 3) - - assert.Equal(t, 0, len(chopped)) -} diff --git a/h2mux/readylist.go b/h2mux/readylist.go deleted file mode 100644 index d1a18c6d..00000000 --- a/h2mux/readylist.go +++ /dev/null @@ -1,151 +0,0 @@ -package h2mux - -import "sync" - -// ReadyList multiplexes several event signals onto a single channel. -type ReadyList struct { - // signalC is used to signal that a stream can be enqueued - signalC chan uint32 - // waitC is used to signal the ID of the first ready descriptor - waitC chan uint32 - // doneC is used to signal that run should terminate - doneC chan struct{} - closeOnce sync.Once -} - -func NewReadyList() *ReadyList { - rl := &ReadyList{ - signalC: make(chan uint32), - waitC: make(chan uint32), - doneC: make(chan struct{}), - } - go rl.run() - return rl -} - -// ID is the stream ID -func (r *ReadyList) Signal(ID uint32) { - select { - case r.signalC <- ID: - // ReadyList already closed - case <-r.doneC: - } -} - -func (r *ReadyList) ReadyChannel() <-chan uint32 { - return r.waitC -} - -func (r *ReadyList) Close() { - r.closeOnce.Do(func() { - close(r.doneC) - }) -} - -func (r *ReadyList) run() { - defer close(r.waitC) - var queue readyDescriptorQueue - var firstReady *readyDescriptor - activeDescriptors := newReadyDescriptorMap() - for { - if firstReady == nil { - select { - case i := <-r.signalC: - firstReady = activeDescriptors.SetIfMissing(i) - case <-r.doneC: - return - } - } - select { - case r.waitC <- firstReady.ID: - activeDescriptors.Delete(firstReady.ID) - firstReady = queue.Dequeue() - case i := <-r.signalC: - newReady := activeDescriptors.SetIfMissing(i) - if newReady != nil { - // key doesn't exist - queue.Enqueue(newReady) - } - case <-r.doneC: - return - } - } -} - -type readyDescriptor struct { - ID uint32 - Next *readyDescriptor -} - -// readyDescriptorQueue is a queue of readyDescriptors in the form of a singly-linked list. -// The nil readyDescriptorQueue is an empty queue ready for use. -type readyDescriptorQueue struct { - Head *readyDescriptor - Tail *readyDescriptor -} - -func (q *readyDescriptorQueue) Empty() bool { - return q.Head == nil -} - -func (q *readyDescriptorQueue) Enqueue(x *readyDescriptor) { - if x.Next != nil { - panic("enqueued already queued item") - } - if q.Empty() { - q.Head = x - q.Tail = x - } else { - q.Tail.Next = x - q.Tail = x - } -} - -// Dequeue returns the first readyDescriptor in the queue, or nil if empty. -func (q *readyDescriptorQueue) Dequeue() *readyDescriptor { - if q.Empty() { - return nil - } - x := q.Head - q.Head = x.Next - x.Next = nil - return x -} - -// readyDescriptorQueue is a map of readyDescriptors keyed by ID. -// It maintains a free list of deleted ready descriptors. -type readyDescriptorMap struct { - descriptors map[uint32]*readyDescriptor - free []*readyDescriptor -} - -func newReadyDescriptorMap() *readyDescriptorMap { - return &readyDescriptorMap{descriptors: make(map[uint32]*readyDescriptor)} -} - -// create or reuse a readyDescriptor if the stream is not in the queue. -// This avoid stream starvation caused by a single high-bandwidth stream monopolising the writer goroutine -func (m *readyDescriptorMap) SetIfMissing(key uint32) *readyDescriptor { - if _, ok := m.descriptors[key]; ok { - return nil - } - - var newDescriptor *readyDescriptor - if len(m.free) > 0 { - // reuse deleted ready descriptors - newDescriptor = m.free[len(m.free)-1] - m.free = m.free[:len(m.free)-1] - } else { - newDescriptor = &readyDescriptor{} - } - newDescriptor.ID = key - m.descriptors[key] = newDescriptor - return newDescriptor -} - -func (m *readyDescriptorMap) Delete(key uint32) { - if descriptor, ok := m.descriptors[key]; ok { - m.free = append(m.free, descriptor) - delete(m.descriptors, key) - } -} diff --git a/h2mux/readylist_test.go b/h2mux/readylist_test.go deleted file mode 100644 index 6ee9cfbf..00000000 --- a/h2mux/readylist_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package h2mux - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func assertEmpty(t *testing.T, rl *ReadyList) { - select { - case <-rl.ReadyChannel(): - t.Fatal("Spurious wakeup") - default: - } -} - -func assertClosed(t *testing.T, rl *ReadyList) { - select { - case _, ok := <-rl.ReadyChannel(): - assert.False(t, ok, "ReadyChannel was not closed") - case <-time.After(100 * time.Millisecond): - t.Fatalf("Timeout") - } -} - -func receiveWithTimeout(t *testing.T, rl *ReadyList) uint32 { - select { - case i := <-rl.ReadyChannel(): - return i - case <-time.After(100 * time.Millisecond): - t.Fatalf("Timeout") - return 0 - } -} - -func TestReadyListEmpty(t *testing.T) { - rl := NewReadyList() - - // no signals, receive should fail - assertEmpty(t, rl) -} -func TestReadyListSignal(t *testing.T) { - rl := NewReadyList() - assertEmpty(t, rl) - - rl.Signal(0) - if receiveWithTimeout(t, rl) != 0 { - t.Fatalf("Received wrong ID of signalled event") - } - - assertEmpty(t, rl) -} - -func TestReadyListMultipleSignals(t *testing.T) { - rl := NewReadyList() - assertEmpty(t, rl) - - // Signals should not block; - // Duplicate unhandled signals should not cause multiple wakeups - signalled := [5]bool{} - for i := range signalled { - rl.Signal(uint32(i)) - rl.Signal(uint32(i)) - } - // All signals should be received once (in any order) - for range signalled { - i := receiveWithTimeout(t, rl) - if signalled[i] { - t.Fatalf("Received signal %d more than once", i) - } - signalled[i] = true - } - for i := range signalled { - if !signalled[i] { - t.Fatalf("Never received signal %d", i) - } - } - assertEmpty(t, rl) -} - -func TestReadyListClose(t *testing.T) { - rl := NewReadyList() - rl.Close() - - // readyList.run() occurs in a separate goroutine, - // so there's no way to directly check that run() has terminated. - // Perform an indirect check: is the ready channel closed? - assertClosed(t, rl) - - // a second rl.Close() shouldn't cause a panic - rl.Close() - - // Signal shouldn't block after Close() - done := make(chan struct{}) - go func() { - for i := 0; i < 5; i++ { - rl.Signal(uint32(i)) - } - close(done) - }() - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.Fatal("Test timed out") - } -} - -func TestReadyDescriptorQueue(t *testing.T) { - var queue readyDescriptorQueue - items := [4]readyDescriptor{} - for i := range items { - items[i].ID = uint32(i) - } - - if !queue.Empty() { - t.Fatalf("nil queue should be empty") - } - queue.Enqueue(&items[3]) - queue.Enqueue(&items[1]) - queue.Enqueue(&items[0]) - queue.Enqueue(&items[2]) - if queue.Empty() { - t.Fatalf("Empty should be false after enqueue") - } - i := queue.Dequeue().ID - if i != 3 { - t.Fatalf("item 3 should have been dequeued, got %d instead", i) - } - i = queue.Dequeue().ID - if i != 1 { - t.Fatalf("item 1 should have been dequeued, got %d instead", i) - } - i = queue.Dequeue().ID - if i != 0 { - t.Fatalf("item 0 should have been dequeued, got %d instead", i) - } - i = queue.Dequeue().ID - if i != 2 { - t.Fatalf("item 2 should have been dequeued, got %d instead", i) - } - if !queue.Empty() { - t.Fatal("queue should be empty after dequeuing all items") - } - if queue.Dequeue() != nil { - t.Fatal("dequeue on empty queue should return nil") - } -} - -func TestReadyDescriptorMap(t *testing.T) { - m := newReadyDescriptorMap() - m.Delete(42) - // (delete of missing key should be a noop) - x := m.SetIfMissing(42) - if x == nil { - t.Fatal("SetIfMissing for new key returned nil") - } - if m.SetIfMissing(42) != nil { - t.Fatal("SetIfMissing for existing key returned non-nil") - } - // this delete has effect - m.Delete(42) - // the next set should reuse the old object - y := m.SetIfMissing(666) - if y == nil { - t.Fatal("SetIfMissing for new key returned nil") - } - if x != y { - t.Fatal("SetIfMissing didn't reuse freed object") - } -} diff --git a/h2mux/rtt.go b/h2mux/rtt.go deleted file mode 100644 index 350233e3..00000000 --- a/h2mux/rtt.go +++ /dev/null @@ -1,29 +0,0 @@ -package h2mux - -import ( - "sync/atomic" -) - -// PingTimestamp is an atomic interface around ping timestamping and signalling. -type PingTimestamp struct { - ts int64 - signal Signal -} - -func NewPingTimestamp() *PingTimestamp { - return &PingTimestamp{signal: NewSignal()} -} - -func (pt *PingTimestamp) Set(v int64) { - if atomic.SwapInt64(&pt.ts, v) != 0 { - pt.signal.Signal() - } -} - -func (pt *PingTimestamp) Get() int64 { - return atomic.SwapInt64(&pt.ts, 0) -} - -func (pt *PingTimestamp) GetUpdateChan() <-chan struct{} { - return pt.signal.WaitChannel() -} diff --git a/h2mux/sample/ghost-url.min.js b/h2mux/sample/ghost-url.min.js deleted file mode 100644 index eb4ecd50..00000000 --- a/h2mux/sample/ghost-url.min.js +++ /dev/null @@ -1 +0,0 @@ -!function(){"use strict";function a(a){var b,c=[];if(!a)return"";for(b in a)a.hasOwnProperty(b)&&(a[b]||a[b]===!1)&&c.push(b+"="+encodeURIComponent(a[b]));return c.length?"?"+c.join("&"):""}var b,c,d,e,f="https://cloudflare.ghost.io/ghost/api/v0.1/";d={api:function(){var d,e=Array.prototype.slice.call(arguments),g=f;return d=e.pop(),d&&"object"!=typeof d&&(e.push(d),d={}),d=d||{},d.client_id=b,d.client_secret=c,e.length&&e.forEach(function(a){g+=a.replace(/^\/|\/$/g,"")+"/"}),g+a(d)}},e=function(a){b=a.clientId?a.clientId:"",c=a.clientSecret?a.clientSecret:"",f=a.url?a.url:f.match(/{\{api-url}}/)?"":f},"undefined"!=typeof window&&(window.ghost=window.ghost||{},window.ghost.url=d,window.ghost.init=e),"undefined"!=typeof module&&(module.exports={url:d,init:e})}(); \ No newline at end of file diff --git a/h2mux/sample/index.html b/h2mux/sample/index.html deleted file mode 100644 index fe91d668..00000000 --- a/h2mux/sample/index.html +++ /dev/null @@ -1,537 +0,0 @@ - - - - - - - Cloudflare Blog - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - -
-
- - - -
-
-

Living In A Multi-Cloud World

-
- Published on - by Sergi Isasi. -
-
-
-

A few months ago at Cloudflare’s Internet Summit, we hosted a discussion on A Cloud Without Handcuffs with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS. The conversation touched on multiple areas, but it’s clear that more and more companies are recognizing the need to have some strategy around hosting their applications on multiple cloud providers. Earlier this year,…

-
- -
- - - -
-
-

The Supreme Court Wanders into the Patent Troll Fight

-
- Published on - by Edo Royker. -
-
-
-

Next Monday, the US Supreme Court will hear oral arguments in Oil States Energy Services, LLC vs. Greene’s Energy Group, LLC, which is a case to determine whether the Inter Partes Review (IPR) administrative process at the US Patent and Trademark Office (USPTO) used to determine the validity of patents is constitutional. The constitutionality of the IPR process is one of the biggest legal issues facing innovative…

-
- -
- - - -
-
-

7 Cloudflare Apps Which Increase User Engagement on Your Site

-
- Published on - by Andrew Fitch. -
-
-
-

Cloudflare Apps now lists 95 apps from apps which grow email lists to apps which acquire new customers to apps which help site owners make more money. The great thing about these apps is that users don't have to have any coding or development skills. They can just sign up for the app and start using it on their sites. Let’s take a moment to highlight some…

-
- -
- - - -
-
-

The Super Secret Cloudflare Master Plan, or why we acquired Neumob

-
- Published on - by John Graham-Cumming. -
-
-
-

We announced today that Cloudflare has acquired Neumob. Neumob’s team built exceptional technology to speed up mobile apps, reduce errors on challenging mobile networks, and increase conversions. Cloudflare will integrate the Neumob technology with our global network to give Neumob truly global reach. It’s tempting to think of the Neumob acquisition as a point product added to the Cloudflare portfolio. But it actually represents a key…

-
- -
- - - -
-
-

Thwarting the Tactics of the Equifax Attackers

-
- Published on - by Alex Cruz Farmer. -
-
-
-

We are now 3 months on from one of the biggest, most significant data breaches in history, but has it redefined people's awareness on security? The answer to that is absolutely yes, awareness is at an all-time high. Awareness, however, does not always result in positive action. The fallacy which is often assumed is "surely, if I keep my software up to date with all the patches, that's…

-
- -
- - - -
-
-

Go, don't collect my garbage

-
- Published on - by Vlad Krasnov. -
-
-
-

Not long ago I needed to benchmark the performance of Golang on a many-core machine. I took several of the benchmarks that are bundled with the Go source code, copied them, and modified them to run on all available threads. In that case the machine has 24 cores and 48 threads. CC BY-SA 2.0 image by sponki25 I started with ECDSA P256 Sign, probably because I have…

-
- -
- - - -
-
-

Cloudflare Wants to Buy Your Meetup Group Pizza

-
- Published on - by Andrew Fitch. -
-
-
-

If you’re a web dev / devops / etc. meetup group that also works toward building a faster, safer Internet, I want to support your awesome group by buying you pizza. If your group’s focus falls within one of the subject categories below and you’re willing to give us a 30 second shout out and tweet a photo of your group and @Cloudflare, your meetup’s pizza…

-
- -
- - - -
-
-

On the dangers of Intel's frequency scaling

-
- Published on - by Vlad Krasnov. -
-
-
-

While I was writing the post comparing the new Qualcomm server chip, Centriq, to our current stock of Intel Skylake-based Xeons, I noticed a disturbing phenomena. When benchmarking OpenSSL 1.1.1dev, I discovered that the performance of the cipher ChaCha20-Poly1305 does not scale very well. On a single thread, it performed at the speed of approximately 2.89GB/s, whereas on 24 cores, and 48 threads it…

-
- -
- - - - - -
- - -
- - - - - - - - - - - - diff --git a/h2mux/sample/index1.html b/h2mux/sample/index1.html deleted file mode 100644 index 7607f3e9..00000000 --- a/h2mux/sample/index1.html +++ /dev/null @@ -1,515 +0,0 @@ - - - - - - - Living In A Multi-Cloud World - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - -
-
- -
- - -
-

Living In A Multi-Cloud World

-
- - by Sergi Isasi. -
- -
- -
-

A few months ago at Cloudflare’s Internet Summit, we hosted a discussion on A Cloud Without Handcuffs with Joe Beda, one of the creators of Kubernetes, and Brandon Phillips, the co-founder of CoreOS. The conversation touched on multiple areas, but it’s clear that more and more companies are recognizing the need to have some strategy around hosting their applications on multiple cloud providers.

- -

Earlier this year, Mary Meeker published her annual Internet Trends report which revealed that 22% of respondents viewed Cloud Vendor Lock-In as a top 3 concern, up from just 7% in 2012. This is in contrast to previous top concerns, Data Security and Cost & Savings, both of which dropped amongst those surveyed.

- -

Internet Trends

- -

At Cloudflare, our mission is to help build a better internet. To fulfill this mission, our customers need to have consistent access to the best technology and services, over time. This is especially the case with respect to storage and compute providers. This means not becoming locked-in to any single provider and taking advantage of multiple cloud computing vendors (such as Amazon Web Services or Google Cloud Platform) for the same end user services.

- -

The Benefits of Having Multiple Cloud Vendors

- -

There are a number of potential challenges when selecting a single cloud provider. Though there may be scenarios where it makes sense to consolidate on a single vendor, our belief is that it is important that customers are aware of their choice and downsides of being potentially locked-in to that particular vendor. In short, know what trade offs you are making should you decide to continue to consolidate parts of your network, compute, and storage with a single cloud provider. While not comprehensive, here are a few trade-offs you may be making if you are locked-in to one cloud.

- -

Cost Efficiences

- -

For some companies, there may be a cost savings involved in spreading traffic across multiple vendors. Some can take advantage of free or reduced cost tiers at lower volumes. Vendors may provide reduced costs for certain times of day that are lower utilized on their infrastructure. Applications can have varying compute requirements amongst layers of the application: some may require faster, immediate processing while others may benefit from delayed processing at a lower cost.

- -

Negotiation Strength

- -

One of the most important reasons to consider deploying in multiple cloud providers is to minimize your reliance on a single vendor’s technology for your critical business processes. As you become more vertically integrated with any vendor, your negotiation posture for pricing or favorable contract terms becomes diminished. Having production ready code available on multiple providers allows you to have less technical debt should you need to change. If you go a step further and are already sending traffic to multiple providers, you have minimized the technical debt required to switch and can negotiate from a position of strength.

- -

Business Continuity or High Availability

- -

While the major cloud providers are generally reliable, there have been a few notable outages in recent years. The most significant in recent memory being Amazon’s US-EAST S3 outage in February. Some organizations may have a policy specifying multiple providers for high availability while others should consider it where necessary and feasible as a best practice. A multi-cloud strategy can lower operational risk from a single vendor’s mistakes causing a significant outage for a mission critical application.

- -

Experimentation

- -

One of the exciting things about having competition in the space is the level of innovation and feature velocity of each provider. Every year there are major announcements of new products or features that may have a significant impact on improving your organization's competitive advantage. Having test and production environments in multiple providers gives your engineers the ability to understand and experiment with a new capability in the context of your technology stack and data. You may even try these features for a portion of your traffic and get real world data on any benefits realized.

- -

Cloudflare’s Role

- -

Cloudflare is an independent third party in your multi-cloud strategy. Our goal is to minimize the layers of lock-in between you and a provider and lower the effort of change. In particular, one area where we can help right away is to minimize the operational changes necessary at the network, similar to what Kubernetes can do at the storage and compute level. As a benefit of our network, you can also have a centralized point for security and operational control.

- -

Cloudflare Multi Cloud

- -

Cloudflare’s Load Balancing can easily be configured to act as your global application traffic aggregator and distribute your traffic amongst origins at as many clouds as you choose to utilize. Active layer 7 health checks continually probe your origins and can automatically move traffic in the case of network or application failure. All consolidated web traffic can be inspected and acted upon by Cloudflare’s best of breed Security services, providing a single control point and visibility across all application traffic, regardless of which cloud the origin may be on. You also have the benefit of Cloudflare’s Global Anycast Network, providing for better speed and higher availability regardless of which clouds your origins are hosted on.

- -

Billforward: Using Cloudflare to Implement Multi-Cloud

- -

Billforward is a San Francisco and London based startup that is focused and mission driven on changing the way people bill and charge their customers, providing a solution to the complexities of Quote-to-Cash. Their platform is built on a number of Rest APIs that other developers call to bill and generate revenue for their own companies.

- -

Billforward is using Cloudflare for its core customer facing application to failover traffic between Google Compute Engine and Amazon Web Services. Acting as a reverse proxy, Cloudflare receives all requests for and decides which of Billforward’s two configured cloud origins to use based upon the availability of that origin in near real-time. This allows Billforward to completely manage the connections to and from two disparate cloud providers using Cloudflare’s UI or API. Billforward is in the process of migrating all of their customer facing domains to a similar setup.

- -

Configuration

- -

Billforward has a single load balanced hostname with two available Pools. They’ve named the two Pools with “gce” and “aws” labels and each Pool has one Origin associated with it. All of the Pools are enabled and the entire LB/hostname is proxied through Cloudflare (as indicated by the orange cloud).

- -

Billforward Configuration UI

- -

Cloudflare probes Billforward’s Origins once every minute from all of Cloudflare’s data centers around the world (a feature available to all Load Balancing Enterprise customers). If Billforward’s GCE Origin goes down, Cloudflare will quickly and automatically failover to the AWS Origin with no actions required from Billforward’s team.

- -

Google Compute Engine was chosen as the primary provider for this application by virtue of cost. Martin Lee, Site Reliability Engineer at Billforward says, “Essentially, GCE is cheaper for our general purpose computing needs but we're more experienced with deployments in AWS. This strategy allows us to switch back and forth at will and avoid being tied in to either platform.” It is likely that Billforward will change the priority as pricing models evolve.
-

- -
-

“It's a fairly fast moving world and features released by cloud providers can have a meaningful impact on performance and cost on a week by week basis - it helps to stay flexible,” says Martin. “We may also change priority based on features.”

-
- -


For orchestration of the compute and storage layers, Billforward uses Docker containers managed through Rancher. They use distinct environments between cloud providers but are considering bridging an environment across cloud providers and using VPNs between them, which will enable them to move load between providers even more easily. “Our system is loosely coupled through a message queue,” adds Martin. “Having a container system across clouds means we can really take advantage of this - we can very easily move workloads across clouds without any danger of dropping tasks or ending up in an inconsistent state.”

- -

Benefits

- -

Billforward manages these connections at Cloudflare’s edge. Through this interface (or via the Cloudflare APIs), they can also manually move traffic from GCE to AWS by just disabling the GCE pool or by rearranging the Pool priority and make AWS the primary. These changes are near instant on the Cloudflare network and require no downtime to Billforward’s customer facing application. This allows them to act on potential advantageous pricing changes between the two cloud providers or move traffic to hit pricing tiers.

- -

In addition, Billforward is now not “locked-in” to either provider’s network; being able to move traffic and without any downtime means they can make traffic changes independent of Amazon or Google. They can also integrate additional cloud providers any time they deem fit: adding Microsoft Azure, for example, as a third Origin would be as simple as creating a new Pool and adding it to the Load Balancer.

- -

Billforward is a good example of a forward thinking company that is taking advantage of technologies from multiple providers to best serve their business and customers, while not being reliant on a single vendor. For further detail on their setup using Cloudflare, please check their blog.

-
- - - - - - -
- - - comments powered by Disqus - -
- - - - - - - -
- - -
- - - - - - - - - - - - diff --git a/h2mux/sample/index2.html b/h2mux/sample/index2.html deleted file mode 100644 index fe59d28e..00000000 --- a/h2mux/sample/index2.html +++ /dev/null @@ -1,502 +0,0 @@ - - - - - - - SCOTUS Wanders into Patent Troll Fight - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - -
-
- -
- - -
-

The Supreme Court Wanders into the Patent Troll Fight

-
- - by Edo Royker. -
- -
- -
-

Next Monday, the US Supreme Court will hear oral arguments in Oil States Energy Services, LLC vs. Greene’s Energy Group, LLC, which is a case to determine whether the Inter Partes Review (IPR) administrative process at the US Patent and Trademark Office (USPTO) used to determine the validity of patents is constitutional.

- -

The constitutionality of the IPR process is one of the biggest legal issues facing innovative technology companies, as the availability of this process has greatly reduced the anticipated costs, and thereby lessened the threat, of patent troll litigation. As we discuss in this blog post, it is ironic that the outcome of a case that is of such great importance to the technology community today may hinge on what courts in Britain were and were not doing more than 200 years ago.

- -

Thomas Rowlandson [Public domain], via Wikimedia Commons

- -

As we have discussed in prior blog posts, the stakes are high: if the Supreme Court finds IPR unconstitutional, then the entire system of administrative review by the USPTO — including IPR and ex parte processes — will be shuttered. This would be a mistake, as administrative recourse at the USPTO is one of the few ways to avoid the considerable costs and delays of federal court litigation, which can take years and run into the millions of dollars. Those heavy costs are often leveraged by patent trolls when they threaten litigation in the effort to procure easy and lucrative settlements from their targets.

- -

Cloudflare is Pursuing Our Fight Against Patent Trolls All the Way to the Steps of the Supreme Court

- -

Cloudflare joined Dell, Facebook, and a number of other companies, all practicing entities with large patent portfolios, in a brief amici curiae (or ‘friend of the court’ brief) in support of the IPR process, because it has a substantial positive impact on technological innovation in the United States. Amicus briefs allow parties who are interested in the outcome of a case, but are not parties to the immediate dispute before the court, to have input into the court’s deliberations.

- -

As many of you are aware, we were sued by Blackbird Technologies, a notorious patent troll, earlier this year for patent infringement, and initiated Project Jengo to crowd source prior art searches and invalidate Blackbird’s patents. One of our strategies for quickly and efficiently invalidating Blackbird’s patents is to take advantage of the IPR process at the USPTO, which can be completed in about half the time and at one tenth of the cost of a federal court case, and to initiate ex parte proceedings against Blackbird’s other patents that are overly broad and invalid.

- -

A full copy of the Amicus Brief we joined in the Oil States case is available here, and a summary of the argument follows.

- -

Oil States Makes its Case

- -

Oil States is an oilfield services and drilling equipment manufacturing company. The USPTO invalidated one of its patents related to oil drilling technology in an IPR proceeding while Oil States had a lawsuit pending against one of its competitors claiming infringement of its patent. After it lost the IPR, Oil States lost an appeal in a lower federal court based on the findings of the IPR proceeding. The Supreme Court agreed to hear the case to determine whether once the USPTO issues a patent, an inventor has a constitutionally protected property right that — under Article III of the U.S. Constitution (which outlines the powers of the judicial branch of the government), and the 7th Amendment (which addresses the right to a jury trial in certain types of cases) — cannot be revoked without intervention by the court system.

- -

Image by Paul Lowry

- -

As the patent owner, Oil States argues that the IPR process violates the relevant provisions of the constitution by allowing an administrative body, the Patent Trial and Appeal Board (PTAB)--a non-judicial forum, to decide a matter which was historically handled by the judiciary. This argument rests upon the premise that there was a historical analogue to cancellation of patent claims available in the judiciary. Since cancellation of patent claims was historically available in the judiciary, the cancellation of patent claims today must be consistent with that history and done exclusively by courts.

- -

This argument is flawed on multiple counts, which are set forth in the “friend of the court” brief we joined.

- -

First Flaw: An Administrative Process Even an Originalist Can Love

- -

As the amicus brief we joined points out, patent revocation did not historically rest within the exclusive province of the common law and chancery courts, the historical equivalents in Britain to the judiciary in the United States. Rather, prior to the Founding of the United States, patent revocation rested entirely with the Crown of England’s Privy Council, a non-judicial body comprising of advisors to the king or queen of England. It wasn’t until later that the Privy Council granted the chancery court (the judiciary branch) concurrent authority to revoke patents. Because a non-judicial body had the authority to revoke patents when the US Constitution was framed, the general principles of separation of powers and the right to trial in the Constitution do not require that patentability challenges be decided solely by courts.

- -

Second Flaw: The Judicial Role was Limited

- -

Not only did British courts share the power to address patent rights historically, the part shared by the the courts was significantly limited. Historically, the common-law and chancery courts only received a partial delegation of the Privy Council’s authority to invalidate patents. Courts only had the authority to invalidate patents for issues related to things like inequitable conduct (e.g., making false statements in the original patent application). The limited authority delegated to the England Courts did not include the authority to seek claim cancellation based on elements intrinsic to the patent or patent application, like lack of novelty or obviousness as done under an IPR proceeding. Rather, such authority remained with the Privy Council, a non-court authority, which decided questions like whether the invention was really new. Thus, like the PTAB, the Privy Council was a non-judicial body charged with responsibility to assess patent validity based on criteria that included the novelty of the invention.

- -

We think these arguments are compelling and provide very strong reasons why the Supreme Court should resist the request that such matters be resolved exclusively in federal courts. We hope that’s the position they do take because the real world implications are significant.

- -

Don’t Mess with a Good Thing

- -

The IPR process is not only consistent with the US Constitution, but it also advances the Patent Clause’s objective of promoting the progress of science and useful arts. That is, the “quid pro quo of the patent system; the public must receive meaningful disclosure in exchange for being excluded from practicing the invention for a limited period of time” by patent rights. (Enzo Biochem, Inc. v. Gen-probe Inc.) Congress created the IPR process in the America Invents Act in 2011 to use administrative review to weed out poor-quality patents that did not satisfy this quid pro quo because they had not actually disclosed very much. Congress sought to provide quick and cost effective administrative procedures for challenging the validity of patent claims that did not disclose novel inventions, or that claimed to disclose substantially more innovation than they actually did, to improve patent quality and restore confidence in the presumption of validity. In other words, Congress created a system to specifically permit the efficient challenge of the zealous assertion of vague and overly broad patents.

- -

As a recent study by the Congressional Research Service found, non-practicing entity (i.e., patent troll) patent litigation “activity cost defendants and licensees $29 billion in 2011, a 400 percent increase over $7 billion in 2005” and “the losses are mostly deadweight, with less than 25 percent flowing to innovation and at least that much going towards legal fees.” (see Brian T. Yeh, Cong. Research sERV., R42668) The IPR process enables innovative companies to navigate patent troll activity in an efficient manner and devote a greater proportion of their resources to research and development, rather than litigation or cost-of-litigation settlement fees for invalid patents.

- -

By EFF-Graphics (Own work), via Wikimedia Commons

- -

Additionally, the IPR process reduces the total number and associated costs of patent disputes in a number of ways.

- -
    -

  • Patent owners, especially patent trolls, are less likely to threaten litigation or file an infringement suit based on patent claims that they know or suspect to be invalid. In fact, patent owners who threaten or file suit merely to seek cost-of-litigation settlements have become far less prevalent because of the availability of the IPR process to reduce the cost of litigation.

    • -

  • Patent owners are less likely to initiate litigation out of concerns that the IPR proceedings may culminate in PTAB’s cancellation of all patent claims asserted in the infringement suit.

    • -

  • Where the PTAB does not cancel all asserted claims, statutory estoppel and the PTAB’s claim construction may serve to narrow the infringement issues to be resolved by the district court.

    • -
- -

Our hope is that the US Supreme Court justices take into full consideration the larger community of innovative companies that are helped by the IPR system in battling patent trolls, and do not limit their consideration to the implications on the parties to Oil States (neither of which is a non-practicing entity). As we have explained, not only does the IPR process enable innovative companies to focus their resources on technological innovation, instead of legal fees, but allowing the USPTO to administer IPR and ex parte proceedings is entirely consistent with the US Constitution.

- -

While we await a decision in Oil States, expect to see Cloudflare initiate IPR and ex parte proceedings against Blackbird Technologies patents in the coming months.

- -

We will make sure to keep you updated.

-
- - - - - - -
- - - comments powered by Disqus - -
- - - - - - - -
- - -
- - - - - - - - - - - - diff --git a/h2mux/sample/jquery.fitvids.js b/h2mux/sample/jquery.fitvids.js deleted file mode 100644 index a8551f6e..00000000 --- a/h2mux/sample/jquery.fitvids.js +++ /dev/null @@ -1,74 +0,0 @@ -/*global jQuery */ -/*jshint multistr:true browser:true */ -/*! -* FitVids 1.0.3 -* -* Copyright 2013, Chris Coyier - http://css-tricks.com + Dave Rupert - http://daverupert.com -* Credit to Thierry Koblentz - http://www.alistapart.com/articles/creating-intrinsic-ratios-for-video/ -* Released under the WTFPL license - http://sam.zoy.org/wtfpl/ -* -* Date: Thu Sept 01 18:00:00 2011 -0500 -*/ - -(function( $ ){ - - "use strict"; - - $.fn.fitVids = function( options ) { - var settings = { - customSelector: null - }; - - if(!document.getElementById('fit-vids-style')) { - - var div = document.createElement('div'), - ref = document.getElementsByTagName('base')[0] || document.getElementsByTagName('script')[0], - cssStyles = '­'; - - div.className = 'fit-vids-style'; - div.id = 'fit-vids-style'; - div.style.display = 'none'; - div.innerHTML = cssStyles; - - ref.parentNode.insertBefore(div,ref); - - } - - if ( options ) { - $.extend( settings, options ); - } - - return this.each(function(){ - var selectors = [ - "iframe[src*='player.vimeo.com']", - "iframe[src*='youtube.com']", - "iframe[src*='youtube-nocookie.com']", - "iframe[src*='kickstarter.com'][src*='video.html']", - "object", - "embed" - ]; - - if (settings.customSelector) { - selectors.push(settings.customSelector); - } - - var $allVideos = $(this).find(selectors.join(',')); - $allVideos = $allVideos.not("object object"); // SwfObj conflict patch - - $allVideos.each(function(){ - var $this = $(this); - if (this.tagName.toLowerCase() === 'embed' && $this.parent('object').length || $this.parent('.fluid-width-video-wrapper').length) { return; } - var height = ( this.tagName.toLowerCase() === 'object' || ($this.attr('height') && !isNaN(parseInt($this.attr('height'), 10))) ) ? parseInt($this.attr('height'), 10) : $this.height(), - width = !isNaN(parseInt($this.attr('width'), 10)) ? parseInt($this.attr('width'), 10) : $this.width(), - aspectRatio = height / width; - if(!$this.attr('id')){ - var videoID = 'fitvid' + Math.floor(Math.random()*999999); - $this.attr('id', videoID); - } - $this.wrap('
').parent('.fluid-width-video-wrapper').css('padding-top', (aspectRatio * 100)+"%"); - $this.removeAttr('height').removeAttr('width'); - }); - }); - }; -// Works with either jQuery or Zepto -})( window.jQuery || window.Zepto ); diff --git a/h2mux/sample/screen.css b/h2mux/sample/screen.css deleted file mode 100644 index 8583251a..00000000 --- a/h2mux/sample/screen.css +++ /dev/null @@ -1,70 +0,0 @@ -html,body,div,span,applet,object,iframe,h1,h2,h3,h4,h5,h6,p,blockquote,pre,a,abbr,acronym,address,big,cite,code,del,dfn,em,img,ins,kbd,q,s,samp,small,strike,strong,sub,sup,tt,var,b,u,i,center,dl,dt,dd,ol,ul,li,fieldset,form,label,legend,table,caption,tbody,tfoot,thead,tr,th,td,article,aside,canvas,details,embed,figure,figcaption,footer,header,menu,nav,output,ruby,section,summary,time,mark,audio,video{margin:0;padding:0;border:0;font:inherit;font-size:100%;vertical-align:baseline}html{line-height:1}ol,ul{list-style:none}table{border-collapse:collapse;border-spacing:0}caption,th,td{text-align:left;font-weight:normal;vertical-align:middle}q,blockquote{quotes:none}q:before,q:after,blockquote:before,blockquote:after{content:"";content:none}a img{border:none}article,aside,details,figcaption,figure,footer,header,menu,nav,section,summary{display:block}.clearfix,.dl-horizontal,.row,.columns,.wrapper,.control-group,.input-picker .ws-picker-body,.input-picker .ws-button-row,.input-picker .picker-grid,.input-picker .picker-list,.footer-nav,.modal-header,.modal-content,.modal-footer,.modal-body-section,.table-meta,.mod-row,.mod-toolbar{*zoom:1}.clearfix:before,.dl-horizontal:before,.row:before,.columns:before,.wrapper:before,.control-group:before,.input-picker .ws-picker-body:before,.input-picker .ws-button-row:before,.input-picker .picker-grid:before,.input-picker .picker-list:before,.footer-nav:before,.modal-header:before,.modal-content:before,.modal-footer:before,.modal-body-section:before,.table-meta:before,.mod-row:before,.mod-toolbar:before,.clearfix:after,.dl-horizontal:after,.row:after,.columns:after,.wrapper:after,.control-group:after,.input-picker .ws-picker-body:after,.input-picker .ws-button-row:after,.input-picker .picker-grid:after,.input-picker .picker-list:after,.footer-nav:after,.modal-header:after,.modal-content:after,.modal-footer:after,.modal-body-section:after,.table-meta:after,.mod-row:after,.mod-toolbar:after{content:'';display:table}.clearfix:after,.dl-horizontal:after,.row:after,.columns:after,.wrapper:after,.control-group:after,.input-picker .ws-picker-body:after,.input-picker .ws-button-row:after,.input-picker .picker-grid:after,.input-picker .picker-list:after,.footer-nav:after,.modal-header:after,.modal-content:after,.modal-footer:after,.modal-body-section:after,.table-meta:after,.mod-row:after,.mod-toolbar:after{clear:both}.border-box,.columns,.columns>.column,.btn,button,input[type="button"],input[type="submit"],input,select,textarea,.switch,.file:before,.proxy .cloud,.control-group,.input-prepend .btn,.input-prepend .add-on,.input-append .btn,.input-append .add-on,.flexbox .control-group,.flexbox .control-label,.flexbox .controls,.ws-input input,.ws-input .ws-input-seperator,pre,.mod-row,.mod-cell,.mod-setting-control,.mod-control-group .ui-block{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}@-webkit-keyframes menuTransition{0%{display:none;opacity:0}1%{display:block;opacity:0;top:80%}100%{display:none;opacity:1;top:102%}}@-moz-keyframes menuTransition{0%{display:none;opacity:0}1%{display:block;opacity:0;top:80%}100%{display:none;opacity:1;top:102%}}@-ms-keyframes menuTransition{0%{display:none;opacity:0}1%{display:block;opacity:0;top:80%}100%{display:none;opacity:1;top:102%}}@keyframes menuTransition{0%{display:none;opacity:0}1%{display:block;opacity:0;top:80%}100%{display:none;opacity:1;top:102%}}@-webkit-keyframes bgFadeOut{100%{background-color:transparent}}@-moz-keyframes bgFadeOut{100%{background-color:transparent}}@-ms-keyframes bgFadeOut{100%{background-color:transparent}}@keyframes bgFadeOut{100%{background-color:transparent}}@font-face{font-family:'Open Sans';font-style:normal;font-weight:300;format("embedded-opentype"),url('../fonts/opensans-300.woff') format("woff"),url('../fonts/opensans-300.ttf') format("truetype"),url('../fonts/opensans-300.svg#open_sanssemibold') format("svg")}@font-face{font-family:'Open Sans';font-style:normal;font-weight:400;src:url('../fonts/opensans-400.eot');src:local("Open Sans"),local("OpenSans"),url('../fonts/opensans-400.eot?#iefix') format("embedded-opentype"),url('../fonts/opensans-400.woff') format("woff"),url('../fonts/opensans-400.ttf') format("truetype"),url('../fonts/opensans-400.svg#open_sansregular') format("svg")}@font-face{font-family:'Open Sans';font-style:normal;font-weight:600;src:local("Open Sans Semibold"),local("OpenSans-Semibold"),format("embedded-opentype"),url('../fonts/opensans-600.woff') format("woff"),url('../fonts/opensans-600.ttf') format("truetype"),url('../fonts/opensans-600.svg#open_sanssemibold') format("svg")}@font-face{font-family:'Open Sans';font-style:normal;font-weight:700;src:url('../fonts/opensans-700.eot');src:local("Open Sans Bold"),local("OpenSans-Bold"),url('../fonts/opensans-700.eot?#iefix') format("embedded-opentype"),url('../fonts/opensans-700.woff') format("woff"),url('../fonts/opensans-700.ttf') format("truetype"),url('../fonts/opensans-700.svg#open_sansbold') format("svg")}@font-face{font-family:'Open Sans';font-style:italic; - font-weight:300; - src:local("Open Sans Light Italic"),local("OpenSansLight-Italic")format("embedded-opentype"),url('../fonts/opensans-300i.woff') format("woff"),url('../fonts/opensans-300i.ttf') format("truetype"),url('../fonts/opensans-300i.svg#open_sanslight_italic') format("svg")}@font-face{font-family:'Open Sans';font-style:italic;font-weight:400;src:url('../fonts/opensans-400i.eot');src:local("Open Sans Italic"),local("OpenSans-Italic"),url('../fonts/opensans-400i.eot?#iefix') format("embedded-opentype"),url('../fonts/opensans-400i.woff') format("woff"),url('../fonts/opensans-400i.ttf') format("truetype"),url('../fonts/opensans-400i.svg#open_sansitalic') format("svg")}.select2-container{position:relative;vertical-align:top;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.select2-container .select2-choice{color:#333}.select2-container.select2-drop-above .select2-choice{background-color:#fff;border-bottom-color:#b1b1b1}.select2-choice{background-color:#fff;border:1px solid #b1b1b1;display:block;font-size:0.93333rem;font-weight:400;line-height:1.2;padding:0.53333rem 0 0.53333rem 0.8rem;position:relative;text-decoration:none;white-space:nowrap;-webkit-user-select:none;-moz-user-select:none;user-select:none;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.select2-choice:hover{border-color:#989898}.select2-choice .select2-chosen{margin-right:3rem;min-height:1em;display:block;overflow:hidden;white-space:nowrap;-o-text-overflow:ellipsis;-ms-text-overflow:ellipsis;text-overflow:ellipsis}.select2-choice .select2-arrow{background:transparent;border-left:1px solid #b1b1b1;display:block;height:100%;position:absolute;right:0;top:0;width:2rem}.select2-choice .select2-arrow b{display:block;width:100%;height:100%;position:relative}.select2-choice .select2-arrow b:before,.select2-choice .select2-arrow b:after{border:4px solid transparent;border-bottom-color:#bebebe;content:'';display:block;height:0;left:50%;margin-left:-5px;margin-top:-9px;position:absolute;top:50%;width:0}.select2-choice .select2-arrow b:before{border-bottom-color:transparent;border-top-color:#bebebe;margin-top:3px}.select2-choice abbr{display:none}.select2-allowclear .select2-choice .select2-chosen{margin-right:3.5rem}.select2-allowclear .select2-choice abbr{display:block}.select2-container,.select2-drop,.select2-search,.select2-search input{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.select2-drop{background:#fff;border:1px solid #b1b1b1;border-top:0;color:#333;font-size:0.86667rem;margin-top:-2px;position:absolute;top:100%;width:100%;z-index:1010;-webkit-box-shadow:0 4px 5px rgba(0,0,0,0.15);-moz-box-shadow:0 4px 5px rgba(0,0,0,0.15);box-shadow:0 4px 5px rgba(0,0,0,0.15)}.select2-drop.select2-drop-above{margin-top:1px;border-top:1px solid #b1b1b1;border-bottom:0;-webkit-box-shadow:0 -4px 5px rgba(0,0,0,0.15);-moz-box-shadow:0 -4px 5px rgba(0,0,0,0.15);box-shadow:0 -4px 5px rgba(0,0,0,0.15)}.select2-drop.select2-drop-above .select2-search input{margin-top:4px}.select2-search{min-height:1.73333rem;margin:0;padding-left:0.26667rem;padding-right:0.26667rem;white-space:nowrap;width:100%;z-index:1020;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.select2-search input{background:#fff url('../images/cloudflare-sprite.png') no-repeat -35px -26px;border:1px solid #b1b1b1;font-size:1em;height:auto;outline:0;margin:0;min-height:1.73333rem;padding:0.26667rem 0.33333rem 0.26667rem 1.73333rem;width:100%;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none;-webkit-transition:none;-moz-transition:none;-o-transition:none;transition:none}.select2-search input.select2-active{background:#fff url('../images/spinner.gif') no-repeat 0 0}.select2-search-hidden{display:block;left:-9999em;position:absolute}.select2-container-active .select2-choice,.select2-container-active .select2-choices{border:1px solid #2f7bbf;outline:none;-webkit-box-shadow:0 0 5px rgba(0,0,0,0.3);-moz-box-shadow:0 0 5px rgba(0,0,0,0.3);box-shadow:0 0 5px rgba(0,0,0,0.3)}.select2-dropdown-open .select2-choice{border:1px solid #b1b1b1;border-bottom-color:transparent;-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.select2-dropdown-open .select2-choice .select2-arrow{background:transparent;border-left:none}.select2-results{margin:0.26667rem 0.26667rem 0.26667rem 0;max-height:20em;overflow-x:hidden;overflow-y:auto;padding:0 0 0 0.26667rem;position:relative}.select2-results .select2-result-sub{margin:0 0 0 0}.select2-results .select2-result-sub>li .select2-result-label{padding-left:1.33333rem}.select2-results .select2-result-sub .select2-result-sub>li .select2-result-label{padding-left:2.66667rem}.select2-results .select2-result-sub .select2-result-sub .select2-result-sub>li .select2-result-label{padding-left:4rem}.select2-results .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub>li .select2-result-label{padding-left:5.33333rem}.select2-results .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub>li .select2-result-label{padding-left:6.66667rem}.select2-results .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub>li .select2-result-label{padding-left:7.33333rem}.select2-results .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub .select2-result-sub>li .select2-result-label{padding-left:8rem}.select2-results li{list-style:none;display:list-item}.select2-results li.select2-result-with-children>.select2-result-label{font-weight:600}.select2-results .select2-no-results,.select2-results .select2-result-label{cursor:pointer;margin:0;padding:0.2rem 0.46667rem 0.26667rem}.select2-results .select2-highlighted{background:#2f7bbf;color:#fff}.select2-results .select2-highlighted em{background:transparent}.select2-results .select2-no-results,.select2-results .select2-searching,.select2-results .select2-selection-limit{display:list-item}.select2-results .select2-disabled{display:none}.select2-more-results.select2-active{background:#f2f2f2 url('../images/spinner.gif') no-repeat 100%}.select2-more-results{background:#f2f2f2;display:list-item}.select2-container.select2-container-disabled .select2-choice{background-color:#f2f2f2;background-image:none;border:1px solid #bebebe;cursor:default}.select2-container.select2-container-disabled .select2-choice div{background-color:#f2f2f2;background-image:none;border-left:0}.select2-container-multi{min-width:10em}.select2-container-multi .select2-choices{background-color:#fff;border:1px solid #b1b1b1;cursor:text;height:auto;height:1%;margin:0;min-height:1.86667rem;overflow:hidden;padding:0.13333em;position:relative;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.select2-container-multi .select2-choices li{float:left;list-style:none}.select2-container-multi .select2-choices .select2-search-field{margin:0;padding:0;white-space:nowrap}.select2-container-multi .select2-choices .select2-search-field input{background:transparent;border:0;color:#333;font-size:1rem;height:1.5rem;margin:1px 0;outline:0;padding:0 0.13333em}.select2-container-multi .select2-choices .select2-search-field input:focus{-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.select2-container-multi .select2-choices .select2-search-field input.select2-active{background:#fff url('../images/spinner.gif') no-repeat 100%}.select2-container-multi .select2-choices .select2-search-choice{background-color:#fff;border:1px solid #b1b1b1;color:#333;cursor:default;line-height:0.86667rem;margin:0.2rem 0 0.13333rem 0.33333rem;padding:0.2rem 0.33333rem 0.2rem 1.2rem;position:relative}.select2-container-multi.select2-container-active .select2-choices{border:1px solid #2f7bbf;outline:none;-webkit-box-shadow:0 0 5px rgba(0,0,0,0.3);-moz-box-shadow:0 0 5px rgba(0,0,0,0.3);box-shadow:0 0 5px rgba(0,0,0,0.3)}.select2-default{color:#333}.select2-container-multi .select2-choices .select2-container-multi .select2-choices .select2-search-choice .select2-search-choice-close{cursor:default}.select2-container-multi .select2-choices .select2-search-choice-focus{background:#f2f2f2}.select2-search-choice-close{background:url('../images/cloudflare-sprite.png') no-repeat top right;border:0;cursor:pointer;display:block;font-size:1px;height:0.8rem;outline:0;position:absolute;right:2.46667rem;text-decoration:none;top:0.6rem;width:0.8rem}.select2-search-choice-close:hover{background-position:right -11px;cursor:pointer}.select2-container-multi .select2-search-choice-close{left:0.53333rem}.select2-container-multi .select2-choices .select2-search-choice .select2-search-choice-close:hover{background-position:right -11px}.select2-container-multi .select2-choices .select2-search-choice-focus .select2-search-choice-close{background-position:right -11px}.select2-container-multi.select2-container-disabled .select2-choices{background-color:#fafafa;background-image:none;border:1px solid #cbcbcb;cursor:default}.select2-container-multi.select2-container-disabled .select2-choices .select2-search-choice{background-image:none;background-color:#fafafa;border:1px solid #cbcbcb;padding:0.2rem 0.33333rem 0.2rem 0.33333rem}.select2-container-multi.select2-container-disabled .select2-choices .select2-search-choice .select2-search-choice-close,.select2-display-none{display:none}.select2-result-selectable .select2-match,.select2-result-unselectable .select2-result-selectable .select2-match{text-decoration:underline}.select2-result-unselectable .select2-match{text-decoration:none}.select2-offscreen{position:absolute;left:-9999px}.select2-drop-mask{bottom:0;left:0;max-height:100%;max-width:100%;position:fixed;right:0;top:0;z-index:1009}@media only screen and (-webkit-min-device-pixel-ratio: 1.5){.select2-search input,.select2-search-choice-close,.select2-container .select2-choice abbr{background-image:url('../images/select2x2-cf.png') !important;background-repeat:no-repeat !important;background-size:60px 45px !important}.select2-search input{background-position:100% -24px !important}}.flashblocker-assumed{min-height:20px;min-width:20px;z-index:2147483647}.cue-display{position:absolute !important;margin:0;padding:0px !important;max-width:100% !important;max-height:100% !important;border:none !important;background:none !important;text-align:center;visibility:hidden;font-family:sans-serif;font-size:12px;white-space:pre-wrap;overflow:hidden}.cue-display b{font-weight:bold}.cue-display i{font-style:italic}.cue-display u{text-decoration:underline}.cue-display span.cue-wrapper{position:absolute;left:0;bottom:0;right:0;display:block;padding:0;margin:0;width:100%;font-size:160%;color:#fff;visibility:visible !important}.cue-display .cue-line{display:block}.cue-display span.cue{display:inline-block;padding:3px 5px;background:#000;background:rgba(0,0,0,0.8);color:#fff}.cue-display .description-cues{position:absolute;top:-99px;left:-99px;display:block;width:5px;height:5px;overflow:hidden}mark{background-color:#ff9;color:#000;font-style:italic;font-weight:bold}.ws-important-hide{display:none !important;visibility:hidden !important;position:absolute;top:-999999px}.webshims-visual-hide{position:absolute !important;top:0 !important;left:0 !important;visibility:hidden !important;width:0 !important;height:0 !important;overflow:hidden !important}.webshims-visual-hide *{visibility:hidden !important}#swflocalstorageshim-wrapper{position:absolute;top:-999px;left:-9999px;overflow:hidden;width:215px;min-width:215px !important;z-index:2147483647}#swflocalstorageshim-wrapper .polyfill-exceeded-message{margin:0 0 5px;padding:5px;background:#ffd;color:#000;font-size:13px}#swflocalstorageshim-wrapper object{min-height:138px !important;min-width:215px !important}details{overflow:hidden}summary{position:relative}.closed-details-child{display:none !important}.closed-details-summary .details-open-indicator{background-position:0 -20px}.polyfill-important .details-open-indicator{margin:-1px 0 0 !important;padding:0 !important;border:0 !important;display:inline-block !important;width:16px !important;height:11px !important; vertical-align:middle !important}.polyfill-important .closed-details-summary .details-open-indicator{background-position:0 -20px !important}summary.summary-has-focus{outline:1px dotted #aaa;outline-offset:-1px}::selection,::-moz-selection{background:rgba(246,139,31,0.2);color:#333}img::selection,img::-moz-selection{background:rgba(246,139,31,0.3)}html,body{color:#333;font-family:"Open Sans",Helvetica,Arial,sans-serif;font-size:15px;line-height:1.5;-webkit-tap-highlight-color:rgba(246,139,31,0.3);-webkit-font-smoothing:antialiased}b,strong{font-weight:700}i,em{font-style:italic}small{font-size:80%}sup,.sup,sub,.sub{font-size:60%;position:relative;vertical-align:top}sup,.sup{top:0.25em}sub,.sub{bottom:0.25em;vertical-align:bottom}p .ui-item{margin:0 0.4rem;vertical-align:baseline}ul{list-style-type:disc}ul.circle{list-style-type:circle}ul.disc{list-style-type:disc}ul.square{list-style-type:square}ol{list-style-type:decimal}ol.roman{list-style-type:lower-roman}ol.roman-upper{list-style-type:upper-roman}ol.alpha{list-style-type:lower-alpha}ol,ul{list-style-position:outside;margin-left:3em}ol.unstyled,ul.unstyled,.exceptions-list{list-style-type:none;margin-left:0}ol.inline,ol.inline li,ul.inline,ul.inline li{display:inline;margin:0}a{color:#2f7bbf;outline:none;text-decoration:none;-webkit-transition:all 0.15s ease;-moz-transition:all 0.15s ease;-o-transition:all 0.15s ease;transition:all 0.15s ease}a:hover{color:#f68b1f}a:focus{color:#62a1d8;outline:none}a:active{color:#c16508;outline:none}h1,h2,h3,h4,h5,h6{font-weight:600}h1 small,h2 small,h3 small,h4 small{color:#7e7e7e;font-size:50%}h1,.h1{font-size:2.4rem;line-height:1.2}h2,.h2{font-size:2rem;line-height:1.3}h3,.h3{font-size:1.66667rem;line-height:1.3}h4,.h4,.lead{font-size:1.2rem;line-height:1.3}h5,.h5{font-size:1rem}h6,.h6{font-size:0.93333rem}header .subheadline{margin-top:0}.section-head,legend{border-bottom:1px solid #e0e0e0;margin-bottom:0.5rem;padding-bottom:0.5rem}dl{margin-bottom:1.5em}dt{font-weight:600}dd{margin-left:1.5em}.dl-horizontal{width:100%}.dl-horizontal dt{clear:left;float:left;text-align:right;width:30%}.dl-horizontal dd{margin-left:30%;padding-left:1em}.small{font-size:0.8em;line-height:1.3}.normal{font-size:1rem;font-weight:400}.screen-reader-text,.assistive-text{height:0;overflow:hidden;position:absolute;text-indent:200%;white-space:nowrap;width:0}blockquote{color:#7e7e7e;font-size:1.13333rem}blockquote cite{display:block;font-style:italic;margin-top:1em}blockquote cite:before{content:'\2014';padding-right:0.35em}.subheadline{color:#7e7e7e;font-weight:300}.lead{font-weight:400}.text-info{color:#2f7bbf}.text-important,.text-error{color:#bd2426}.text-success{color:#9bca3e}.text-warning{color:#f68b1f}.text-nonessential{color:#7e7e7e}.well{background-color:#f5f5f5;padding:1.5em;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.well.compact{padding:1em}p+p,p+ul,p+ol,p+dl,p+table,ul+p,ul+h2,ul+h3,ul+h4,ul+h5,ul+h6,ol+p,ol+h2,ol+h3,ol+h4,ol+h5,ol+h6{margin-top:1.5em}h1+p,p+h1,p+h2,p+h3,p+h4,p+h5,p+h6{margin-top:1.25em}h1+h2,h1+h3,h2+h3,h3+h4,h4+h5{margin-top:0.25em}h2+p{margin-top:1em}h1+h4,h1+h5,h1+h6,h2+h4,h2+h5,h2+h6,h3+h5,h3+h6,h3+p,h4+p,h5+p{margin-top:0.5em}.navigation ul{list-style:none;margin-left:0}.navigation a{color:#2f7bbf;text-decoration:none;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.navigation a:hover{color:#333}.sidebar blockquote{font-size:1.13333rem}.sidebar .title{color:#777}#main{min-height:400px}img,object{height:auto;max-width:100%}section,.section{margin-bottom:2.5rem;margin-top:2.5rem}section.compact,.section.compact{margin-bottom:1.5rem;margin-top:1.5rem}audio{display:none;height:0;width:0;overflow:hidden}video{overflow:hidden}video,audio[controls],audio.webshims-controls{display:inline-block;min-height:3rem;min-width:2.66667rem}video>*,audio>*{visibility:hidden}.no-swf video>*,.no-swf audio>*{visibility:inherit}.row{clear:both;display:block}.col-1{margin-left:0;max-width:100%;width:100%}.col-2{margin-left:0;max-width:100%;width:100%}.col-3{margin-left:0;max-width:100%;width:100%}.col-4{margin-left:0;max-width:100%;width:100%}.col-5{margin-left:0;max-width:100%;width:100%}.col-6{margin-left:0;max-width:100%;width:100%}.col-7{margin-left:0;max-width:100%;width:100%}.col-8{margin-left:0;max-width:100%;width:100%}.col-9{margin-left:0;max-width:100%;width:100%}.col-10{margin-left:0;max-width:100%;width:100%}.col-11{margin-left:0;max-width:100%;width:100%}.col-12{margin-left:0;max-width:100%;width:100%}.col-13{margin-left:0;max-width:100%;width:100%}.col-14{margin-left:0;max-width:100%;width:100%}.col-15{margin-left:0;max-width:100%;width:100%}.col-16{margin-left:0;max-width:100%;width:100%}.columns{display:block;list-style:none;padding:0}.columns img,.columns input,.columns select,.columns object,.columns textarea{max-width:100%}.columns>.column{float:left;padding-bottom:3rem}.columns,.columns>.column{width:100%}.width-third{width:33.3334%}.width-half{width:50%}.width-full,.mod-group{width:100%}.wrapper{margin-left:auto;margin-right:auto;width:90%}.primary-content{margin-bottom:2.66667em;margin-top:1.33333em}@media screen and (max-width: 49.1em){.tablet-only,.desktop-only{display:none !important}}@media screen and (min-width: 49.2em){.wrapper{width:47.2rem}.primary-content{float:left;margin:0 0 0 16.8rem;width:30.4rem}.sidebar{float:left;margin-left:-47.2rem;width:13.6rem}.reverse-sidebar .primary-content{margin-left:0}.reverse-sidebar .sidebar{margin-left:3.2rem}.primary-content:only-child{float:none;margin-left:auto;margin-right:auto}.columns>.column{padding-bottom:0}.columns.two>.column,.columns.cols-2>.column,.columns.four>.column,.columns.cols-4>.column{padding-left:0;padding-right:1.5rem;width:50%}.columns.two>.column:nth-child(even),.columns.cols-2>.column:nth-child(even),.columns.four>.column:nth-child(even),.columns.cols-4>.column:nth-child(even){padding-left:1.5rem;padding-right:0}.columns.two>.column:nth-child(odd),.columns.cols-2>.column:nth-child(odd),.columns.four>.column:nth-child(odd),.columns.cols-4>.column:nth-child(odd){clear:left}.columns.two>.column:nth-child(n+3),.columns.cols-2>.column:nth-child(n+3),.columns.four>.column:nth-child(n+3),.columns.cols-4>.column:nth-child(n+3){padding-top:3rem}.columns.three>.column,.columns.cols-3>.column{padding-left:2rem;width:33.3333333333333%}.columns.three>.column:first-child,.columns.three>.column:nth-child(3n+1),.columns.cols-3>.column:first-child,.columns.cols-3>.column:nth-child(3n+1){clear:left;padding-left:0;padding-right:2rem}.columns.three>.column:nth-child(3n+2),.columns.cols-3>.column:nth-child(3n+2){padding-left:1rem;padding-right:1rem}.columns.three>.column:nth-child(n+4),.columns.cols-3>.column:nth-child(n+4){padding-top:3rem}.columns.three>.column:nth-child(-n+3),.columns.cols-3>.column:nth-child(-n+3){padding-top:0}}@media screen and (min-width: 66em){.col-1{display:block;float:left;margin-left:48px;width:1rem}.col-2{display:block;float:left;margin-left:48px;width:5.2rem}.col-3{display:block;float:left;margin-left:48px;width:9.4rem}.col-4{display:block;float:left;margin-left:48px;width:13.6rem}.col-5{display:block;float:left;margin-left:48px;width:17.8rem}.col-6{display:block;float:left;margin-left:48px;width:22rem}.col-7{display:block;float:left;margin-left:48px;width:26.2rem}.col-8{display:block;float:left;margin-left:48px;width:30.4rem}.col-9{display:block;float:left;margin-left:48px;width:34.6rem}.col-10{display:block;float:left;margin-left:48px;width:38.8rem}.col-11{display:block;float:left;margin-left:48px;width:43rem}.col-12{display:block;float:left;margin-left:48px;width:47.2rem}.col-13{display:block;float:left;margin-left:48px;width:51.4rem}.col-14{display:block;float:left;margin-left:48px;width:55.6rem}.col-15{display:block;float:left;margin-left:48px;width:59.8rem}.col-16{display:block;float:left;margin-left:48px;width:64rem}[class*="col-"]:first-child{margin-left:0}.wrapper{width:64rem}.wrapper.wide{max-width:100%;width:72.4rem}.primary-content{float:left;margin-left:21rem;width:43rem}.sidebar{float:left;margin-left:-64rem;width:17.8rem}.wide .primary-content{width:51.4rem}.wide .sidebar{margin-left:-72.4rem}.columns>.column{padding-bottom:0}.columns.four>.column,.columns.cols-4>.column{padding-left:2.25rem;width:25%}.columns.four>.column:nth-child(odd),.columns.cols-4>.column:nth-child(odd){clear:none}.columns.four>.column:first-child,.columns.four>.column:nth-child(4n+1),.columns.cols-4>.column:first-child,.columns.cols-4>.column:nth-child(4n+1){clear:left;padding-left:0;padding-right:2.25rem}.columns.four>.column:nth-child(4n+2),.columns.cols-4>.column:nth-child(4n+2){padding-left:0.75rem;padding-right:1.5rem}.columns.four>.column:nth-child(4n+3),.columns.cols-4>.column:nth-child(4n+3){padding-left:1.5rem;padding-right:0.75rem}.columns.four>.column:nth-child(n+5),.columns.cols-4>.column:nth-child(n+5){padding-top:3rem}.columns.four>.column:nth-child(-n+4),.columns.cols-4>.column:nth-child(-n+4){padding-top:0}}.btn,button,input[type="button"],input[type="submit"]{background-color:transparent;border:1px solid #dedede;color:#333;font-size:0.93333rem;font-weight:400;line-height:1.2;margin:0;padding:0.6em 1.33333em 0.53333em;-webkit-user-select:none;-moz-user-select:none;user-select:none;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px;-webkit-transition:all 0.2s ease;-moz-transition:all 0.2s ease;-o-transition:all 0.2s ease;transition:all 0.2s ease}.btn:hover,.input-picker .picker-list td button.checked-value,button:hover,input[type="button"]:hover,input[type="submit"]:hover{background-color:rgba(0,0,0,0.05);border-color:#585858;color:#333}.btn:focus,button:focus,input[type="button"]:focus,input[type="submit"]:focus{color:inherit;outline:none;-webkit-box-shadow:inset 0 0 4px rgba(0,0,0,0.3);-moz-box-shadow:inset 0 0 4px rgba(0,0,0,0.3);box-shadow:inset 0 0 4px rgba(0,0,0,0.3)}.btn.active,.btn:active,button.active,button:active,input[type="button"].active,input[type="button"]:active,input[type="submit"].active,input[type="submit"]:active{background-color:rgba(0,0,0,0.05);border-color:#333;color:#1a1a1a}.btn::-moz-focus-inner,button::-moz-focus-inner,input[type="button"]::-moz-focus-inner,input[type="submit"]::-moz-focus-inner{padding:0;border:0}.btn .caret,button .caret,input[type="button"] .caret,input[type="submit"] .caret{border-top-color:inherit;margin-left:0.25em;margin-top:0.18333em}.btn-large{padding:1rem 1.66667rem}.btn-cta,.btn-cta-alt{padding:1rem 3rem}.btn-std,.btn-primary,.btn-std-alt,.btn-primary-alt{background-color:#2f7bbf;border-color:transparent;color:#fff}.btn-std:hover,.btn-primary:hover,.btn-std-alt:hover,.btn-primary-alt:hover{background-color:#62a1d8;border-color:#2f7bbf;color:#fff}.btn-std.active,.btn-std:focus,.btn-std:active,.btn-primary.active,.btn-primary:focus,.btn-primary:active,.btn-std-alt.active,.btn-std-alt:focus,.btn-std-alt:active,.btn-primary-alt.active,.btn-primary-alt:focus,.btn-primary-alt:active{background-color:#62a1d8;border-color:#163959;color:#fff}.btn-std-alt,.btn-primary-alt{background-color:transparent;border-color:#2f7bbf;color:#2f7bbf}.btn-cta,.btn-success,.btn-accept,.btn-accept-alt,.btn-cancel,.btn-cancel-alt,.btn-delete,.btn-cta-alt,.btn-success-alt,.btn-accept-alt{background-color:#9bca3e;border-color:transparent;color:#fff}.btn-cta:hover,.btn-success:hover,.btn-accept:hover,.btn-accept-alt:hover,.btn-cancel:hover,.btn-cancel-alt:hover,.btn-delete:hover,.btn-cta-alt:hover,.btn-success-alt:hover,.btn-accept-alt:hover{background-color:#bada7a;border-color:#9bca3e;color:#fff}.btn-cta.active,.btn-cta:focus,.btn-cta:active,.btn-success.active,.active.btn-accept,.active.btn-accept-alt,.active.btn-cancel,.active.btn-cancel-alt,.active.btn-delete,.btn-success:focus,.btn-accept:focus,.btn-accept-alt:focus,.btn-cancel:focus,.btn-cancel-alt:focus,.btn-delete:focus,.btn-success:active,.btn-accept:active,.btn-accept-alt:active,.btn-cancel:active,.btn-cancel-alt:active,.btn-delete:active,.btn-cta-alt.active,.btn-cta-alt:focus,.btn-cta-alt:active,.btn-success-alt.active,.active.btn-accept-alt,.btn-success-alt:focus,.btn-accept-alt:focus,.btn-success-alt:active,.btn-accept-alt:active{background-color:#bada7a;border-color:#516b1d;color:#fff}.btn-accept,.btn-accept-alt,.btn-cancel,.btn-cancel-alt,.btn-delete{font-family:FontAwesome;font-weight:normal;font-style:normal;text-decoration:inherit;-webkit-font-smoothing:antialiased;min-height:2.425em;overflow:hidden;padding-left:1.13333em;padding-right:1.13333em;position:relative;text-align:left;text-indent:-9999px;width:0;white-space:nowrap}.btn-accept:after,.btn-accept-alt:after,.btn-cancel:after,.btn-cancel-alt:after,.btn-delete:after{content:'\f00c';display:block;font-size:1.75em;height:100%;left:0;line-height:0;position:absolute;speak:none;text-align:center;text-indent:0;top:50%;width:100%;-webkit-transition-delay:0.2s;-moz-transition-delay:0.2s;-o-transition-delay:0.2s;transition-delay:0.2s}.btn-accept:before,.btn-accept-alt:before,.btn-cancel:before,.btn-cancel-alt:before,.btn-delete:before,.btn-accept:after,.btn-accept-alt:after,.btn-cancel:after,.btn-cancel-alt:after,.btn-delete:after{-webkit-transition:opacity 0.2s ease;-moz-transition:opacity 0.2s ease;-o-transition:opacity 0.2s ease;transition:opacity 0.2s ease}.btn-cta-alt,.btn-success-alt,.btn-accept-alt{background-color:transparent;border-color:#9bca3e;color:#9bca3e}.btn-secondary,.btn-delete{background-color:#ededed;border-color:transparent;color:#7e7e7e}.btn-secondary:hover,.btn-delete:hover{background-color:#ededed;border-color:#7e7e7e;color:#333}.btn-secondary.active,.active.btn-delete,.btn-secondary:focus,.btn-delete:focus,.btn-secondary:active,.btn-delete:active{background-color:#ededed;border-color:#585858;color:#0d0d0d}.btn-danger,.btn-cancel,.btn-cancel-alt,.btn-important,.btn-error,.btn-danger-alt,.btn-cancel-alt,.btn-important-alt,.btn-error-alt{background-color:#bd2426;border-color:transparent;color:#fff}.btn-danger:hover,.btn-cancel:hover,.btn-cancel-alt:hover,.btn-important:hover,.btn-error:hover,.btn-danger-alt:hover,.btn-cancel-alt:hover,.btn-important-alt:hover,.btn-error-alt:hover{background-color:#de5052;border-color:#bd2426;color:#fff}.btn-danger.active,.active.btn-cancel,.active.btn-cancel-alt,.btn-danger:focus,.btn-cancel:focus,.btn-cancel-alt:focus,.btn-danger:active,.btn-cancel:active,.btn-cancel-alt:active,.btn-important.active,.btn-important:focus,.btn-important:active,.btn-error.active,.btn-error:focus,.btn-error:active,.btn-danger-alt.active,.active.btn-cancel-alt,.btn-danger-alt:focus,.btn-cancel-alt:focus,.btn-danger-alt:active,.btn-cancel-alt:active,.btn-important-alt.active,.btn-important-alt:focus,.btn-important-alt:active,.btn-error-alt.active,.btn-error-alt:focus,.btn-error-alt:active{background-color:#de5052;border-color:#521010;color:#fff}.btn-danger-alt,.btn-cancel-alt,.btn-important-alt,.btn-error-alt{background-color:transparent;border-color:#bd2426;color:#bd2426}.btn-warning,.btn-warning-alt{background-color:#f68b1f;border-color:transparent;color:#fff}.btn-warning:hover,.btn-warning-alt:hover{background-color:#f9b169;border-color:#f68b1f;color:#fff}.btn-warning.active,.btn-warning:focus,.btn-warning:active,.btn-warning-alt.active,.btn-warning-alt:focus,.btn-warning-alt:active{background-color:#f9b169;border-color:#904b06;color:#fff}.btn-warning-alt{background-color:transparent;border-color:#f68b1f;color:#f68b1f}.btn-link{background-color:transparent;border-color:transparent;color:#2f7bbf}.btn-cancel:after,.btn-cancel-alt:after{content:'\f00d'}.btn-cancel-alt{border-color:#dedede;color:#dedede}.btn-delete:after{content:'\f014'}.btn.disabled,.btn.loading,button[disabled],input.btn[disabled]{cursor:default;background-color:#ededed;border-color:transparent;color:#a1a1a1;filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=50);opacity:0.5}.btn.disabled:hover,.btn.loading:hover,.input-picker .picker-list td button.loading.checked-value,.input-picker .picker-list td button.disabled.checked-value,.btn.disabled:focus,.btn.loading:focus,.btn.disabled:active,.btn.loading:active,button[disabled]:hover,button[disabled]:focus,button[disabled]:active,input.btn[disabled]:hover,input.btn[disabled]:focus,input.btn[disabled]:active{background-color:#ededed;border-color:transparent;color:#a1a1a1}.btn.loading{filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=80);opacity:0.8;overflow:hidden;position:relative}.btn.loading:before{background:transparent url('../images/spinner.gif') no-repeat center;content:'';display:block;height:100%;left:0;position:absolute;top:0;width:100%;filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=100);opacity:1;-webkit-transform:scale(1, 1);-moz-transform:scale(1, 1);-ms-transform:scale(1, 1);-o-transform:scale(1, 1);transform:scale(1, 1);-webkit-animation:fadeInZoom 0.2s ease-in-out;-moz-animation:fadeInZoom 0.2s ease-in-out;-ms-animation:fadeInZoom 0.2s ease-in-out;-o-animation:fadeInZoom 0.2s ease-in-out;animation:fadeInZoom 0.2s ease-in-out}.btn.loading,.btn.loading:after,.btn.loading:hover,.input-picker .picker-list td button.loading.checked-value{color:#ededed}.btn.loading:after{filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=0);opacity:0}@-webkit-keyframes fadeInZoom{0%{opacity:0;-webkit-transform:scale(0)}100%{opacity:1;-webkit-transform:scale(1)}}@-moz-keyframes fadeInZoom{0%{opacity:0;-moz-transform:scale(0)}100%{opacity:1;-moz-transform:scale(1)}}@-ms-keyframes fadeInZoom{0%{opacity:0;-ms-transform:scale(0)}100%{opacity:1;-ms-transform:scale(1)}}@keyframes fadeInZoom{0%{topacity:0;ransform:scale(0)}100%{opacity:1;transform:scale(1)}}input[type="button"].btn,input[type="submit"].btn{height:auto;padding:0.53333em 1.33333em 0.6em}.btn-single-line{max-width:100%;overflow:hidden;padding-bottom:0.6em;position:relative;text-overflow:ellipsis;}.btn-single-line.btn-wicons{padding-left:2em;padding-right:2em}.btn-single-line .icon,.btn-single-line .caret{line-height:1.5;margin-top:-0.6em;position:absolute;top:50%}.btn-single-line .caret{margin-left:0;margin-top:0;right:0.8em}.btn-single-line .icon:first-child{left:0.6em}.btn-single-line .icon:last-child{right:0.6em}.btn+.btn,.btn+.ui-item{margin-left:0.4rem}.btn-block{display:block;margin-bottom:0.4rem;text-align:center}.btn-block+.btn-block{margin-left:auto}.btn a{color:#333}.btn-std a,.btn-primary a{color:#fff}.btn-error a,.btn-danger a,.btn-cancel a,.btn-cancel-alt a,.btn-important a{color:#fff}.btn-cta a,.btn-success a,.btn-accept a,.btn-accept-alt a,.btn-cancel a,.btn-cancel-alt a,.btn-delete a{color:#fff}legend{display:block;margin-bottom:1.5rem;width:100%}input,select,textarea,.switch{background:#fff;border:1px solid #b1b1b1;color:#333;font-family:"Open Sans",Helvetica,Arial,sans-serif;font-size:0.86667em;line-height:1.24;margin:0 0 0.75em;max-width:100%;outline:none;padding:0.45em 0.75em;vertical-align:middle;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;-webkit-transition:all 0.2s ease;-moz-transition:all 0.2s ease;-o-transition:all 0.2s ease;transition:all 0.2s ease;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}input:hover,select:hover,textarea:hover,.switch:hover{border-color:#989898}input:focus,select:focus,textarea:focus,.switch:focus{border-color:#2f7bbf;outline:none;-webkit-box-shadow:0 0 8px rgba(47,123,191,0.5);-moz-box-shadow:0 0 8px rgba(47,123,191,0.5);box-shadow:0 0 8px rgba(47,123,191,0.5)}input.readonly,input.disabled,input[disabled],input[readonly],select.readonly,select.disabled,select[disabled],select[readonly],textarea.readonly,textarea.disabled,textarea[disabled],textarea[readonly],.switch.readonly,.switch.disabled,.switch[disabled],.switch[readonly]{background-color:#f7f7f7;border-color:#cbcbcb;color:#7e7e7e;cursor:not-allowed}select{position:relative;-webkit-appearance:none;-moz-appearance:none;appearance:none}input.ui-state-error,input.ui-state-invalid,input.user-error{border-color:#bd2426}input,select{height:2.26667rem}select[size],select[multiple]{height:auto}input[type="radio"],input[type="checkbox"]{height:16px;line-height:normal;margin:1px 0 0;padding:0.45em;position:relative;width:16px;-webkit-appearance:none;-moz-appearance:none;appearance:none}input[type="radio"]:before,input[type="checkbox"]:before{background-color:transparent;color:transparent;content:'';position:absolute;-webkit-transition:all 0.15s ease-out;-moz-transition:all 0.15s ease-out;-o-transition:all 0.15s ease-out;transition:all 0.15s ease-out}input[type="radio"].ui-state-valid,input[type="radio"].user-success,input[type="checkbox"].ui-state-valid,input[type="checkbox"].user-success{border-color:#b1b1b1}input[type="radio"],input[type="radio"]:before{-webkit-border-radius:50%;-moz-border-radius:50%;-ms-border-radius:50%;-o-border-radius:50%;border-radius:50%}input[type="radio"]:before{height:20%;left:40%;top:40%;width:20%}input[type="radio"]:checked:before{background-color:#333;height:60%;left:20%;top:20%;width:60%}input[type="checkbox"]:before{content:'\f00c';font-family:FontAwesome;font-size:1.25em;left:-0.06667em;top:-0.2em}input[type="checkbox"]:checked:before{color:#333}label{display:block;font-size:0.86667rem;margin-bottom:0.38333em}.radio,.checkbox{min-height:1rem;padding-left:2em}.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-2em;margin-top:0.26667em}.radio.inline,.checkbox.inline{display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;margin-bottom:0;padding-top:0.13333em;vertical-align:middle}.radio.inline+.inline,.checkbox.inline+.inline{margin-left:0.4rem}.input-mini{width:4rem}.input-small{width:9.4rem}.input-medium{width:17.8rem}.input-large{width:26.2rem}.input-xlarge{width:34.6rem}.input-xxlarge{width:43rem}.file{border:1px solid #b1b1b1;margin-bottom:1em;position:relative;padding:0;width:24rem;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.file input{border:0;margin-bottom:0;padding:0;width:10em;filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=0);opacity:0}.file:before{background-color:#2f7bbf;color:#fff;content:'Choose File';height:100%;left:0;padding:0.53333rem 1.33333rem;position:absolute;top:0}.switch,.proxy{background-color:#fff;border:1px solid #b1b1b1;color:#fff;cursor:pointer;font-size:0;height:2.26667rem;overflow:hidden;margin:0;padding:0;position:relative;width:5.334rem}.switch input,.proxy input{position:absolute;-webkit-user-select:none;-moz-user-select:none;user-select:none;-webkit-appearance:checkbox-container;-moz-appearance:checkbox-container;appearance:checkbox-container}.switch input,.switch input.user-success,.proxy input,.proxy input.user-success{background:transparent;border-color:transparent}.switch input:checked:before,.proxy input:checked:before{content:''}.switch input:focus,.proxy input:focus{outline:none;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.switch:after,.switch:before{background-color:#9bca3e;color:#fff;content:'On';font-size:0.86667rem;line-height:1.5;height:100%;left:0;padding:0.45rem 0;position:absolute;text-align:center;top:0;width:51%}.switch:before{background-color:#b1b1b1;content:'Off';left:auto;right:0;width:50%}.switch .knob{background:#f7f7f7;border:1px solid #b1b1b1;border-bottom:none;border-top:none;display:block;font-size:0.86667rem;height:100%;left:-1px;top:0;width:2.667rem;z-index:2;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px;-webkit-transition:all 0.15s linear;-moz-transition:all 0.15s linear;-o-transition:all 0.15s linear;transition:all 0.15s linear;position:relative}.switch .knob:before,.switch .knob:after{border:4px solid transparent;border-left-color:inherit;content:'';display:block;height:0;left:50%;margin-left:2px;margin-top:-3px;position:absolute;top:50%;width:0}.switch .knob:before{border-left-color:transparent;border-right-color:inherit;margin-left:-10px}.switch input:checked+.knob{left:50%}.proxy{background:transparent;border:0;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;height:34px;width:55px}.proxy .cloud{border:1px solid transparent;display:block;height:100%;left:0;position:absolute;top:0;width:100%}.proxy .cloud:before,.proxy .cloud:after{background:transparent url('../images/cloudflare-sprite-small.png') 0 -120px no-repeat;content:'';display:block;height:100%;left:0;opacity:1;position:absolute;top:0;width:100%;-webkit-transition:opacity,0.15s ease;-moz-transition:opacity,0.15s ease;-o-transition:opacity,0.15s ease;transition:opacity,0.15s ease}.proxy .cloud:after{opacity:0;background-position:0 -188px}.proxy input:checked+.cloud:before{opacity:0}.proxy input:checked+.cloud:after{opacity:1}.proxy input:focus+.cloud{border-color:#dedede}.control-group{padding:0.75em 0;position:relative;width:100%}.control-group input,.control-group select{margin-bottom:0}.control-group.info input,.control-group.info select,.control-group.info .select2-container,.control-group.info .select2-choice,.control-group.info .select2-choice div{border-color:#2f7bbf;color:#333}.control-group.info label,.control-group.info .control-label{color:#2f7bbf}.control-group.info .help-inline,.control-group.info .help-block{color:#2f7bbf}.control-group.error input,.control-group.error select,.control-group.error .select2-container,.control-group.error .select2-choice,.control-group.error .select2-choice div{border-color:#bd2426;color:#521010}.control-group.error label,.control-group.error .control-label{color:#bd2426}.control-group.error .help-inline,.control-group.error .help-block{color:#bd2426}.control-group.success input,.control-group.success select,.control-group.success .select2-container,.control-group.success .select2-choice,.control-group.success .select2-choice div{border-color:#9bca3e;color:#333}.control-group.success label,.control-group.success .control-label{color:#516b1d}.control-group.success .help-inline,.control-group.success .help-block{color:#516b1d}.control-group.warning input,.control-group.warning select,.control-group.warning .select2-container,.control-group.warning .select2-choice,.control-group.warning .select2-choice div{border-color:#f68b1f;color:#904b06}.control-group.warning label,.control-group.warning .control-label{color:#f68b1f}.control-group.warning .help-inline,.control-group.warning .help-block{color:#f68b1f}.controls input,.controls select,.controls textarea{max-width:100%}.controls .radio:only-child,.controls .checkbox:only-child{margin-bottom:0}.controls label,.control-label label{line-height:1.3}.input-stacked input,.input-stacked select,.input-stacked textarea,.input-stacked .select2-container{display:block;margin-bottom:0}.input-stacked input ~ input,.input-stacked input ~ select,.input-stacked input ~ .select2-container,.input-stacked select ~ input,.input-stacked select ~ select,.input-stacked select ~ .select2-container,.input-stacked textarea ~ input,.input-stacked textarea ~ select,.input-stacked textarea ~ .select2-container,.input-stacked .select2-container ~ input,.input-stacked .select2-container ~ select,.input-stacked .select2-container ~ .select2-container{margin-top:0.4rem}.input-prepend,.input-append{font-size:0;margin:0 0 1rem;vertical-align:middle;white-space:nowrap;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.input-prepend input,.input-prepend select,.input-append input,.input-append select{font-size:0.86667rem;margin:0;position:relative;vertical-align:top;-webkit-border-radius:0 2px 2px 0;-moz-border-radius:0 2px 2px 0;-ms-border-radius:0 2px 2px 0;-o-border-radius:0 2px 2px 0;border-radius:0 2px 2px 0}.input-prepend .btn,.input-prepend .add-on,.input-append .btn,.input-append .add-on{border-color:#b1b1b1;font-size:0.86667rem;line-height:1.24;height:2.26667rem;margin:0 -1px 0 0;min-width:1.06667em;position:relative;text-align:center;width:auto;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;-webkit-border-radius:0;-moz-border-radius:0;-ms-border-radius:0;-o-border-radius:0;border-radius:0}.input-prepend .btn:first-child,.input-prepend .add-on:first-child,.input-append .btn:first-child,.input-append .add-on:first-child{-webkit-border-radius:2px 0 0 2px;-moz-border-radius:2px 0 0 2px;-ms-border-radius:2px 0 0 2px;-o-border-radius:2px 0 0 2px;border-radius:2px 0 0 2px}.input-prepend .btn:last-child,.input-prepend .add-on:last-child,.input-append .btn:last-child,.input-append .add-on:last-child{-webkit-border-radius:0 2px 2px 0;-moz-border-radius:0 2px 2px 0;-ms-border-radius:0 2px 2px 0;-o-border-radius:0 2px 2px 0;border-radius:0 2px 2px 0}.input-prepend .add-on,.input-append .add-on{background-color:#dedede;border:1px solid;color:#7e7e7e;padding:0.53333rem 0.66667rem}.input-prepend input:hover,.input-prepend input:active,.input-prepend input:focus,.input-prepend select:hover,.input-prepend select:active,.input-prepend select:focus,.input-prepend .add-on:hover,.input-prepend .add-on:active,.input-prepend .add-on:focus,.input-append input:hover,.input-append input:active,.input-append input:focus,.input-append select:hover,.input-append select:active,.input-append select:focus,.input-append .add-on:hover,.input-append .add-on:active,.input-append .add-on:focus{z-index:5}.input-append input,.input-append select{-webkit-border-radius:2px 0 0 2px;-moz-border-radius:2px 0 0 2px;-ms-border-radius:2px 0 0 2px;-o-border-radius:2px 0 0 2px;border-radius:2px 0 0 2px}.input-append .btn,.input-append .add-on{margin-left:-1px;margin-right:0}.input-prepend.input-append input,.input-prepend.input-append select{-webkit-border-radius:0;-moz-border-radius:0;-ms-border-radius:0;-o-border-radius:0;border-radius:0}.input-prepend.input-append .btn,.input-prepend.input-append .add-on{margin-left:-1px;margin-right:0}.input-prepend.input-append .btn:first-child,.input-prepend.input-append .add-on:first-child{margin-left:0;margin-right:-1px}.form-stacked .control-group>input[name]:only-of-type,.form-stacked .control-group>select:only-of-type,.form-stacked .control-group>.select2-container:only-of-type{display:block;width:100%}.form-stacked input[type="checkbox"],.form-stacked input[type="button"],.form-stacked input[type="submit"]{display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;width:auto}.form-inline .btn,.form-inline button,.form-inline label,.form-inline input,.form-inline select,.form-inline .help-inline{display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;margin-bottom:0;margin-left:0.4rem;vertical-align:middle}.form-inline .btn:first-child,.form-inline button:first-child,.form-inline label:first-child,.form-inline input:first-child,.form-inline select:first-child,.form-inline .help-inline:first-child{margin-left:0}.form-inline input[type="radio"],.form-inline input[type="checkbox"]{float:none;margin:0 0.2em 0 0}.form-inline .radio,.form-inline .checkbox{padding-left:0}.ui-search{font-size:1rem;position:relative}.ui-search input{padding-left:2.5em}.ui-search .icon-search,.ui-search .clear-icon{font-size:1.08333em;line-height:1.3;padding:0.45em 0.75em;position:absolute}.ui-search .icon-search{color:#4d4d4d;left:0;position:absolute;top:0}.ui-search .clear-icon{color:#e4e4e4;cursor:pointer;display:none;top:0;right:0}.ui-search .clear-icon:hover{color:#989898}.ui-search .clear-icon:active{color:#7e7e7e}.help-inline,.help-block{font-size:0.86667rem}.help-inline:empty,.help-block:empty{display:none}.help-inline{display:inline;padding:0 0.5em}.help-block{display:block;margin:0 0 1em}.input-assist{font-size:0.8rem;line-height:2.26667rem;position:absolute;right:0.75em;top:0;-webkit-user-select:none;-moz-user-select:none;user-select:none}input ~ .alert,select ~ .alert,form .alert{margin-top:0.5em}.input-alert{position:relative}.input-alert:after{border:10px solid transparent;border-bottom-color:inherit;content:'';display:block;height:0;left:50%;margin-left:-10px;position:absolute;top:-20px;width:0;-webkit-filter:drop-shadow(0 -1px 0px rgba(0,0,0,0.6));-moz-filter:drop-shadow(0 -1px 0px rgba(0,0,0,0.6));filter:drop-shadow(0 -1px 0px rgba(0,0,0,0.6))}.input-alert.alert-error:after{border-bottom-color:#de5052}.input-alert.alert-success:after{border-bottom-color:#bada7a}.input-alert.alert-warning:after{border-bottom-color:#f9b169}.input-alert.alert-info:after{border-bottom-color:#62a1d8}.flexbox .input-prepend,.flexbox .input-append{display:-webkit-box;display:-moz-box;display:-ms-box;display:box}.flexbox .input-prepend input,.flexbox .input-prepend select,.flexbox .input-append input,.flexbox .input-append select{display:block;-webkit-box-flex:1;-moz-box-flex:1;-ms-box-flex:1;box-flex:1}.flexbox .input-prepend .btn,.flexbox .input-prepend .add-on,.flexbox .input-append .btn,.flexbox .input-append .add-on{display:block;-webkit-box-flex:0;-moz-box-flex:0;-ms-box-flex:0;box-flex:0}.ws-range,.ws-range *,.placeholder-box,.placeholder-text,.input-datetime-local,.input-buttons,.input-buttons *,.details-open-indicator,.ws-input-seperator,progress span.progress-value{margin:0;padding:0;border:none;width:auto;background:transparent none}output{position:relative}.placeholder-box{position:relative;display:inline-block;zoom:1}.polyfill-important .placeholder-box{position:relative !important;display:inline-block !important;margin:0 !important;padding:0 !important;width:auto !important;height:auto !important}.placeholder-box-input{vertical-align:bottom}.placeholder-box-left{float:left}.placeholder-box-right{float:right}.placeholder-text{position:absolute;display:none;top:0;left:0;overflow:hidden;color:#999;line-height:1;cursor:text}.polyfill-important .placeholder-text{margin:0 !important;padding-right:0 !important;padding-bottom:0 !important;display:none !important}.placeholder-visible .placeholder-text,.placeholder-text.placeholder-visible{display:inline-block}.placeholder-box-input .placeholder-text{white-space:nowrap}.placeholder-visible{color:#999}.placeholder-focused.placeholder-visible{color:#ccc}.polyfill-important .placeholder-visible .placeholder-text,.polyfill-important .placeholder-text.placeholder-visible{display:inline-block !important}.has-input-buttons{display:inline-block}.polyfill-important .has-input-buttons{display:inline-block !important}.input-buttons,.step-controls,.ws-popover-opener{zoom:1;overflow:hidden;display:inline-block;font-size:0;vertical-align:middle;margin-left:-20px}.step-controls,.ws-popover-opener{position:relative;float:left;margin:0;height:19px;width:15px}.ws-popover-opener{cursor:pointer;overflow:visible;margin:0;position:relative;width:20px;zoom:1}.ws-popover-opener:hover{background:none;border-color:transparent;color:#989898}.ws-popover-opener:before{content:'\f073';font-family:FontAwesome;font-size:15px}.ws-popover-opener span{display:none}.polyfill-important .input-buttons{display:inline-block !important;padding:0 !important;vertical-align:middle !important}.input-buttons.input-button-size-1.month-input-buttons,.input-buttons.input-button-size-1.date-input-buttons{margin-left:-24px}.input-buttons.input-button-size-2{margin-left:-39px}.input-buttons.ws-disabled{opacity:0.95}.input-buttons.ws-disabled *,.input-buttons.ws-readonly *{cursor:default}.step-controls span{border:4px solid transparent;position:absolute;display:inline-block;left:3px;overflow:hidden;margin:0 !important;padding:0 !important;cursor:pointer;font-size:0;line-height:0;height:0;width:0}.step-controls span:hover{border-bottom-color:#989898}.step-controls span.mousepress-ui{border-bottom-color:#2f7bbf}.ws-disabled .step-controls span{border-bottom-color:#cbcbcb}.polyfill-important .step-controls span{display:inline-block !important;margin:0 !important;padding:0 !important;font-size:0 !important}.step-controls span.step-up{border-bottom-color:#b1b1b1;top:0}.step-controls span.step-down{border-top-color:#b1b1b1;top:12px}.ws-input{letter-spacing:-0.31em;word-spacing:-0.43em}.ws-input>*{text-align:center;letter-spacing:normal;word-spacing:normal}.ws-input .ws-input-seperator{vertical-align:middle;width:2%;overflow:hidden}.ws-input+.input-buttons{margin-left:2px}.ws-input input,.ws-input .ws-input-seperator{text-align:center;display:inline-block}.polyfill-important .ws-input input,.polyfill-important .ws-input .ws-input-seperator{display:inline-block !important}.ws-date .mm,.ws-date .dd{width:23.5%}.no-boxsizing .ws-date .mm,.no-boxsizing .ws-date .dd{width:16%}.ws-date .yy{width:48%}.no-boxsizing .ws-date .yy{width:40%}.ws-month .mm,.ws-month .yy{width:47.9%}.no-boxsizing .ws-month .mm,.no-boxsizing .ws-month .yy{width:41%}.ws-range{position:relative;display:inline-block;vertical-align:middle;margin:0;zoom:1;height:1px;width:155px;border-radius:1px;cursor:pointer;font-size:0;line-height:0;top:12px}.ws-range:focus{outline:none}.polyfill-important .ws-range{display:inline-block !important;padding:0 !important;font-size:0 !important}.ws-range .ws-range-thumb{background-color:#7e7e7e;border:1px solid #333;top:-1px;position:absolute;display:block;z-index:4;overflow:hidden;margin:0 0 0 -7px;height:14px;width:14px;cursor:pointer;outline:none;font-size:0;line-height:0;-webkit-border-radius:50%;-moz-border-radius:50%;-ms-border-radius:50%;-o-border-radius:50%;border-radius:50%}.ws-range .ws-range-thumb:hover{border-color:#7e7e7e}.ws-range .ws-range-thumb:active{background-color:#333}.ws-range.ws-focus .ws-range-thumb{background-position:-20px 1px}.ws-range.ws-active .ws-range-thumb{background-position:-37px 1px}.ws-range[aria-disabled="true"],.ws-range[aria-readonly="true"]{cursor:default;opacity:0.95}.ws-range[aria-disabled="true"] .ws-range-thumb,.ws-range[aria-readonly="true"] .ws-range-thumb{cursor:default}.ws-range[aria-disabled="true"] .ws-range-thumb{background-position:-54px 1px}.ws-range .ws-range-rail{border-top:1px solid #b1b1b1;position:absolute;display:block;top:-10px;left:5px;right:5px;bottom:0;margin:0;zoom:1}.ws-range .ws-range-min{position:absolute !important;display:block;padding:0 !important;top:-10px;height:1px;left:0;z-index:1;overflow:hidden;background:#f68b1f}.ws-range .ws-range-ticks{overflow:hidden;position:absolute;bottom:0px;left:0;height:4px;width:1px;margin:0 0 0 -1.5px;font-size:0;line-height:0;text-indent:-999px;background:#ccc}.ws-range.vertical-range .ws-range-thumb:hover,.ws-range.vertical-range.ws-focus .ws-range-thumb{background-position:0 -34px}.ws-range.vertical-range.ws-active .ws-range-thumb{background-position:0 -17px}.ws-range.vertical-range[aria-disabled="true"] .ws-range-thumb{background-position:0 0}.ws-range.vertical-range .ws-range-min{top:auto;bottom:1px;left:0;width:1px;height:0}.ws-range.vertical-range .ws-range-rail{top:5px;left:0;right:0;bottom:5px}.ws-range.vertical-range .ws-range-ticks{bottom:auto;left:auto;right:0;height:1px;width:4px}.ws-popover{display:block;visibility:hidden;overflow:hidden;position:absolute;top:0;left:0;padding:0 6px;margin:0 0 0 -6px;z-index:1600;min-width:90px;transition:visibility 400ms ease-in-out}.ws-popover button{display:inline-block;overflow:visible;position:relative;margin:0;border:0;padding:0;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:none;appearance:none;box-sizing:content-box;font-family:arial, sans-serif;background:transparent;cursor:pointer}.ws-popover button::-moz-focus-inner{border:0;padding:0}.ws-popover button[disabled]{cursor:default;color:#888}.ws-popover.ws-po-visible{visibility:visible}.ws-po-outerbox{position:relative;opacity:0;padding:11px 0 4px;-webkit-transition:all 250ms ease-in-out;-moz-transition:all 250ms ease-in-out;-o-transition:all 250ms ease-in-out;transition:all 250ms ease-in-out}.ws-popover.ws-po-visible .ws-po-outerbox{opacity:1}.ws-po-box{border:1px solid #dedede;background:#fff;padding:0.5rem 1rem;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.ws-po-arrow{position:absolute;top:4px;left:20px;display:block;width:0;height:0;border-left:9px solid transparent;border-right:9px solid transparent;border-bottom:7px solid #ccc;border-top:none;zoom:1;font-size:0}html .ws-po-arrow{border-left-color:transparent;border-right-color:transparent}html .ws-po-arrow .ws-po-arrowbox{border-left-color:transparent;border-right-color:transparent}.polyfill-important .ws-po-arrow{border-left-color:transparent !important;border-right-color:transparent !important}.polyfill-important .ws-po-arrow .ws-po-arrowbox{border-left-color:transparent !important;border-right-color:transparent !important}* html .ws-po-arrow{display:none}.ws-po-arrow .ws-po-arrowbox{position:relative;top:1px;left:-9px;display:block;width:0;height:0;border-left:9px solid transparent;border-right:9px solid transparent;border-bottom:7px solid #fefefe;border-top:none;z-index:999999999}.validity-alert{display:inline-block;font-size:0.86667rem;margin:0;padding:0;z-index:1000000000}.validity-alert .ws-po-outerbox{padding:6px 0 0}.validity-alert .ws-po-box{background-color:#de5052;border:1px solid #521010;color:#fff}.validity-alert .ws-po-arrow{border-bottom-color:#521010;top:0}.validity-alert .ws-po-arrow .ws-po-arrowbox{border-bottom-color:#de5052}.input-picker{outline:none;text-align:center;font-family:sans-serif;width:300px}.input-picker.ws-size-2{width:538px}.input-picker.ws-size-3{width:796px}.input-picker abbr[title]{cursor:help}.input-picker li,.input-picker button{font-size:13px;line-height:16px;color:#000;transition:all 400ms}.input-picker .ws-focus,.input-picker :focus{outline:1px solid #2f7bbf}.input-picker .ws-po-box{position:relative;padding:0;box-shadow:0 0 6px rgba(0,0,0,0.1);-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.input-picker .ws-prev,.input-picker .ws-next{position:absolute;top:0;padding:0;width:40px;height:40px;right:0;z-index:1}.input-picker .ws-prev:after,.input-picker .ws-next:after{border:6px solid transparent;border-left-color:#333;content:'';left:50%;margin-left:-3px;margin-top:-6px;position:absolute;top:50%}.input-picker .ws-prev span,.input-picker .ws-next span{display:none}.input-picker .ws-picker-body{position:relative;padding:40px 0 0;zoom:1}.input-picker .ws-prev{left:0;right:auto}.input-picker .ws-prev:after{border-left-color:transparent;border-right-color:#333;margin-left:-10px}.input-picker .ws-button-row{position:relative;margin:10px 0 0;border-top:1px solid #dedede;text-align:left;z-index:2}.input-picker .ws-button-row button{padding:10px}.input-picker .ws-button-row button.ws-empty{float:right}.input-picker[data-currentview="setMonthList"] .ws-picker-header select{max-width:95%}.input-picker[data-currentview="setDayList"] .ws-picker-header select{max-width:40%}.input-picker[data-currentview="setDayList"] .ws-picker-header select.month-select{max-width:55%}.input-picker .ws-picker-header{position:absolute;top:-30px;right:0;left:0;margin:0 40px}.input-picker .ws-picker-header button{display:inline-block;width:100%;margin:0;padding:4px 0;font-weight:700}.input-picker .ws-picker-header button:hover{text-decoration:underline}.input-picker .ws-picker-header button[disabled]:hover{text-decoration:none}.input-picker .picker-grid{position:relative;zoom:1;overflow:hidden}.input-picker.ws-size-1 .picker-list{float:none;width:auto}.input-picker .picker-list{position:relative;zoom:1;width:238px;float:left;margin:0 10px}.input-picker .picker-list tr{border:0}.input-picker .picker-list th,.input-picker .picker-list td{padding:3px 5px}.input-picker .picker-list.day-list td{padding:2px 1px}.input-picker .picker-list td button{display:block;width:100%}.input-picker .picker-list td button.othermonth{color:#7e7e7e}.input-picker .picker-list table{width:100%;border:0 none;border-collapse:collapse}.input-picker .picker-list th,.input-picker .picker-list td.week-cell{font-size:13px;line-height:1.1em;padding-bottom:3px;text-transform:uppercase;font-weight:700}.input-picker .picker-list th,.input-picker .picker-list td{width:14.2856%}.input-picker .ws-options{margin:10px 0 0;border-top:1px solid #dedede;padding:10px 0 0;text-align:left}.input-picker .ws-options h5{margin:0 0 5px;padding:0;font-size:14px;font-weight:bold}.input-picker .ws-options ul,.input-picker .ws-options li{padding:0;margin:0;list-style:none}.input-picker .ws-options button{display:block;padding:2px 0;width:100%;text-align:left}.input-picker .ws-options button.ws-focus,.input-picker .ws-options button:focus,.input-picker .ws-options button:hover{text-decoration:underline}.input-picker .ws-options button[disabled],.input-picker .ws-options button[disabled].ws-focus,.input-picker .ws-options button[disabled]:focus,.input-picker .ws-options button[disabled]:hover{color:#7e7e7e;text-decoration:none}datalist{display:none}.datalist-polyfill{position:absolute !important;font-size:100%}.datalist-polyfill .datalist-box{position:relative;max-height:200px;overflow:hidden;overflow-x:hidden !important;overflow-y:auto}.datalist-polyfill .ws-po-box{padding:0}.datalist-polyfill ul,.datalist-polyfill li{font-size:100%;list-style:none !important}.datalist-polyfill ul{position:static !important;overflow:hidden;margin:0;padding:0;height:auto !important;background-color:#fff;color:#333}.datalist-polyfill li{margin:0;padding:0.25em 0.5em;overflow:hidden;white-space:nowrap;cursor:default;zoom:1;overflow:hidden;text-overflow:ellipsis;background-color:#fff;transition:background-color 250ms}.datalist-polyfill mark{font-weight:normal;font-style:normal}.datalist-polyfill .option-value{display:inline-block;text-overflow:ellipsis;max-width:100%;color:#333;float:left;transition:color 250ms}.datalist-polyfill .option-label{display:none;max-width:100%;float:right;font-size:90%;color:#7e7e7e;text-overflow:ellipsis;vertical-align:bottom;margin-top:0.15em;margin-left:10px;text-align:right;transition:color 400ms}.datalist-polyfill .has-option-label .option-label{display:inline-block}.datalist-polyfill .hidden-item{display:none !important}.datalist-polyfill .active-item{background-color:#2f7bbf;cursor:default}.datalist-polyfill .active-item .option-value{color:#fff}.datalist-polyfill .active-item .option-label{color:#dedede}progress{border:0;display:inline-block;height:12px;position:relative;width:auto;-webkit-appearance:none;-moz-appearance:none;appearance:none}progress[data-position]{background:#f5f5f5;border:none;vertical-align:-0.2em}progress>*{display:none}progress span.progress-value{background:#2f7bbf;display:block !important;top:0;left:0;bottom:0;height:100%;position:absolute}progress[aria-valuenow] span.progress-value{background:#2f7bbf}progress:indeterminate{background-color:#f68b1f;color:#f68b1f}@media screen and (min-width: 49.2em){form .columns.two>.column,form .columns.cols-2>.column,form .columns.four>.column,form .columns.cols-4>.column{padding-left:0;padding-right:0.45714em;width:50%}form .columns.two>.column:nth-child(even),form .columns.cols-2>.column:nth-child(even),form .columns.four>.column:nth-child(even),form .columns.cols-4>.column:nth-child(even){padding-left:0.45714em;padding-right:0}.form-horizontal legend{padding-left:30%;width:70%}.form-horizontal .control-label{color:#333;float:left;font-size:0.86667rem;margin-bottom:0;padding:0.6em 1.5rem 0.46667em 0;text-align:right;width:30%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.form-horizontal .control-label label{display:block}.form-horizontal .checkbox-label{padding-top:0}.form-horizontal .controls{margin-left:30%;width:70%}.form-horizontal .controls .columns{overflow:hidden}}code,pre{background-color:#e8e8e8;border:1px solid #dbdbdb;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}pre{color:#4d4d4d;display:block;font-family:monaco, courier, monospace;font-size:0.86667rem;margin:2rem 0;overflow:auto;padding:0.5rem;width:100%;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}code{color:#bd2426;margin-left:0.13333em;margin-right:0.13333em;padding:0 0.4em;vertical-align:baseline;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.pagination .btn{background-color:transparent;border-color:transparent;color:#2f7bbf;padding-left:1em;padding-right:1em}.pagination .btn:hover,.pagination .input-picker .picker-list td button.checked-value,.input-picker .picker-list td .pagination button.checked-value{background-color:transparent;border-color:transparent;color:#f68b1f}.pagination .btn.active,.pagination .btn:active{background-color:transparent;border-color:transparent;color:#a1a1a1}.pagination .btn.inactive,.pagination .btn.disabled,.pagination .btn.loading,.pagination .btn[disabled]{background-color:transparent;border-color:transparent;color:#a1a1a1}.pagination .btn+.pagination-set,.pagination .pagination-set+.btn{margin-left:0.4rem}.pagination li+li{margin-left:0.4rem}.pagination .num-break{color:#a1a1a1;cursor:default;padding-left:0;padding-right:0}.pagination,.pagination ol,.pagination ul,.pagination li{list-style:none;margin:0;padding:0}.pagination ol,.pagination ul,.pagination li{display:inline;font-size:0}.caret{border:0.33333em solid transparent;border-top-color:inherit;content:"";height:0;width:0;vertical-align:top;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.dropup,.dropdown{position:relative}.dropup .caret,.dropdown .caret{margin-top:0.25em;margin-left:0.13333em}.dropup .caret{border-top-color:transparent;border-bottom-color:inherit}.dropdown-toggle:active,.open .dropdown-toggle{outline:0}.dropdown-menu{background-color:#fff;border:1px solid #dedede;display:none;float:left;left:0;list-style:none;opacity:0;margin:5px 0 0;min-width:10.66667rem;padding:0.33333rem 0;position:absolute;top:102%;z-index:1040;*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px;-webkit-box-shadow:0 3px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 3px 10px rgba(0,0,0,0.2);box-shadow:0 3px 10px rgba(0,0,0,0.2);background-clip:padding-box;-webkit-animation:menuTransition 0.15s ease-out;-moz-animation:menuTransition 0.15s ease-out;-ms-animation:menuTransition 0.15s ease-out;-o-animation:menuTransition 0.15s ease-out;animation:menuTransition 0.15s ease-out}.dropdown-menu:before{border:10px solid transparent;border-bottom-color:#fff;content:'';left:1rem;height:0;position:absolute;top:-20px;width:0}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu.pull-right:before{left:auto;right:1rem}.dropdown-menu .divider{background-color:#dedede;height:1px;margin:0.53333rem 0;overflow:hidden}.dropdown-menu li>a{clear:both;display:block;line-height:1.5;padding:0.2rem 1.06667rem;white-space:nowrap}.dropdown-menu li>a:hover,.dropdown-menu li>a:focus,.dropdown-menu li>a:active{color:#fff}.dropdown-menu li>a:hover{background-color:#2f7bbf}.dropdown-menu li>a:focus{background-color:#62a1d8}.dropdown-menu li>a:active{background-color:#c16508}.open .dropdown-menu{display:block;opacity:1}.menu-sidebar{list-style:none;margin:0;padding:0}.menu-sidebar li{background:#fff}.menu-sidebar li a{display:block}.menu-sidebar li.active>a,.menu-sidebar a:hover{background-color:#2f7bbf;color:#fff}.menu-sidebar li.active>a:after,.menu-sidebar a:hover:after{border-left-color:#fff}.menu-sidebar>li{background-clip:border-box}.menu-sidebar>li+li{margin-top:1px}.menu-sidebar>li:first-child,.menu-sidebar>li:first-child>a{-moz-border-radius-topleft:2px;-webkit-border-top-left-radius:2px;border-top-left-radius:2px;-moz-border-radius-topright:2px;-webkit-border-top-right-radius:2px;border-top-right-radius:2px}.menu-sidebar>li:last-child,.menu-sidebar>li:last-child>a{-moz-border-radius-bottomleft:2px;-webkit-border-bottom-left-radius:2px;border-bottom-left-radius:2px;-moz-border-radius-bottomright:2px;-webkit-border-bottom-right-radius:2px;border-bottom-right-radius:2px}.menu-sidebar>li>a{padding:1rem 2rem 1rem 1rem;position:relative}.menu-sidebar>li>a:after{border:0.4rem solid transparent;border-left-color:#2f7bbf;content:'';display:block;height:0;margin-top:-0.4rem;position:absolute;top:50%;right:0.75rem;width:0;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.menu-sidebar>li.current-menu-ancestor>a:after{border-left-color:transparent;border-top-color:#fff;margin-top:-0.3rem}.menu-sidebar+.menu-sidebar{margin-top:1.5rem}.sub-menu{overflow:hidden;margin:0;padding:0}.sub-menu li:last-child{padding-bottom:0.75rem}.sub-menu a{padding:0.25rem;padding-left:2rem}.sub-menu li.active>a,.sub-menu a:hover{background-color:#3988ce}.sub-menu .sub-menu li:last-child{padding-bottom:0}.sub-menu .sub-menu a{padding-left:3rem}.sub-menu .sub-menu a:before{content:'\21B3';padding-right:0.26667rem}.js .menu-sidebar .sub-menu{display:none}.js .menu-sidebar .active>.sub-menu{display:block}.logo{background:transparent url('../images/cloudflare-sprite-small.png') 0 0 no-repeat;display:inline-block;overflow:hidden;text-indent:-9999em;height:60px;width:240px}.close{color:#7e7e7e;cursor:pointer;display:inline-block;font-size:2.3rem;float:right;height:1.5rem;line-height:0.6;overflow:hidden;position:relative;text-indent:200%;width:1.5rem;-webkit-transition:all 0.2s ease;-moz-transition:all 0.2s ease;-o-transition:all 0.2s ease;transition:all 0.2s ease}.close:hover{color:#656565}.close:before{content:'\00D7';left:0;height:100%;position:absolute;text-align:center;text-indent:0;top:0;width:100%}.cf-proxied,.cf-unproxied,.cf-unproxiable{background:transparent url('../images/cloudflare-sprite-small.png') no-repeat;cursor:pointer;overflow:hidden;padding:0;text-indent:200%;height:34px;width:55px;background-position:0 -188px;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.cf-unproxied{background-position:0 -120px}.cf-unproxiable{background-position:0 -154px}@media (-webkit-min-device-pixel-ratio: 1.3), (-o-min-device-pixel-ratio: 2.6 / 2), (min--moz-device-pixel-ratio: 1.3), (min-device-pixel-ratio: 1.3), (min-resolution: 1.3dppx){.logo,.cf-proxied,.cf-unproxied{background-image:url('../images/cloudflare-sprite-small.png')}.logo{background-position:0 -122px;-webkit-background-size:100% auto;-moz-background-size:100% auto;-o-background-size:100% auto;background-size:100% auto}.cf-proxied{background-position:0 -380px;-webkit-background-size:55px,auto;-moz-background-size:55px,auto;-o-background-size:55px,auto;background-size:55px,auto}.cf-unproxied{background-position:0 -244px;-webkit-background-size:55px,auto;-moz-background-size:55px,auto;-o-background-size:55px,auto;background-size:55px,auto}.cf-unproxiable{background-position:0 -312px;-webkit-background-size:55px,auto;-moz-background-size:55px,auto;-o-background-size:55px,auto;background-size:55px,auto}}.header{background-color:#333;border-bottom:1px solid #1a1a1a;color:#fff;height:60px;margin:0;padding:0;position:relative;top:0;z-index:1000}.header-main .btn{font-size:0.93333rem}.logo-header{display:block;margin:0 auto;background-position:0 -60px}.header-navigation{display:none;font-size:0.93333rem}.header-navigation li{position:relative}.header-navigation li.btn{padding:0}.header-navigation a{color:#fff;display:block;padding:0.4rem 1rem}.header-navigation a:hover{background-color:#4d4d4d}.icon-menu{cursor:pointer;height:60px;left:0;overflow:hidden;position:absolute;text-indent:200%;top:0;width:60px;white-space:nowrap}.icon-menu:before{background:transparent url('../images/cloudflare-sprite-small.png') 0 -222px no-repeat;content:'';display:block;height:100%;left:50%;margin-left:-10px;margin-top:-10px;position:absolute;top:50%;width:100%}@media screen and (-webkit-min-device-pixel-ratio: 1.3), (-o-min-device-pixel-ratio: 2.6 / 2), (min--moz-device-pixel-ratio: 1.3), (min-device-pixel-ratio: 1.3), (min-resolution: 1.3dppx){.logo-header{}}@media screen and (min-width: 49.2em){.mobile-navigation{display:none}.header{background-color:#fff;border-bottom:0;border-top:3px solid #f68b1f;color:#333;height:auto;margin-bottom:2.66667rem}.header a:hover,.header li.active a{color:#333}.header .menu li:hover .sub-menu{display:block}.header-main{padding:0.5rem 0}.logo-header{background-position:0 0}.header-navigation{display:block;text-align:center;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.header-navigation li{display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.header-navigation a{color:#2f7bbf;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.header-navigation a:hover{background-color:transparent}.header-navigation .btn{color:#fff;padding:0.6em 1.33333em 0.53333em}.header-navigation .sub-menu{background-color:#fff;border:1px solid #dedede;border-top:0;display:none;left:0;min-width:100%;position:absolute;top:100%;width:13.33333rem;-webkit-border-radius:0 0 3px 3px;-moz-border-radius:0 0 3px 3px;-ms-border-radius:0 0 3px 3px;-o-border-radius:0 0 3px 3px;border-radius:0 0 3px 3px}.header-navigation .sub-menu li{display:block}.header-navigation .sub-menu li:last-child{padding-bottom:0}.header-navigation .sub-menu a{display:block;padding:0.53333em 0.8em;-webkit-border-radius:0;-moz-border-radius:0;-ms-border-radius:0;-o-border-radius:0;border-radius:0}.header-navigation .sub-menu a:hover{color:#fff}}@media screen and (min-width: 66em){.header{text-align:left}.logo-header{float:left}.header-navigation{float:right;line-height:60px;text-align:left}.header-navigation li{line-height:1.5;vertical-align:middle}}.footer{background-color:#fff;margin-top:1.33333rem;padding-bottom:2.33333rem;padding-top:2.33333rem}.footer-nav{font-size:0.86667rem}.footer-column{float:left;list-style:none;margin-left:1%;margin-right:1%;width:48%}.footer-column+.footer-column{margin-bottom:1rem}.footer-language-select{margin:0 auto 1.33333rem;width:13.6rem}.footer-language-select select,.footer-language-select .select2-container{width:100%}@media screen and (min-width: 49.2em){.footer{margin-top:2.66667rem}.footer-column{float:left;margin-left:0.5%;margin-right:0.5%;width:19%}.footer-column+.footer-column{margin-bottom:0}}@media screen and (min-width: 66em){.footer-language-select{float:left;margin-bottom:0}.footer-nav{float:left;width:47.2rem}}.modal-backdrop{background-color:#000;bottom:0;left:0;position:fixed;top:0;right:0;z-index:10;-webkit-transition:opacity 0.2s linear;-moz-transition:opacity 0.2s linear;-o-transition:opacity 0.2s linear;transition:opacity 0.2s linear}.modal-backdrop.fade{filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=0);opacity:0}.modal-backdrop,.modal-backdrop.fade.in{filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=70);opacity:0.7}.modal-container{display:block;min-height:100%;position:relative;width:100%}.modal{background-color:#fff;left:50%;margin:0 0 0 -15rem;max-width:95%;outline:none;position:fixed;top:10%;width:30rem;z-index:20;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px;-webkit-box-shadow:0 1px 15px rgba(0,0,0,0.75);-moz-box-shadow:0 1px 15px rgba(0,0,0,0.75);box-shadow:0 1px 15px rgba(0,0,0,0.75);-webkit-transition:opacity 0.25s linear;-moz-transition:opacity 0.25s linear;-o-transition:opacity 0.25s linear;transition:opacity 0.25s linear}.modal.fade{filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=0);opacity:0}.modal.fade.in,.modal.visible{filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=100);opacity:1}.modal-header,.modal-content,.modal-footer,.modal-body-section{padding:1.5rem;position:relative}.modal-header{padding:1.5rem}.modal-header ~ .modal-body,.modal-header ~ .modal-content,.modal-header ~ .modal-body-section,.modal-header ~ .modal-body .modal-content{padding-top:0}.modal-body+.modal-body,.modal-body+.modal-body-section,.modal-body-section+.modal-body,.modal-body-section+.modal-body-section{border-top:1px solid #f5f5f5;padding-top:1.5rem}.modal-close{position:absolute;right:1.5rem;top:1.5rem}.modal-title{font-weight:400}.modal-body{overflow-y:auto;max-height:100%}.modal-section{background-color:#ebebeb;border:1px solid #dedede;border-left:0;border-right:0}.modal-section .control-group{padding:1rem 1.5rem}.modal-section .control-group+.control-group{border-top:1px solid #dedede}.modal-footer{background-color:#f5f5f5}.footer-simple{background-color:transparent}.modal-actions{float:right}.modal-nonessential{line-height:2.2rem;vertical-align:middle}.modal-confirm .modal-footer{padding-top:0}body.modal-active{overflow:hidden}#overlays{height:0;left:0;overflow-y:auto;position:absolute;top:0;width:100%;z-index:1500}table{background-color:#fff;border-collapse:collapse;border-spacing:0;max-width:100%}thead{background-color:#dedede}thead,thead a{color:#333}thead a{cursor:pointer;display:block;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}thead input,thead select,thead textarea{border-color:#989898}thead td:first-child,thead th:first-child{-webkit-border-radius:2px 0 0 0;-moz-border-radius:2px 0 0 0;-ms-border-radius:2px 0 0 0;-o-border-radius:2px 0 0 0;border-radius:2px 0 0 0}thead td:last-child,thead th:last-child{-webkit-border-radius:0 2px 0 0;-moz-border-radius:0 2px 0 0;-ms-border-radius:0 2px 0 0;-o-border-radius:0 2px 0 0;border-radius:0 2px 0 0}thead.inverse{background-color:#7e7e7e}thead.inverse,thead.inverse a,thead.inverse .sort-caret{color:#fff}thead .sortable:hover{background-color:#d2d2d2}thead .sortable:hover,thead .sortable:hover a{color:#1a1a1a}.sort-caret{border:4px solid transparent;content:"";display:inline-block;height:0;margin-left:0.5em;width:0;vertical-align:middle}.ascending .sort-caret{border-bottom-color:#333;margin-top:-4px}.descending .sort-caret{border-top-color:#333;margin-top:4px}.table,.table-container{width:100%}.table{margin-bottom:1.5rem}.table th,.table td{border-top:1px solid #d2d2d2;line-height:1.5;padding:0.86667rem;vertical-align:middle}.table th{font-weight:600}.table thead th{vertical-align:bottom}.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0}.table tbody+tbody{border-top:2px solid #d2d2d2}.table-condensed th,.table-condensed td{padding:0.43333rem 0.93333rem}.table-bordered{border:1px solid #d2d2d2;border-collapse:separate;*border-collapse:collapse;border-left:0;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.table-bordered th,.table-bordered td{border-left:1px solid #d2d2d2}.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child th,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child>th:first-child,.table-bordered tbody:first-child tr:first-child>td:first-child,.table-bordered tbody:first-child tr:first-child>th:first-child{-webkit-border-radius:2px 0 0 0;-moz-border-radius:2px 0 0 0;-ms-border-radius:2px 0 0 0;-o-border-radius:2px 0 0 0;border-radius:2px 0 0 0}.table-bordered thead:first-child tr:first-child>th:last-child,.table-bordered tbody:first-child tr:first-child>td:last-child,.table-bordered tbody:first-child tr:first-child>th:last-child{-webkit-border-radius:0 2px 0 0;-moz-border-radius:0 2px 0 0;-ms-border-radius:0 2px 0 0;-o-border-radius:0 2px 0 0;border-radius:0 2px 0 0}.table-bordered thead:last-child tr:last-child>th:first-child,.table-bordered tbody:last-child tr:last-child>td:first-child,.table-bordered tbody:last-child tr:last-child>th:first-child,.table-bordered tfoot:last-child tr:last-child>td:first-child,.table-bordered tfoot:last-child tr:last-child>th:first-child{-webkit-border-radius:0 0 0 2px;-moz-border-radius:0 0 0 2px;-ms-border-radius:0 0 0 2px;-o-border-radius:0 0 0 2px;border-radius:0 0 0 2px}.table-bordered thead:last-child tr:last-child>th:last-child,.table-bordered tbody:last-child tr:last-child>td:last-child,.table-bordered tbody:last-child tr:last-child>th:last-child,.table-bordered tfoot:last-child tr:last-child>td:last-child,.table-bordered tfoot:last-child tr:last-child>th:last-child{-webkit-border-radius:0 0 2px 0;-moz-border-radius:0 0 2px 0;-ms-border-radius:0 0 2px 0;-o-border-radius:0 0 2px 0;border-radius:0 0 2px 0}.table-bordered tfoot+tbody:last-child tr:last-child td:first-child{-webkit-border-radius:0;-moz-border-radius:0;-ms-border-radius:0;-o-border-radius:0;border-radius:0}.table-bordered tfoot+tbody:last-child tr:last-child td:last-child{-webkit-border-radius:0;-moz-border-radius:0;-ms-border-radius:0;-o-border-radius:0;border-radius:0}.table-bordered caption+thead tr:first-child th:first-child,.table-bordered caption+tbody tr:first-child td:first-child,.table-bordered colgroup+thead tr:first-child th:first-child,.table-bordered colgroup+tbody tr:first-child td:first-child{-webkit-border-radius:2px 0 0 0;-moz-border-radius:2px 0 0 0;-ms-border-radius:2px 0 0 0;-o-border-radius:2px 0 0 0;border-radius:2px 0 0 0}.table-bordered caption+thead tr:first-child th:last-child,.table-bordered caption+tbody tr:first-child td:last-child,.table-bordered colgroup+thead tr:first-child th:last-child,.table-bordered colgroup+tbody tr:first-child td:last-child{-webkit-border-radius:0 2px 0 0;-moz-border-radius:0 2px 0 0;-ms-border-radius:0 2px 0 0;-o-border-radius:0 2px 0 0;border-radius:0 2px 0 0}.table-striped tbody>tr:nth-child(even)>td,.table-striped tbody>tr:nth-child(even)>th{background-color:#fafafa}.table-hover tbody tr:hover>td,.table-hover tbody tr:hover>th{background-color:#f0f0f0}.table-bare td{border-top:0}.table tbody tr td.success,.table tbody tr.success>td{background-color:#e6f2d0}.table tbody tr td.error,.table tbody tr.error>td{background-color:#f3c1c2}.table tbody tr td.warning,.table tbody tr.warning>td{background-color:#fff5db}.table tbody tr td.info,.table tbody tr.info>td{background-color:#c8def1}tr.fade td{-webkit-animation:bgFadeOut 1.5s ease 1;-moz-animation:bgFadeOut 1.5s ease 1;-ms-animation:bgFadeOut 1.5s ease 1;-o-animation:bgFadeOut 1.5s ease 1;animation:bgFadeOut 1.5s ease 1}.table-hover td{-webkit-transition:background-color 0.2s ease;-moz-transition:background-color 0.2s ease;-o-transition:background-color 0.2s ease;transition:background-color 0.2s ease}.table-hover tbody tr td.success:hover,.table-hover tbody tr.success:hover>td{background-color:#dbecbc}.table-hover tbody tr td.error:hover,.table-hover tbody tr.error:hover>td{background-color:#efacad}.table-hover tbody tr td.warning:hover,.table-hover tbody tr.warning:hover>td{background-color:#ffeec2}.table-hover tbody tr td.info:hover,.table-hover tbody tr.info:hover>td{background-color:#b4d2ec}td.editable:hover{cursor:text;outline:1px dotted #a4a4a4;outline-offset:-5px}td.editor{padding:0}td.editor,td.editor.editable{background-color:#fff;outline:1px solid #a4a4a4;outline-offset:-5px}td.editor input,td.editor select{background:transparent;border:0;display:block;font-size:1em;height:100%;margin:0;padding:0.86667rem;width:100%}td.editor input:focus,td.editor select:focus{outline:none;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}td.select-cell,td.select-cell.editable.editor,td.select-cell.editor:hover,td.select2-cell,td.select2-cell.editable.editor,td.select2-cell.editor:hover,td.edit-always,td.edit-always.editable.editor,td.edit-always.editor:hover,td.proxy-cell,td.proxy-cell.editable.editor,td.proxy-cell.editor:hover,td.boolean-cell,td.boolean-cell.editable.editor,td.boolean-cell.editor:hover{outline:none}td.boolean-cell,td.select-row-cell,th.select-all-header-cell{text-align:center;width:1.13333em}.text-cell{max-width:25em;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.select2-cell .select2-container{width:100%}.icon-cell{text-align:center;width:1.6em}.icon-cell img{height:auto;max-width:100%}.icon-cell:first-child{padding-right:0}.icon-cell:last-child{padding-left:0}.table-top:empty,.table-content:empty,.table-pagination:empty,.table-meta:empty,.table-info:empty,.table-bottom:empty{display:none}.table-meta .pagination a,.table-meta .link-nav-list a{font-size:1rem;line-height:1.5}@media screen and (max-width: 49.2em){.table-content{max-width:100%;overflow-x:auto}}@media screen and (min-width: 49.2em){.table-pagination{float:left}.table-info{float:right}}.modunit{background-color:#fff;margin-top:1.5rem;margin-bottom:1.5rem;padding:0;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px;-webkit-transition:all 0.35s ease;-moz-transition:all 0.35s ease;-o-transition:all 0.35s ease;transition:all 0.35s ease}.modunit.needs-upgrade .mod-setting-control:after{background-color:#2f7bbf;content:'\0024';color:#fff;height:1.5rem;position:absolute;right:0;text-align:center;top:0;width:1.5rem;-webkit-border-radius:0 2px 0 2px;-moz-border-radius:0 2px 0 2px;-ms-border-radius:0 2px 0 2px;-o-border-radius:0 2px 0 2px;border-radius:0 2px 0 2px}.modunit.extended{margin-left:auto;margin-right:auto;width:98%}.modunit.highlighted{-webkit-box-shadow:0 0 20px rgba(0,0,0,0.15);-moz-box-shadow:0 0 20px rgba(0,0,0,0.15);box-shadow:0 0 20px rgba(0,0,0,0.15)}.modunit>*+*{border-top:1px solid #f5f5f5}.modunit>*:empty{display:none !important}.mod-content{padding:1.5rem}.mod-content+.mod-content{padding-top:0}.mod-content hr{margin:1.5rem 0}.mod-header{padding:1.5rem;position:relative}.mod-header:only-child{border-bottom:0}.mod-title{font-weight:400;margin-bottom:1rem}.mod-title small{padding-left:0.4rem;white-space:nowrap}.ancillary-info{color:#dedede;line-height:1;position:absolute;right:1.5rem;top:1.5rem}.ancillary-info i{cursor:pointer;font-size:1.25rem;margin:0;vertical-align:middle;-webkit-transition:color 150ms ease;-moz-transition:color 150ms ease;-o-transition:color 150ms ease;transition:color 150ms ease}.ancillary-info i:hover{color:#7e7e7e}.ancillary-info i+i{margin-left:0.26667rem}.mod-row{background-color:#fff;clear:both;font-size:0;padding:0.86667rem;position:relative;white-space:nowrap;width:100%}.striped .mod-row:nth-child(even),.mod-row.stripe{background-color:#f7f7f7}.mod-row.ui-toolbar{margin:0;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.mod-cell{display:block;font-size:1rem;overflow:visible;padding:0;position:relative;white-space:normal}.mod-cell .select2-container{width:100%}.mod-cell+.mod-cell,.mod-cell+.ui-group{padding-left:0.4rem}.mod-cell:first-child{padding-left:0}.mod-cell:only-child{width:100%}.input-row{-webkit-box-align:stretch;-moz-box-align:stretch;-ms-box-align:stretch;box-align:stretch}.input-row .mod-cell{width:100%}.input-row .mod-cell+.mod-cell,.input-row .mod-cell+.ui-group{padding-left:0;margin-top:0.5rem}.mod-cell,.cell-primary,.cell-actions{width:auto}.cell-icon{height:100%;text-align:center}.cell-primary input,.cell-primary select,.cell-primary textarea{width:100%}.cell-input input,.cell-input select,.cell-input textarea{margin-bottom:0}.simple-actions{text-align:right}.mod-table-adjustable .mod-cell,.mod-row-adjustable .mod-cell{width:100%}.mod-table-adjustable .mod-cell+.mod-cell,.mod-table-adjustable .mod-cell+.ui-group,.mod-row-adjustable .mod-cell+.mod-cell,.mod-row-adjustable .mod-cell+.ui-group{padding-left:0;margin-top:0.4rem}.mod-setting{display:table;width:100%}.mod-setting .mod-header,.mod-setting .mod-setting-control{display:table-cell;vertical-align:middle}.input-row,.mod-setting,.mod-setting-control,.mod-table-adjustable .mod-row{-webkit-box-orient:vertical;-moz-box-orient:vertical;-ms-box-orient:vertical;box-orient:vertical}.mod-setting-control{background-color:rgba(0,0,0,0.02);border-left:1px solid #f5f5f5;padding:2rem;position:relative;text-align:center;-moz-border-radius-topright:2px;-webkit-border-top-right-radius:2px;border-top-right-radius:2px}.mod-radio-group,.mod-checkbox-group{text-align:left;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.mod-radio-group label,.mod-checkbox-group label{font-size:1rem}.mod-radio-group label+label,.mod-checkbox-group label+label{margin-top:1em}.mod-radio-group input[type="radio"],.mod-checkbox-group input[type="checkbox"]{margin-top:0.4em}.mod-control-group.mod-setting-control{padding:0;text-align:left}.mod-control-group .ui-block{margin-bottom:0;padding:1rem 1.4rem 1.26667rem;width:100%}.mod-control-group .ui-block+.ui-block{border-top:1px solid #e8e8e8;margin-left:0}.mod-control-group label{font-weight:300}.mod-toolbar{color:#7e7e7e;overflow:hidden;position:relative}.mod-notification,.modunit .link-nav-list a{padding:1.11667rem 1.5rem;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.mod-notification{max-width:65%}.modunit .link-nav-list{float:right;height:100%;list-style:none;margin:0;padding:0;text-align:right}.modunit .link-nav-list li{display:inline}.modunit .link-nav-list li a{border-left:1px solid #f5f5f5}.mod-more-link,.modunit .link-nav-list .mod-more-link{padding-right:2.25em;position:relative}.mod-more-link:after,.modunit .link-nav-list .mod-more-link:after{content:'\f0da';font-family:FontAwesome;font-style:normal;font-weight:normal;margin-top:-0.6em;position:absolute;top:50%;right:1rem;-webkit-transition:all 0.2s ease;-moz-transition:all 0.2s ease;-o-transition:all 0.2s ease;transition:all 0.2s ease}.mod-more-link.active,.modunit .link-nav-list .mod-more-link.active{color:#f68b1f}.mod-more-link.active:after,.modunit .link-nav-list .mod-more-link.active:after{content:'\f0d7'}.cssanimations .mod-more-link.active:after,.cssanimations .modunit .link-nav-list .mod-more-link.active:after{content:'\f0da';-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.modunit-exception+.modunit-exception{border-top:1px solid #f5f5f5}.modunit.loading .mod-setting-control{pointer-events:none;-webkit-user-select:none;-moz-user-select:none;user-select:none}.modunit.loading .mod-setting-control:before{background:transparent url('../images/spinner.gif') no-repeat 1.5rem 1.5rem;content:'';cursor:wait;display:block;height:100%;left:0;position:absolute;top:0;width:100%;z-index:10;filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=80);opacity:0.8;-webkit-user-select:none;-moz-user-select:none;user-select:none}.modunit .table{margin-bottom:0}.modunit .table-top .ui-toolbar{padding-left:0.86667rem;padding-right:0.86667rem}.modunit .table-meta{border-top:1px solid #f5f5f5}.modunit .table-pagination{overflow-x:auto;max-width:100%;padding:0 0.86667rem;white-space:nowrap}.modunit .pagination .btn{border:0;padding:1.11667rem 0.5em;vertical-align:baseline;-webkit-border-radius:0;-moz-border-radius:0;-ms-border-radius:0;-o-border-radius:0;border-radius:0}.modunit .pagination .btn.disabled,.modunit .pagination .btn.loading{background-color:transparent}.modunit .pagination.centered{text-align:center}.modunit .pagination li+li{margin-left:0}.flexbox .mod-row,.flexbox .mod-toolbar,.flexbox .mod-toolbar-menu{display:-webkit-box;display:-moz-box;display:-ms-box;display:box}.flexbox .mod-row{-webkit-box-pack:start;-moz-box-pack:start;-ms-box-pack:start;box-pack:start;-webkit-box-align:center;-moz-box-align:center;-ms-box-align:center;box-align:center}.flexbox .mod-cell,.flexbox .mod-row>.ui-group{display:block;float:none;-webkit-box-flex:0;-moz-box-flex:0;-ms-box-flex:0;box-flex:0}.flexbox .cell-primary{-webkit-box-flex:10;-moz-box-flex:10;-ms-box-flex:10;box-flex:10}.flexbox .mod-setting{-webkit-box-align:stretch;-moz-box-align:stretch;-ms-box-align:stretch;box-align:stretch;-webkit-flex-align:stretch;-moz-flex-align:stretch;-ms-flex-align:stretch;flex-align:stretch}.flexbox .mod-setting .mod-header{-webkit-box-flex:1;-moz-box-flex:1;-ms-box-flex:1;box-flex:1}.flexbox .mod-setting,.flexbox .mod-setting .mod-setting-control{display:-webkit-box;display:-moz-box;display:-ms-box;display:box;-webkit-box-pack:center;-moz-box-pack:center;-ms-box-pack:center;box-pack:center}.flexbox .mod-setting .mod-header{display:block}.flexbox .mod-setting-control{-webkit-box-align:start;-moz-box-align:start;-ms-box-align:start;box-align:start;-webkit-flex-align:start;-moz-flex-align:start;-ms-flex-align:start;flex-align:start;-webkit-box-flex:0;-moz-box-flex:0;-ms-box-flex:0;box-flex:0}.flexbox .mod-toolbar .mod-notification,.flexbox .mod-toolbar .mod-toolbar-menu li{display:block}.flexbox .mod-toolbar .mod-notification{-webkit-box-align:start;-moz-box-align:start;-ms-box-align:start;box-align:start;-webkit-flex-align:start;-moz-flex-align:start;-ms-flex-align:start;flex-align:start;-webkit-box-flex:4;-moz-box-flex:4;-ms-box-flex:4;box-flex:4}.flexbox .mod-toolbar .mod-toolbar-menu{float:none;-webkit-box-flex:1;-moz-box-flex:1;-ms-box-flex:1;box-flex:1;-webkit-box-pack:end;-moz-box-pack:end;-ms-box-pack:end;box-pack:end}@media screen and (min-width: 49.2em){.modunit{margin-top:3rem;margin-bottom:3rem}.modunit .table-pagination{overflow-x:visible}.mod-cell{display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.cell-icon{width:3%}.cell-primary{line-height:2;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.cell-expanded{line-height:1.5;overflow:visible;white-space:normal}.cell-input{line-height:1.5;overflow:visible}.input-row{-webkit-box-align:center;-moz-box-align:center;-ms-box-align:center;box-align:center}.input-row .mod-cell{width:auto}.input-row .mod-cell+.mod-cell,.input-row .mod-cell+.ui-group{margin-top:0;padding-left:0.4rem}.mod-control{width:10rem}.singular-row .mod-cell{margin-left:0;width:100%}.mod-table-adjustable .mod-cell,.mod-row-adjustable .mod-cell{width:auto}.mod-table-adjustable .mod-cell+.mod-cell,.mod-table-adjustable .mod-cell+.ui-group,.mod-row-adjustable .mod-cell+.mod-cell,.mod-row-adjustable .mod-cell+.ui-group{padding-left:0.4rem;margin-top:0}.mod-table-adjustable .cell-primary,.mod-row-adjustable .cell-primary{width:69%}.mod-table-adjustable .cell-actions,.mod-row-adjustable .cell-actions{min-width:15.66667rem;width:28%}.mod-table-adjustable .input-row .cell-primary,.mod-row-adjustable.input-row .cell-primary{width:72%}.mod-setting-control{padding:1rem;width:40%}.modunit-exception+.modunit-exception{border-top:0}.flexbox .mod-row{-webkit-box-align:center;-moz-box-align:center;-ms-box-align:center;box-align:center}.flexbox .input-row,.flexbox .mod-setting,.flexbox .mod-setting-control,.flexbox .mod-table-adjustable .mod-row{-webkit-box-orient:horizontal;-moz-box-orient:horizontal;-ms-box-orient:horizontal;box-orient:horizontal}.flexbox .mod-setting-control{-webkit-box-align:center;-moz-box-align:center;-ms-box-align:center;box-align:center;-webkit-flex-align:center;-moz-flex-align:center;-ms-flex-align:center;flex-align:center}}@media screen and (min-width: 66em){.mod-setting-control{width:30%}}.ui-item{position:relative;max-width:100%;display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline}.ui-item+.ui-item,.ui-item+.btn{margin-left:0.4rem}.ui-item select,.ui-item .select2-container{width:100%}.ui-item input,.ui-item select{margin-bottom:0}.ui-block{display:block;margin-bottom:1em}.ui-group,.btn-group{display:-moz-inline-stack;display:inline-block;vertical-align:middle;*vertical-align:middle;zoom:1;*display:inline;font-size:0;position:relative;vertical-align:middle;white-space:nowrap}.ui-group+.ui-group,.ui-group+.btn-group,.ui-group+.ui-item,.btn-group+.ui-group,.btn-group+.btn-group,.btn-group+.ui-item{margin-left:0.4rem}.ui-group>.btn,.ui-group>.dropdown-menu,.ui-group>.popover,.ui-group>.select2-container,.ui-group>.ui-item,.btn-group>.btn,.btn-group>.dropdown-menu,.btn-group>.popover,.btn-group>.select2-container,.btn-group>.ui-item{font-size:0.93333rem;-webkit-user-select:none;-moz-user-select:none;user-select:none}.ui-item:empty,.ui-group:empty,.btn-group:empty{display:none}.ui-item>input,.ui-group>input,.btn-group>input{font-size:0.86667rem;margin-bottom:0}.ui-toolbar{display:block;font-size:0;margin-bottom:0.66667rem;margin-top:0.66667rem}.ui-toolbar .btn+.ui-item,.ui-toolbar .btn+.ui-group,.ui-toolbar .btn+.btn-group,.ui-toolbar .ui-group+.btn,.ui-toolbar .ui-group+.btn-group,.ui-toolbar .btn-group+.btn,.ui-toolbar .btn-group+.ui-item,.ui-toolbar .ui-item+.btn,.ui-toolbar .ui-item+.btn-group,.ui-toolbar .ui-item+.ui-group{margin-left:0.4rem}.ui-toolbar>.btn,.ui-toolbar>.dropdown-menu,.ui-toolbar>.popover,.ui-toolbar>.select2-container,.ui-toolbar>.ui-item{font-size:0.93333rem;margin-bottom:0}.ui-toolbar>select,.ui-toolbar>input{font-size:13px;margin-bottom:0}.ui-toolbar>i{height:2.26667rem}.btn-group>.btn{position:relative;-webkit-border-radius:0;-moz-border-radius:0;-ms-border-radius:0;-o-border-radius:0;border-radius:0}.btn-group>.btn:hover,.input-picker .picker-list td .btn-group>button.checked-value,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:5}.btn-group>.btn:first-child{-moz-border-radius-topleft:2px;-webkit-border-top-left-radius:2px;border-top-left-radius:2px;-moz-border-radius-bottomleft:2px;-webkit-border-bottom-left-radius:2px;border-bottom-left-radius:2px}.btn-group>.btn:last-child{-moz-border-radius-topright:2px;-webkit-border-top-right-radius:2px;border-top-right-radius:2px;-moz-border-radius-bottomright:2px;-webkit-border-bottom-right-radius:2px;border-bottom-right-radius:2px}.btn-group>.btn+.btn,.btn-group>.btn+.btn-group{margin-left:-1px}.btn-group .btn-group+.btn{margin-left:-1px}.btn-group-vertical>.btn{display:block;float:none;margin:0.4rem auto;max-width:100%}.vert-arrows{width:24px;position:relative}.vert-arrows:before,.vert-arrows:after{border:4px solid transparent;border-bottom-color:#333;content:'';display:block;height:0;left:50%;margin-left:-5px;margin-top:-9px;position:absolute;top:50%;width:0}.vert-arrows:before{border-bottom-color:transparent;border-top-color:#333;margin-top:3px}.horz-arrows{width:24px;position:relative}.horz-arrows:before,.horz-arrows:after{border:4px solid transparent;border-left-color:#333;content:'';display:block;height:0;left:50%;margin-left:2px;margin-top:-3px;position:absolute;top:50%;width:0}.horz-arrows:before{border-left-color:transparent;border-right-color:#333;margin-left:-10px}/* - * Font Awesome 3.0.2 - * the iconic font designed for use with Twitter Bootstrap - * ------------------------------------------------------- - * The full suite of pictographic icons, examples, and documentation - * can be found at: http://fortawesome.github.com/Font-Awesome/ - * - * License - * ------------------------------------------------------- - * - The Font Awesome font is licensed under the SIL Open Font License - http://scripts.sil.org/OFL - * - Font Awesome CSS, LESS, and SASS files are licensed under the MIT License - - * http://opensource.org/licenses/mit-license.html - * - The Font Awesome pictograms are licensed under the CC BY 3.0 License - http://creativecommons.org/licenses/by/3.0/ - * - Attribution is no longer required in Font Awesome 3.0, but much appreciated: - * "Font Awesome by Dave Gandy - http://fortawesome.github.com/Font-Awesome" - * - * Contact - * ------------------------------------------------------- - * Email: dave@davegandy.com - * Twitter: http://twitter.com/fortaweso_me - * Work: Lead Product Designer @ http://kyruus.com - */@font-face{ - font-family:'FontAwesome'; - src:url('../fonts/fontawesome-cloudflare.eot?v=3.0.1'); - src:url('../fonts/fontawesome-cloudflare.eot?#iefix&v=3.0.1') format("embedded-opentype"),url('../fonts/fontawesome-cloudflare.woff?v=3.0.1') format("woff"),url('../fonts/fontawesome-cloudflare.ttf?v=3.0.1') format("truetype"); - font-weight:normal; - font-style:normal -} -.icon{font-family:FontAwesome;font-weight:normal;font-style:normal;text-decoration:inherit;-webkit-font-smoothing:antialiased;display:inline;width:auto;height:auto;line-height:normal;vertical-align:baseline;background-image:none;background-position:0% 0%;background-repeat:repeat;margin-top:0}.icon:before{text-decoration:inherit;display:inline-block;speak:none}.icon-white,.dropdown-menu>li>a:hover>[class^="icon-"],.dropdown-menu>li>a:hover>[class*=" icon-"],.dropdown-menu>.active>a>[class^="icon-"],.dropdown-menu>.active>a>[class*=" icon-"],.dropdown-submenu:hover>a>[class^="icon-"],.dropdown-submenu:hover>a>[class*=" icon-"]{background-image:none}a .icon{display:inline-block}.icon-large:before{vertical-align:-10%;font-size:1.3333333333333334em}.btn .icon,.nav .icon{display:inline}.btn .icon:first-child,.nav .icon:first-child{padding-right:0.25em}.btn .icon:last-child,.nav .icon:last-child{padding-left:0.25em}.btn .icon.icon-large,.nav .icon.icon-large{line-height:.9em}.btn .icon.icon-spin,.nav .icon.icon-spin{display:inline-block}.tabs .icon,.tabs .icon.icon-large{line-height:.9em}li .icon,.nav li .icon{display:inline-block;width:1.25em;text-align:center}li .icon.icon-large,.nav li .icon.icon-large{width:1.5625em}ul.icons{list-style-type:none;text-indent:-.75em}ul.icons li .icon{width:.75em}.icon-muted{color:#fafafa}.icon-border{border:solid 1px #fafafa;padding:.2em .25em .15em;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.icon-2x{font-size:2em}.icon-2x.icon-border{border-width:2px;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.icon-3x{font-size:3em}.icon-3x.icon-border{border-width:3px;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.icon-4x{font-size:4em}.icon-4x.icon-border{border-width:4px;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}.icon.pull-left{margin-right:.3em}.icon.pull-right{margin-left:.3em}.btn .icon.pull-left.icon-2x,.btn .icon.pull-right.icon-2x{margin-top:.18em}.btn .icon.icon-spin.icon-large{line-height:.8em}.btn.btn-small .icon.pull-left.icon-2x,.btn.btn-small .icon.pull-right.icon-2x{margin-top:.25em}.btn.btn-large .icon{margin-top:0}.btn.btn-large .icon.pull-left.icon-2x,.btn.btn-large .icon.pull-right.icon-2x{margin-top:.05em}.btn.btn-large .icon.pull-left.icon-2x{margin-right:.2em}.btn.btn-large .icon.pull-right.icon-2x{margin-left:.2em}.icon-spin{display:inline-block;-moz-animation:spin 2s infinite linear;-o-animation:spin 2s infinite linear;-webkit-animation:spin 2s infinite linear;animation:spin 2s infinite linear}@-moz-keyframes spin{0%{-moz-transform:rotate(0deg)}100%{-moz-transform:rotate(359deg)}}@-webkit-keyframes spin{0%{-webkit-transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg)}}@-o-keyframes spin{0%{-o-transform:rotate(0deg)}100%{-o-transform:rotate(359deg)}}@-ms-keyframes spin{0%{-ms-transform:rotate(0deg)}100%{-ms-transform:rotate(359deg)}}@keyframes spin{0%{transform:rotate(0deg)}100%{transform:rotate(359deg)}}@-moz-document url-prefix(){.icon-spin{height:.9em}.btn .icon-spin{height:auto}.icon-spin.icon-large{height:1.25em}.btn .icon-spin.icon-large{height:.75em}}.icon-glass:before{content:"\f000"}.icon-music:before{content:"\f001"}.icon-search:before{content:"\f002"}.icon-envelope:before{content:"\f003"}.icon-heart:before{content:"\f004"}.icon-star:before{content:"\f005"}.icon-star-empty:before{content:"\f006"}.icon-user:before{content:"\f007"}.icon-film:before{content:"\f008"}.icon-th-large:before{content:"\f009"}.icon-th:before{content:"\f00a"}.icon-th-list:before{content:"\f00b"}.icon-ok:before{content:"\f00c"}.icon-remove:before{content:"\f00d"}.icon-zoom-in:before{content:"\f00e"}.icon-zoom-out:before{content:"\f010"}.icon-off:before{content:"\f011"}.icon-signal:before{content:"\f012"}.icon-cog:before{content:"\f013"}.icon-trash:before{content:"\f014"}.icon-home:before{content:"\f015"}.icon-file:before{content:"\f016"}.icon-time:before{content:"\f017"}.icon-road:before{content:"\f018"}.icon-download-alt:before{content:"\f019"}.icon-download:before{content:"\f01a"}.icon-upload:before{content:"\f01b"}.icon-inbox:before{content:"\f01c"}.icon-play-circle:before{content:"\f01d"}.icon-repeat:before{content:"\f01e"}.icon-refresh:before{content:"\f021"}.icon-list-alt:before{content:"\f022"}.icon-lock:before{content:"\f023"}.icon-flag:before{content:"\f024"}.icon-headphones:before{content:"\f025"}.icon-volume-off:before{content:"\f026"}.icon-volume-down:before{content:"\f027"}.icon-volume-up:before{content:"\f028"}.icon-qrcode:before{content:"\f029"}.icon-barcode:before{content:"\f02a"}.icon-tag:before{content:"\f02b"}.icon-tags:before{content:"\f02c"}.icon-book:before{content:"\f02d"}.icon-bookmark:before{content:"\f02e"}.icon-print:before{content:"\f02f"}.icon-camera:before{content:"\f030"}.icon-font:before{content:"\f031"}.icon-bold:before{content:"\f032"}.icon-italic:before{content:"\f033"}.icon-text-height:before{content:"\f034"}.icon-text-width:before{content:"\f035"}.icon-align-left:before{content:"\f036"}.icon-align-center:before{content:"\f037"}.icon-align-right:before{content:"\f038"}.icon-align-justify:before{content:"\f039"}.icon-list:before{content:"\f03a"}.icon-indent-left:before{content:"\f03b"}.icon-indent-right:before{content:"\f03c"}.icon-facetime-video:before{content:"\f03d"}.icon-picture:before{content:"\f03e"}.icon-pencil:before{content:"\f040"}.icon-map-marker:before{content:"\f041"}.icon-adjust:before{content:"\f042"}.icon-tint:before{content:"\f043"}.icon-edit:before{content:"\f044"}.icon-share:before{content:"\f045"}.icon-check:before{content:"\f046"}.icon-move:before{content:"\f047"}.icon-step-backward:before{content:"\f048"}.icon-fast-backward:before{content:"\f049"}.icon-backward:before{content:"\f04a"}.icon-play:before{content:"\f04b"}.icon-pause:before{content:"\f04c"}.icon-stop:before{content:"\f04d"}.icon-forward:before{content:"\f04e"}.icon-fast-forward:before{content:"\f050"}.icon-step-forward:before{content:"\f051"}.icon-eject:before{content:"\f052"}.icon-chevron-left:before{content:"\f053"}.icon-chevron-right:before{content:"\f054"}.icon-plus-sign:before{content:"\f055"}.icon-minus-sign:before{content:"\f056"}.icon-remove-sign:before{content:"\f057"}.icon-ok-sign:before{content:"\f058"}.icon-question-sign:before{content:"\f059"}.icon-info-sign:before{content:"\f05a"}.icon-screenshot:before{content:"\f05b"}.icon-remove-circle:before{content:"\f05c"}.icon-ok-circle:before{content:"\f05d"}.icon-ban-circle:before{content:"\f05e"}.icon-arrow-left:before{content:"\f060"}.icon-arrow-right:before{content:"\f061"}.icon-arrow-up:before{content:"\f062"}.icon-arrow-down:before{content:"\f063"}.icon-share-alt:before{content:"\f064"}.icon-resize-full:before{content:"\f065"}.icon-resize-small:before{content:"\f066"}.icon-plus:before{content:"\f067"}.icon-minus:before{content:"\f068"}.icon-asterisk:before{content:"\f069"}.icon-exclamation-sign:before{content:"\f06a"}.icon-gift:before{content:"\f06b"}.icon-leaf:before{content:"\f06c"}.icon-fire:before{content:"\f06d"}.icon-eye-open:before{content:"\f06e"}.icon-eye-close:before{content:"\f070"}.icon-warning-sign:before{content:"\f071"}.icon-plane:before{content:"\f072"}.icon-calendar:before{content:"\f073"}.icon-random:before{content:"\f074"}.icon-comment:before{content:"\f075"}.icon-magnet:before{content:"\f076"}.icon-chevron-up:before{content:"\f077"}.icon-chevron-down:before{content:"\f078"}.icon-retweet:before{content:"\f079"}.icon-shopping-cart:before{content:"\f07a"}.icon-folder-close:before{content:"\f07b"}.icon-folder-open:before{content:"\f07c"}.icon-resize-vertical:before{content:"\f07d"}.icon-resize-horizontal:before{content:"\f07e"}.icon-bar-chart:before{content:"\f080"}.icon-twitter-sign:before{content:"\f081"}.icon-facebook-sign:before{content:"\f082"}.icon-camera-retro:before{content:"\f083"}.icon-key:before{content:"\f084"}.icon-cogs:before{content:"\f085"}.icon-comments:before{content:"\f086"}.icon-thumbs-up:before{content:"\f087"}.icon-thumbs-down:before{content:"\f088"}.icon-star-half:before{content:"\f089"}.icon-heart-empty:before{content:"\f08a"}.icon-signout:before{content:"\f08b"}.icon-linkedin-sign:before{content:"\f08c"}.icon-pushpin:before{content:"\f08d"}.icon-external-link:before{content:"\f08e"}.icon-signin:before{content:"\f090"}.icon-trophy:before{content:"\f091"}.icon-github-sign:before{content:"\f092"}.icon-upload-alt:before{content:"\f093"}.icon-lemon:before{content:"\f094"}.icon-phone:before{content:"\f095"}.icon-check-empty:before{content:"\f096"}.icon-bookmark-empty:before{content:"\f097"}.icon-phone-sign:before{content:"\f098"}.icon-twitter:before{content:"\f099"}.icon-facebook:before{content:"\f09a"}.icon-github:before{content:"\f09b"}.icon-unlock:before{content:"\f09c"}.icon-credit-card:before{content:"\f09d"}.icon-rss:before{content:"\f09e"}.icon-hdd:before{content:"\f0a0"}.icon-bullhorn:before{content:"\f0a1"}.icon-bell:before{content:"\f0a2"}.icon-certificate:before{content:"\f0a3"}.icon-hand-right:before{content:"\f0a4"}.icon-hand-left:before{content:"\f0a5"}.icon-hand-up:before{content:"\f0a6"}.icon-hand-down:before{content:"\f0a7"}.icon-circle-arrow-left:before{content:"\f0a8"}.icon-circle-arrow-right:before{content:"\f0a9"}.icon-circle-arrow-up:before{content:"\f0aa"}.icon-circle-arrow-down:before{content:"\f0ab"}.icon-globe:before{content:"\f0ac"}.icon-wrench:before{content:"\f0ad"}.icon-tasks:before{content:"\f0ae"}.icon-filter:before{content:"\f0b0"}.icon-briefcase:before{content:"\f0b1"}.icon-fullscreen:before{content:"\f0b2"}.icon-group:before{content:"\f0c0"}.icon-link:before{content:"\f0c1"}.icon-cloud:before{content:"\f0c2"}.icon-beaker:before{content:"\f0c3"}.icon-cut:before{content:"\f0c4"}.icon-copy:before{content:"\f0c5"}.icon-paper-clip:before{content:"\f0c6"}.icon-save:before{content:"\f0c7"}.icon-sign-blank:before{content:"\f0c8"}.icon-reorder:before{content:"\f0c9"}.icon-list-ul:before{content:"\f0ca"}.icon-list-ol:before{content:"\f0cb"}.icon-strikethrough:before{content:"\f0cc"}.icon-underline:before{content:"\f0cd"}.icon-table:before{content:"\f0ce"}.icon-magic:before{content:"\f0d0"}.icon-truck:before{content:"\f0d1"}.icon-pinterest:before{content:"\f0d2"}.icon-pinterest-sign:before{content:"\f0d3"}.icon-google-plus-sign:before{content:"\f0d4"}.icon-google-plus:before{content:"\f0d5"}.icon-money:before{content:"\f0d6"}.icon-caret-down:before{content:"\f0d7"}.icon-caret-up:before{content:"\f0d8"}.icon-caret-left:before{content:"\f0d9"}.icon-caret-right:before{content:"\f0da"}.icon-columns:before{content:"\f0db"}.icon-sort:before{content:"\f0dc"}.icon-sort-down:before{content:"\f0dd"}.icon-sort-up:before{content:"\f0de"}.icon-envelope-alt:before{content:"\f0e0"}.icon-linkedin:before{content:"\f0e1"}.icon-undo:before{content:"\f0e2"}.icon-legal:before{content:"\f0e3"}.icon-dashboard:before{content:"\f0e4"}.icon-comment-alt:before{content:"\f0e5"}.icon-comments-alt:before{content:"\f0e6"}.icon-bolt:before{content:"\f0e7"}.icon-sitemap:before{content:"\f0e8"}.icon-umbrella:before{content:"\f0e9"}.icon-paste:before{content:"\f0ea"}.icon-lightbulb:before{content:"\f0eb"}.icon-exchange:before{content:"\f0ec"}.icon-cloud-download:before{content:"\f0ed"}.icon-cloud-upload:before{content:"\f0ee"}.icon-user-md:before{content:"\f0f0"}.icon-stethoscope:before{content:"\f0f1"}.icon-suitcase:before{content:"\f0f2"}.icon-bell-alt:before{content:"\f0f3"}.icon-coffee:before{content:"\f0f4"}.icon-food:before{content:"\f0f5"}.icon-file-alt:before{content:"\f0f6"}.icon-building:before{content:"\f0f7"}.icon-hospital:before{content:"\f0f8"}.icon-ambulance:before{content:"\f0f9"}.icon-medkit:before{content:"\f0fa"}.icon-fighter-jet:before{content:"\f0fb"}.icon-beer:before{content:"\f0fc"}.icon-h-sign:before{content:"\f0fd"}.icon-plus-sign-alt:before{content:"\f0fe"}.icon-double-angle-left:before{content:"\f100"}.icon-double-angle-right:before{content:"\f101"}.icon-double-angle-up:before{content:"\f102"}.icon-double-angle-down:before{content:"\f103"}.icon-angle-left:before{content:"\f104"}.icon-angle-right:before{content:"\f105"}.icon-angle-up:before{content:"\f106"}.icon-angle-down:before{content:"\f107"}.icon-desktop:before{content:"\f108"}.icon-laptop:before{content:"\f109"}.icon-tablet:before{content:"\f10a"}.icon-mobile-phone:before{content:"\f10b"}.icon-circle-blank:before{content:"\f10c"}.icon-quote-left:before{content:"\f10d"}.icon-quote-right:before{content:"\f10e"}.icon-spinner:before{content:"\f110"}.icon-circle:before{content:"\f111"}.icon-reply:before{content:"\f112"}.icon-github-alt:before{content:"\f113"}.icon-folder-close-alt:before{content:"\f114"}.icon-folder-open-alt:before{content:"\f115"}body{background-color:#fff;-webkit-text-size-adjust:none}.site-wrapper{background-color:#f5f5f5}hr,.hr{border:0;border-top:1px solid #dedede;display:block;height:0;margin:2rem 0;width:100%}hr.double,.hr.double{border-top:3px double #dedede}.drag-handle{cursor:move}.drag-handle.vert-arrows{cursor:ns-resize}.drag-handle.horz-arrows{cursor:ew-resize}.login-form{max-width:30.4rem}.flexbox .flex{display:-webkit-box;display:-moz-box;display:-ms-box;display:box;-webkit-box-pack:center;-moz-box-pack:center;-ms-box-pack:center;box-pack:center;-webkit-box-align:start;-moz-box-align:start;-ms-box-align:start;box-align:start;-webkit-flex-align:start;-moz-flex-align:start;-ms-flex-align:start;flex-align:start;-webkit-box-orient:vertical;-moz-box-orient:vertical;-ms-box-orient:vertical;box-orient:vertical}.flexbox .flex>*{display:block;-webkit-box-flex:0;-moz-box-flex:0;-ms-box-flex:0;box-flex:0}.flexbox .flex>*+*{margin-top:0.4rem}.flexbox .flex.flex-horz{-webkit-box-orient:horizontal;-moz-box-orient:horizontal;-ms-box-orient:horizontal;box-orient:horizontal}.flexbox .flex.flex-horz>*+*{margin-top:0}.flexbox .flex-primary{-webkit-box-flex:10;-moz-box-flex:10;-ms-box-flex:10;box-flex:10}.sticky-item{z-index:1300;-webkit-box-shadow:0 2px 4px rgba(0,0,0,0.25);-moz-box-shadow:0 2px 4px rgba(0,0,0,0.25);box-shadow:0 2px 4px rgba(0,0,0,0.25)}@media screen and (min-width: 49.2em){.flexbox .flex{-webkit-box-align:center;-moz-box-align:center;-ms-box-align:center;box-align:center;-webkit-flex-align:center;-moz-flex-align:center;-ms-flex-align:center;flex-align:center;-webkit-box-orient:horizontal;-moz-box-orient:horizontal;-ms-box-orient:horizontal;box-orient:horizontal}.flexbox .flex>*+*{margin-top:auto}.flexbox .flex.flex-vert{-webkit-box-orient:vertical;-moz-box-orient:vertical;-ms-box-orient:vertical;box-orient:vertical}.flexbox .flex.flex-vert>*+*{margin-top:0.4rem}.flexbox .flex.flex-start{-webkit-box-align:start;-moz-box-align:start;-ms-box-align:start;box-align:start;-webkit-flex-align:start;-moz-flex-align:start;-ms-flex-align:start;flex-align:start}}body{background-color:#f5f5f5}hr,.hr{background:#c9c9c9;border:0;display:block;height:1px;margin:2.25em 0;width:100%}.post+.post{border-top:1px solid #dedede;margin-top:2em;padding-top:1em}.post .title a{color:#4d4d4d}.post .title a:hover{color:#f68b1f}.post header{margin-bottom:1.5rem}.post footer{margin-top:0.66667rem}.meta{font-size:0.86667em;font-style:italic}.social{margin-top:1em}.social div,.social span,.social iframe{display:inline-block;}.featured-image{margin-bottom:2.66667rem}.featured-image img{display:block;margin:0 auto;width:100%}.widget+.widget{margin-top:1.5em}.widget-title{margin-bottom:0.26667rem}.sidebar blockquote{color:#7e7e7e;font-size:0.86667rem}.sidebar .widget-title{color:#777}.widget-title+.menu-sidebar{margin-top:0.5rem}.post-content img{display:block;margin:0 auto}.post-content iframe{margin:1.5em auto;display:block;width:100%}.learn-more{background:#eee;padding:1em;-webkit-border-radius:2px;-moz-border-radius:2px;-ms-border-radius:2px;-o-border-radius:2px;border-radius:2px}h1,.h1,h2,.h2{line-height:1.4}.sidebar .widget p,.sidebar .widget ul{font-size:80%;color:#7e7e7e}ol,ul{margin-left:2em}@media screen and (min-width: 49.2em){.social{height:25px}}@media print{#header,footer,aside,.social,#footer,.sidebar{display:none}h1,h2,h3,h4,h5,h6,p,li{color:black}body{font-size:12px}p,a,li{font-size:12px}h1{font-size:24px}h2{font-size:16px}h3{font-size:14px}.post-content a:link:after,.post-content a:visited:after{content:" (" attr(href) ") ";font-size:85%}} - .fb-like { - line-height: 1; - vertical-align: top; - margin: 0 10px; -} -.older-posts { - float:right; -} -.credits { - clear: both; - padding: 2rem 0 0 0; - font-size: 12px; - color: #bbb; -} -.credits a { - color: #bbb; - vertical-align: bottom; -} - -code{ - background-clip: padding-box; - border: 1px solid #ccc; - color: #333; - background-color: #eaeaea; - display: inline-block; - line-height: 20px; - margin: 0 2px -1px; - padding: 0px 3px; - vertical-align: baseline; - font-family: monospace; -} - -pre{ - background-color: #eaeaea; -} - -pre code{ - border: none !important; -} \ No newline at end of file diff --git a/h2mux/shared_buffer.go b/h2mux/shared_buffer.go deleted file mode 100644 index 31868c8c..00000000 --- a/h2mux/shared_buffer.go +++ /dev/null @@ -1,67 +0,0 @@ -package h2mux - -import ( - "bytes" - "io" - "sync" -) - -type SharedBuffer struct { - cond *sync.Cond - buffer bytes.Buffer - eof bool -} - -func NewSharedBuffer() *SharedBuffer { - return &SharedBuffer{ - cond: sync.NewCond(&sync.Mutex{}), - } -} - -func (s *SharedBuffer) Read(p []byte) (n int, err error) { - totalRead := 0 - s.cond.L.Lock() - for totalRead == 0 { - n, err = s.buffer.Read(p[totalRead:]) - totalRead += n - if err == io.EOF { - if s.eof { - break - } - err = nil - if n > 0 { - break - } - s.cond.Wait() - } - } - s.cond.L.Unlock() - return totalRead, err -} - -func (s *SharedBuffer) Write(p []byte) (n int, err error) { - s.cond.L.Lock() - defer s.cond.L.Unlock() - if s.eof { - return 0, io.EOF - } - n, err = s.buffer.Write(p) - s.cond.Signal() - return -} - -func (s *SharedBuffer) Close() error { - s.cond.L.Lock() - defer s.cond.L.Unlock() - if !s.eof { - s.eof = true - s.cond.Signal() - } - return nil -} - -func (s *SharedBuffer) Closed() bool { - s.cond.L.Lock() - defer s.cond.L.Unlock() - return s.eof -} diff --git a/h2mux/shared_buffer_test.go b/h2mux/shared_buffer_test.go deleted file mode 100644 index fe438fae..00000000 --- a/h2mux/shared_buffer_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package h2mux - -import ( - "bytes" - "io" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func AssertIOReturnIsGood(t *testing.T, expected int) func(int, error) { - return func(actual int, err error) { - if expected != actual { - t.Fatalf("Expected %d bytes, got %d", expected, actual) - } - if err != nil { - t.Fatalf("Unexpected error %s", err) - } - } -} - -func TestSharedBuffer(t *testing.T) { - b := NewSharedBuffer() - testData := []byte("Hello world") - AssertIOReturnIsGood(t, len(testData))(b.Write(testData)) - bytesRead := make([]byte, len(testData)) - AssertIOReturnIsGood(t, len(testData))(b.Read(bytesRead)) -} - -func TestSharedBufferBlockingRead(t *testing.T) { - b := NewSharedBuffer() - testData1 := []byte("Hello") - testData2 := []byte(" world") - result := make(chan []byte) - go func() { - bytesRead := make([]byte, len(testData1)+len(testData2)) - nRead, err := b.Read(bytesRead) - AssertIOReturnIsGood(t, len(testData1))(nRead, err) - result <- bytesRead[:nRead] - nRead, err = b.Read(bytesRead) - AssertIOReturnIsGood(t, len(testData2))(nRead, err) - result <- bytesRead[:nRead] - }() - time.Sleep(time.Millisecond * 250) - select { - case <-result: - t.Fatalf("read returned early") - default: - } - AssertIOReturnIsGood(t, len(testData1))(b.Write([]byte(testData1))) - select { - case r := <-result: - assert.Equal(t, testData1, r) - case <-time.After(time.Second): - t.Fatalf("read timed out") - } - AssertIOReturnIsGood(t, len(testData2))(b.Write([]byte(testData2))) - select { - case r := <-result: - assert.Equal(t, testData2, r) - case <-time.After(time.Second): - t.Fatalf("read timed out") - } -} - -// This is quite slow under the race detector -func TestSharedBufferConcurrentReadWrite(t *testing.T) { - b := NewSharedBuffer() - var expectedResult, actualResult bytes.Buffer - var wg sync.WaitGroup - wg.Add(2) - go func() { - block := make([]byte, 256) - for i := range block { - block[i] = byte(i) - } - for blockSize := 1; blockSize <= 256; blockSize++ { - for i := 0; i < 256; i++ { - expectedResult.Write(block[:blockSize]) - n, err := b.Write(block[:blockSize]) - if n != blockSize || err != nil { - t.Errorf("write error: %d %s", n, err) - return - } - } - } - wg.Done() - }() - go func() { - block := make([]byte, 256) - // Change block sizes in opposition to the write thread, to test blocking for new data. - for blockSize := 256; blockSize > 0; blockSize-- { - for i := 0; i < 256; i++ { - n, err := io.ReadFull(b, block[:blockSize]) - if n != blockSize || err != nil { - t.Errorf("read error: %d %s", n, err) - return - } - actualResult.Write(block[:blockSize]) - } - } - wg.Done() - }() - wg.Wait() - if bytes.Compare(expectedResult.Bytes(), actualResult.Bytes()) != 0 { - t.Fatal("Result diverged") - } -} - -func TestSharedBufferClose(t *testing.T) { - b := NewSharedBuffer() - testData := []byte("Hello world") - AssertIOReturnIsGood(t, len(testData))(b.Write(testData)) - err := b.Close() - if err != nil { - t.Fatalf("unexpected error from Close: %s", err) - } - bytesRead := make([]byte, len(testData)) - AssertIOReturnIsGood(t, len(testData))(b.Read(bytesRead)) - n, err := b.Read(bytesRead) - if n != 0 { - t.Fatalf("extra bytes received: %d", n) - } - if err != io.EOF { - t.Fatalf("expected EOF, got %s", err) - } -} diff --git a/h2mux/signal.go b/h2mux/signal.go deleted file mode 100644 index d716aed2..00000000 --- a/h2mux/signal.go +++ /dev/null @@ -1,34 +0,0 @@ -package h2mux - -// Signal describes an event that can be waited on for at least one signal. -// Signalling the event while it is in the signalled state is a noop. -// When the waiter wakes up, the signal is set to unsignalled. -// It is a way for any number of writers to inform a reader (without blocking) -// that an event has happened. -type Signal struct { - c chan struct{} -} - -// NewSignal creates a new Signal. -func NewSignal() Signal { - return Signal{c: make(chan struct{}, 1)} -} - -// Signal signals the event. -func (s Signal) Signal() { - // This channel is buffered, so the nonblocking send will always succeed if the buffer is empty. - select { - case s.c <- struct{}{}: - default: - } -} - -// Wait for the event to be signalled. -func (s Signal) Wait() { - <-s.c -} - -// WaitChannel returns a channel that is readable after Signal is called. -func (s Signal) WaitChannel() <-chan struct{} { - return s.c -} diff --git a/h2mux/streamerrormap.go b/h2mux/streamerrormap.go deleted file mode 100644 index 926b5ff2..00000000 --- a/h2mux/streamerrormap.go +++ /dev/null @@ -1,47 +0,0 @@ -package h2mux - -import ( - "sync" - - "golang.org/x/net/http2" -) - -// StreamErrorMap is used to track stream errors. This is a separate structure to ActiveStreamMap because -// errors can be raised against non-existent or closed streams. -type StreamErrorMap struct { - sync.RWMutex - // errors tracks per-stream errors - errors map[uint32]http2.ErrCode - // hasError is signaled whenever an error is raised. - hasError Signal -} - -// NewStreamErrorMap creates a new StreamErrorMap. -func NewStreamErrorMap() *StreamErrorMap { - return &StreamErrorMap{ - errors: make(map[uint32]http2.ErrCode), - hasError: NewSignal(), - } -} - -// RaiseError raises a stream error. -func (s *StreamErrorMap) RaiseError(streamID uint32, err http2.ErrCode) { - s.Lock() - s.errors[streamID] = err - s.Unlock() - s.hasError.Signal() -} - -// GetSignalChan returns a channel that is signalled when an error is raised. -func (s *StreamErrorMap) GetSignalChan() <-chan struct{} { - return s.hasError.WaitChannel() -} - -// GetErrors retrieves all errors currently raised. This resets the currently-tracked errors. -func (s *StreamErrorMap) GetErrors() map[uint32]http2.ErrCode { - s.Lock() - errors := s.errors - s.errors = make(map[uint32]http2.ErrCode) - s.Unlock() - return errors -} diff --git a/h2mux/booleanfuse.go b/supervisor/fuse.go similarity index 69% rename from h2mux/booleanfuse.go rename to supervisor/fuse.go index 8407ecc7..3a143437 100644 --- a/h2mux/booleanfuse.go +++ b/supervisor/fuse.go @@ -1,23 +1,23 @@ -package h2mux +package supervisor import "sync" -// BooleanFuse is a data structure that can be set once to a particular value using Fuse(value). +// booleanFuse is a data structure that can be set once to a particular value using Fuse(value). // Subsequent calls to Fuse() will have no effect. -type BooleanFuse struct { +type booleanFuse struct { value int32 mu sync.Mutex cond *sync.Cond } -func NewBooleanFuse() *BooleanFuse { - f := &BooleanFuse{} +func newBooleanFuse() *booleanFuse { + f := &booleanFuse{} f.cond = sync.NewCond(&f.mu) return f } // Value gets the value -func (f *BooleanFuse) Value() bool { +func (f *booleanFuse) Value() bool { // 0: unset // 1: set true // 2: set false @@ -26,7 +26,7 @@ func (f *BooleanFuse) Value() bool { return f.value == 1 } -func (f *BooleanFuse) Fuse(result bool) { +func (f *booleanFuse) Fuse(result bool) { f.mu.Lock() defer f.mu.Unlock() newValue := int32(2) @@ -40,7 +40,7 @@ func (f *BooleanFuse) Fuse(result bool) { } // Await blocks until Fuse has been called at least once. -func (f *BooleanFuse) Await() bool { +func (f *booleanFuse) Await() bool { f.mu.Lock() defer f.mu.Unlock() for f.value == 0 { diff --git a/supervisor/tunnel.go b/supervisor/tunnel.go index 03d7f930..340d4b8d 100644 --- a/supervisor/tunnel.go +++ b/supervisor/tunnel.go @@ -19,7 +19,6 @@ import ( "github.com/cloudflare/cloudflared/edgediscovery" "github.com/cloudflare/cloudflared/edgediscovery/allregions" "github.com/cloudflare/cloudflared/features" - "github.com/cloudflare/cloudflared/h2mux" "github.com/cloudflare/cloudflared/ingress" "github.com/cloudflare/cloudflared/management" "github.com/cloudflare/cloudflared/orchestration" @@ -199,7 +198,7 @@ func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolF haConnections.Inc() defer haConnections.Dec() - connectedFuse := h2mux.NewBooleanFuse() + connectedFuse := newBooleanFuse() go func() { if connectedFuse.Await() { connectedSignal.Notify() @@ -375,7 +374,7 @@ func (e *EdgeTunnelServer) serveTunnel( connLog *ConnAwareLogger, addr *allregions.EdgeAddr, connIndex uint8, - fuse *h2mux.BooleanFuse, + fuse *booleanFuse, backoff *protocolFallback, protocol connection.Protocol, ) (err error, recoverable bool) { @@ -441,7 +440,7 @@ func (e *EdgeTunnelServer) serveConnection( connLog *ConnAwareLogger, addr *allregions.EdgeAddr, connIndex uint8, - fuse *h2mux.BooleanFuse, + fuse *booleanFuse, backoff *protocolFallback, protocol connection.Protocol, ) (err error, recoverable bool) { @@ -645,7 +644,7 @@ func listenReconnect(ctx context.Context, reconnectCh <-chan ReconnectSignal, gr } type connectedFuse struct { - fuse *h2mux.BooleanFuse + fuse *booleanFuse backoff *protocolFallback }