cloudflared-mirror/h2mux/muxreader.go

514 lines
15 KiB
Go

package h2mux
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"net/url"
"time"
"github.com/cloudflare/cloudflared/logger"
"golang.org/x/net/http2"
)
const (
CloudflaredProxyTunnelHostnameHeader = "cf-cloudflared-proxy-tunnel-hostname"
)
type MuxReader struct {
// f is used to read HTTP2 frames.
f *http2.Framer
// handler provides a callback to receive new streams. if nil, new streams cannot be accepted.
handler MuxedStreamHandler
// streams tracks currently-open streams.
streams *activeStreamMap
// readyList is used to signal writable streams.
readyList *ReadyList
// streamErrors lets us report stream errors to the MuxWriter.
streamErrors *StreamErrorMap
// goAwayChan is used to tell the writer to send a GOAWAY message.
goAwayChan chan<- http2.ErrCode
// abortChan is used when shutting down ungracefully. When this becomes readable, all activity should stop.
abortChan <-chan struct{}
// pingTimestamp is an atomic value containing the latest received ping timestamp.
pingTimestamp *PingTimestamp
// connActive is used to signal to the writer that something happened on the connection.
// This is used to clear idle timeout disconnection deadlines.
connActive Signal
// The initial value for the send and receive window of a new stream.
initialStreamWindow uint32
// The max value for the send window of a stream.
streamWindowMax uint32
// The max size for the write buffer of a stream
streamWriteBufferMaxLen int
// r is a reference to the underlying connection used when shutting down.
r io.Closer
// metricsUpdater is used to report metrics
metricsUpdater muxMetricsUpdater
// bytesRead is the amount of bytes read from data frames since the last time we called metricsUpdater.updateInBoundBytes()
bytesRead *AtomicCounter
// dictionaries holds the h2 cross-stream compression dictionaries
dictionaries h2Dictionaries
}
// Shutdown blocks new streams from being created.
// It returns a channel that is closed once the last stream has closed.
func (r *MuxReader) Shutdown() <-chan struct{} {
done, alreadyInProgress := r.streams.Shutdown()
if alreadyInProgress {
return done
}
r.sendGoAway(http2.ErrCodeNo)
go func() {
// close reader side when last stream ends; this will cause the writer to abort
<-done
r.r.Close()
}()
return done
}
func (r *MuxReader) run(logger logger.Service) error {
defer logger.Debug("mux - read: event loop finished")
// routine to periodically update bytesRead
go func() {
tickC := time.Tick(updateFreq)
for {
select {
case <-r.abortChan:
return
case <-tickC:
r.metricsUpdater.updateInBoundBytes(r.bytesRead.Count())
}
}
}()
for {
frame, err := r.f.ReadFrame()
if err != nil {
errorString := fmt.Sprintf("mux - read: %s", err)
if errorDetail := r.f.ErrorDetail(); errorDetail != nil {
errorString = fmt.Sprintf("%s: errorDetail: %s", errorString, errorDetail)
}
switch e := err.(type) {
case http2.StreamError:
logger.Infof("%s: stream error", errorString)
// Ideally we wouldn't return here, since that aborts the muxer.
// We should communicate the error to the relevant MuxedStream
// data structure, so that callers of MuxedStream.Read() and
// MuxedStream.Write() would see it. Then we could `continue`
// and keep the muxer going.
return r.streamError(e.StreamID, e.Code)
case http2.ConnectionError:
logger.Infof("%s: stream error", errorString)
return r.connectionError(err)
default:
if isConnectionClosedError(err) {
if r.streams.Len() == 0 {
// don't log the error here -- that would just be extra noise
logger.Debug("mux - read: shutting down")
return nil
}
logger.Infof("%s: connection closed unexpectedly", errorString)
return err
} else {
logger.Infof("%s: frame read error", errorString)
return r.connectionError(err)
}
}
}
r.connActive.Signal()
logger.Debugf("mux - read: read frame: data %v", frame)
switch f := frame.(type) {
case *http2.DataFrame:
err = r.receiveFrameData(f, logger)
case *http2.MetaHeadersFrame:
err = r.receiveHeaderData(f)
case *http2.RSTStreamFrame:
streamID := f.Header().StreamID
if streamID == 0 {
return ErrInvalidStream
}
if stream, ok := r.streams.Get(streamID); ok {
stream.Close()
}
r.streams.Delete(streamID)
case *http2.PingFrame:
r.receivePingData(f)
case *http2.GoAwayFrame:
err = r.receiveGoAway(f)
// The receiver of a flow-controlled frame sends a WINDOW_UPDATE frame as it
// consumes data and frees up space in flow-control windows
case *http2.WindowUpdateFrame:
err = r.updateStreamWindow(f)
case *http2.UnknownFrame:
switch f.Header().Type {
case FrameUseDictionary:
err = r.receiveUseDictionary(f)
case FrameSetDictionary:
err = r.receiveSetDictionary(f)
default:
err = ErrUnexpectedFrameType
}
default:
err = ErrUnexpectedFrameType
}
if err != nil {
logger.Debugf("mux - read: read error: data %v", frame)
return r.connectionError(err)
}
}
}
func (r *MuxReader) newMuxedStream(streamID uint32) *MuxedStream {
return &MuxedStream{
streamID: streamID,
readBuffer: NewSharedBuffer(),
writeBuffer: &bytes.Buffer{},
writeBufferMaxLen: r.streamWriteBufferMaxLen,
writeBufferHasSpace: make(chan struct{}, 1),
receiveWindow: r.initialStreamWindow,
receiveWindowCurrentMax: r.initialStreamWindow,
receiveWindowMax: r.streamWindowMax,
sendWindow: r.initialStreamWindow,
readyList: r.readyList,
dictionaries: r.dictionaries,
}
}
// getStreamForFrame returns a stream if valid, or an error describing why the stream could not be returned.
func (r *MuxReader) getStreamForFrame(frame http2.Frame) (*MuxedStream, error) {
sid := frame.Header().StreamID
if sid == 0 {
return nil, ErrUnexpectedFrameType
}
if stream, ok := r.streams.Get(sid); ok {
return stream, nil
}
if r.streams.IsLocalStreamID(sid) {
// no stream available, but no error
return nil, ErrClosedStream
}
if sid < r.streams.LastPeerStreamID() {
// no stream available, stream closed error
return nil, ErrClosedStream
}
return nil, ErrUnknownStream
}
func (r *MuxReader) defaultStreamErrorHandler(err error, header http2.FrameHeader) error {
if header.Flags.Has(http2.FlagHeadersEndStream) {
return nil
} else if err == ErrUnknownStream || err == ErrClosedStream {
return r.streamError(header.StreamID, http2.ErrCodeStreamClosed)
} else {
return err
}
}
// Receives header frames from a stream. A non-nil error is a connection error.
func (r *MuxReader) receiveHeaderData(frame *http2.MetaHeadersFrame) error {
var stream *MuxedStream
sid := frame.Header().StreamID
if sid == 0 {
return ErrUnexpectedFrameType
}
newStream := r.streams.IsPeerStreamID(sid)
if newStream {
// header request
// TODO support trailers (if stream exists)
ok, err := r.streams.AcquirePeerID(sid)
if !ok {
// ignore new streams while shutting down
return r.streamError(sid, err)
}
stream = r.newMuxedStream(sid)
// Set stream. Returns false if a stream already existed with that ID or we are shutting down, return false.
if !r.streams.Set(stream) {
// got HEADERS frame for an existing stream
// TODO support trailers
return r.streamError(sid, http2.ErrCodeInternal)
}
} else {
// header response
var err error
if stream, err = r.getStreamForFrame(frame); err != nil {
return r.defaultStreamErrorHandler(err, frame.Header())
}
}
headers := make([]Header, 0, len(frame.Fields))
for _, header := range frame.Fields {
switch header.Name {
case ":method":
stream.method = header.Value
case ":path":
u, err := url.Parse(header.Value)
if err == nil {
stream.path = u.Path
}
case "accept-encoding":
// remove accept-encoding if dictionaries are enabled
if r.dictionaries.write != nil {
continue
}
case CloudflaredProxyTunnelHostnameHeader:
stream.tunnelHostname = TunnelHostname(header.Value)
}
headers = append(headers, Header{Name: header.Name, Value: header.Value})
}
stream.Headers = headers
if frame.Header().Flags.Has(http2.FlagHeadersEndStream) {
stream.receiveEOF()
return nil
}
if newStream {
go r.handleStream(stream)
} else {
close(stream.responseHeadersReceived)
}
return nil
}
func (r *MuxReader) handleStream(stream *MuxedStream) {
defer stream.Close()
r.handler.ServeStream(stream)
}
// Receives a data frame from a stream. A non-nil error is a connection error.
func (r *MuxReader) receiveFrameData(frame *http2.DataFrame, logger logger.Service) error {
stream, err := r.getStreamForFrame(frame)
if err != nil {
return r.defaultStreamErrorHandler(err, frame.Header())
}
data := frame.Data()
if len(data) > 0 {
n, err := stream.readBuffer.Write(data)
if err != nil {
return r.streamError(stream.streamID, http2.ErrCodeInternal)
}
r.bytesRead.IncrementBy(uint64(n))
}
if frame.Header().Flags.Has(http2.FlagDataEndStream) {
if stream.receiveEOF() {
r.streams.Delete(stream.streamID)
logger.Debugf("mux - read: stream closed: streamID: %d", frame.Header().StreamID)
} else {
logger.Debugf("mux - read: shutdown receive side: streamID: %d", frame.Header().StreamID)
}
return nil
}
if !stream.consumeReceiveWindow(uint32(len(data))) {
return r.streamError(stream.streamID, http2.ErrCodeFlowControl)
}
r.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow())
return nil
}
// Receive a PING from the peer. Update RTT and send/receive window metrics if it's an ACK.
func (r *MuxReader) receivePingData(frame *http2.PingFrame) {
ts := int64(binary.LittleEndian.Uint64(frame.Data[:]))
if !frame.IsAck() {
r.pingTimestamp.Set(ts)
return
}
// Update the computed RTT aggregations with a new measurement.
// `ts` is the time that the probe was sent.
// We assume that `time.Now()` is the time we received that probe.
r.metricsUpdater.updateRTT(&roundTripMeasurement{
receiveTime: time.Now(),
sendTime: time.Unix(0, ts),
})
}
// Receive a GOAWAY from the peer. Gracefully shut down our connection.
func (r *MuxReader) receiveGoAway(frame *http2.GoAwayFrame) error {
r.Shutdown()
// Close all streams above the last processed stream
lastStream := r.streams.LastLocalStreamID()
for i := frame.LastStreamID + 2; i <= lastStream; i++ {
if stream, ok := r.streams.Get(i); ok {
stream.Close()
}
}
return nil
}
// Receive a USE_DICTIONARY from the peer. Setup dictionary for stream.
func (r *MuxReader) receiveUseDictionary(frame *http2.UnknownFrame) error {
payload := frame.Payload()
streamID := frame.StreamID
// Check frame is formatted properly
if len(payload) != 1 {
return r.streamError(streamID, http2.ErrCodeProtocol)
}
stream, err := r.getStreamForFrame(frame)
if err != nil {
return err
}
if stream.receivedUseDict == true || stream.dictionaries.read == nil {
return r.streamError(streamID, http2.ErrCodeInternal)
}
stream.receivedUseDict = true
dictID := payload[0]
dictReader := stream.dictionaries.read.newReader(stream.readBuffer.(*SharedBuffer), dictID)
if dictReader == nil {
return r.streamError(streamID, http2.ErrCodeInternal)
}
stream.readBufferLock.Lock()
stream.readBuffer = dictReader
stream.readBufferLock.Unlock()
return nil
}
// Receive a SET_DICTIONARY from the peer. Update dictionaries accordingly.
func (r *MuxReader) receiveSetDictionary(frame *http2.UnknownFrame) (err error) {
payload := frame.Payload()
flags := frame.Flags
stream, err := r.getStreamForFrame(frame)
if err != nil && err != ErrClosedStream {
return err
}
reader, ok := stream.readBuffer.(*h2DictionaryReader)
if !ok {
return r.streamError(frame.StreamID, http2.ErrCodeProtocol)
}
// A SetDictionary frame consists of several
// Dictionary-Entries that specify how existing dictionaries
// are to be updated using the current stream data
// +---------------+---------------+
// | Dictionary-Entry (+) ...
// +---------------+---------------+
for {
// Each Dictionary-Entry is formatted as follows:
// +-------------------------------+
// | Dictionary-ID (8) |
// +---+---------------------------+
// | P | Size (7+) |
// +---+---------------------------+
// | E?| D?| Truncate? (6+) |
// +---+---------------------------+
// | Offset? (8+) |
// +-------------------------------+
var size, truncate, offset uint64
var p, e, d bool
// Parse a single Dictionary-Entry
if len(payload) < 2 { // Must have at least id and size
return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
dictID := uint8(payload[0])
p = (uint8(payload[1]) >> 7) == 1
payload, size, err = http2ReadVarInt(7, payload[1:])
if err != nil {
return
}
if flags.Has(FlagSetDictionaryAppend) {
// Presence of FlagSetDictionaryAppend means we expect e, d and truncate
if len(payload) < 1 {
return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
e = (uint8(payload[0]) >> 7) == 1
d = (uint8((payload[0])>>6) & 1) == 1
payload, truncate, err = http2ReadVarInt(6, payload)
if err != nil {
return
}
}
if flags.Has(FlagSetDictionaryOffset) {
// Presence of FlagSetDictionaryOffset means we expect offset
if len(payload) < 1 {
return MuxerStreamError{"unexpected EOF", http2.ErrCodeProtocol}
}
payload, offset, err = http2ReadVarInt(8, payload)
if err != nil {
return
}
}
setdict := setDictRequest{streamID: stream.streamID,
dictID: dictID,
dictSZ: size,
truncate: truncate,
offset: offset,
P: p,
E: e,
D: d}
// Find the right dictionary
dict, err := r.dictionaries.read.getDictByID(dictID)
if err != nil {
return err
}
// Register a dictionary update order for the dictionary and reader
updateEntry := &dictUpdate{reader: reader, dictionary: dict, s: setdict}
dict.queue = append(dict.queue, updateEntry)
reader.queue = append(reader.queue, updateEntry)
// End of frame
if len(payload) == 0 {
break
}
}
return nil
}
// Receives header frames from a stream. A non-nil error is a connection error.
func (r *MuxReader) updateStreamWindow(frame *http2.WindowUpdateFrame) error {
stream, err := r.getStreamForFrame(frame)
if err != nil && err != ErrUnknownStream && err != ErrClosedStream {
return err
}
if stream == nil {
// ignore window updates on closed streams
return nil
}
stream.replenishSendWindow(frame.Increment)
r.metricsUpdater.updateSendWindow(stream.getSendWindow())
return nil
}
// Raise a stream processing error, closing the stream. Runs on the write thread.
func (r *MuxReader) streamError(streamID uint32, e http2.ErrCode) error {
r.streamErrors.RaiseError(streamID, e)
return nil
}
func (r *MuxReader) connectionError(err error) error {
http2Code := http2.ErrCodeInternal
switch e := err.(type) {
case http2.ConnectionError:
http2Code = http2.ErrCode(e)
case MuxerProtocolError:
http2Code = e.h2code
}
r.sendGoAway(http2Code)
return err
}
// Instruct the writer to send a GOAWAY message if possible. This may fail in
// the case where an existing GOAWAY message is in flight or the writer event
// loop already ended.
func (r *MuxReader) sendGoAway(errCode http2.ErrCode) {
select {
case r.goAwayChan <- errCode:
default:
}
}