2022-08-22 16:41:51 +00:00
|
|
|
//go:build darwin
|
|
|
|
|
|
|
|
package ingress
|
|
|
|
|
2022-09-02 16:29:50 +00:00
|
|
|
// This file implements ICMPProxy for Darwin. It uses a non-privileged ICMP socket to send echo requests and listen for
|
|
|
|
// echo replies. The source IP of the requests are rewritten to the bind IP of the socket and the socket reads all
|
|
|
|
// messages, so we use echo ID to distinguish the replies. Each (source IP, destination IP, echo ID) is assigned a
|
|
|
|
// unique echo ID.
|
|
|
|
|
2022-08-22 16:41:51 +00:00
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"fmt"
|
|
|
|
"math"
|
|
|
|
"net/netip"
|
|
|
|
"strconv"
|
|
|
|
"sync"
|
2022-09-02 16:29:50 +00:00
|
|
|
"time"
|
2022-08-22 16:41:51 +00:00
|
|
|
|
|
|
|
"github.com/rs/zerolog"
|
2022-10-13 10:01:25 +00:00
|
|
|
"go.opentelemetry.io/otel/attribute"
|
2022-08-22 16:41:51 +00:00
|
|
|
"golang.org/x/net/icmp"
|
|
|
|
|
|
|
|
"github.com/cloudflare/cloudflared/packet"
|
2022-10-13 10:01:25 +00:00
|
|
|
"github.com/cloudflare/cloudflared/tracing"
|
2022-08-22 16:41:51 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type icmpProxy struct {
|
2022-09-02 16:29:50 +00:00
|
|
|
srcFunnelTracker *packet.FunnelTracker
|
|
|
|
echoIDTracker *echoIDTracker
|
|
|
|
conn *icmp.PacketConn
|
|
|
|
// Response is handled in one-by-one, so encoder can be shared between funnels
|
|
|
|
encoder *packet.Encoder
|
|
|
|
logger *zerolog.Logger
|
|
|
|
idleTimeout time.Duration
|
2022-08-22 16:41:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// echoIDTracker tracks which ID has been assigned. It first loops through assignment from lastAssignment to then end,
|
|
|
|
// then from the beginning to lastAssignment.
|
|
|
|
// ICMP echo are short lived. By the time an ID is revisited, it should have been released.
|
|
|
|
type echoIDTracker struct {
|
2022-09-09 15:48:42 +00:00
|
|
|
lock sync.Mutex
|
|
|
|
// maps the source IP, destination IP and original echo ID to a unique echo ID obtained from assignment
|
|
|
|
mapping map[flow3Tuple]uint16
|
2022-08-22 16:41:51 +00:00
|
|
|
// assignment tracks if an ID is assigned using index as the ID
|
|
|
|
// The size of the array is math.MaxUint16 because echo ID is 2 bytes
|
|
|
|
assignment [math.MaxUint16]bool
|
|
|
|
// nextAssignment is the next number to check for assigment
|
|
|
|
nextAssignment uint16
|
|
|
|
}
|
|
|
|
|
|
|
|
func newEchoIDTracker() *echoIDTracker {
|
|
|
|
return &echoIDTracker{
|
2022-09-09 15:48:42 +00:00
|
|
|
mapping: make(map[flow3Tuple]uint16),
|
2022-08-22 16:41:51 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-09 15:48:42 +00:00
|
|
|
// Get assignment or assign a new ID.
|
|
|
|
func (eit *echoIDTracker) getOrAssign(key flow3Tuple) (id uint16, success bool) {
|
2022-08-22 16:41:51 +00:00
|
|
|
eit.lock.Lock()
|
|
|
|
defer eit.lock.Unlock()
|
2022-09-09 15:48:42 +00:00
|
|
|
id, exists := eit.mapping[key]
|
|
|
|
if exists {
|
|
|
|
return id, true
|
|
|
|
}
|
2022-08-22 16:41:51 +00:00
|
|
|
|
|
|
|
if eit.nextAssignment == math.MaxUint16 {
|
|
|
|
eit.nextAssignment = 0
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, assigned := range eit.assignment[eit.nextAssignment:] {
|
|
|
|
if !assigned {
|
|
|
|
echoID := uint16(i) + eit.nextAssignment
|
2022-09-09 15:48:42 +00:00
|
|
|
eit.set(key, echoID)
|
2022-08-22 16:41:51 +00:00
|
|
|
return echoID, true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for i, assigned := range eit.assignment[0:eit.nextAssignment] {
|
|
|
|
if !assigned {
|
|
|
|
echoID := uint16(i)
|
2022-09-09 15:48:42 +00:00
|
|
|
eit.set(key, echoID)
|
2022-08-22 16:41:51 +00:00
|
|
|
return echoID, true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0, false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Caller should hold the lock
|
2022-09-09 15:48:42 +00:00
|
|
|
func (eit *echoIDTracker) set(key flow3Tuple, assignedEchoID uint16) {
|
|
|
|
eit.assignment[assignedEchoID] = true
|
|
|
|
eit.mapping[key] = assignedEchoID
|
|
|
|
eit.nextAssignment = assignedEchoID + 1
|
2022-08-22 16:41:51 +00:00
|
|
|
}
|
|
|
|
|
2022-09-09 15:48:42 +00:00
|
|
|
func (eit *echoIDTracker) release(key flow3Tuple, assigned uint16) bool {
|
2022-08-22 16:41:51 +00:00
|
|
|
eit.lock.Lock()
|
|
|
|
defer eit.lock.Unlock()
|
|
|
|
|
2022-09-09 15:48:42 +00:00
|
|
|
currentEchoID, exists := eit.mapping[key]
|
|
|
|
if exists && assigned == currentEchoID {
|
|
|
|
delete(eit.mapping, key)
|
|
|
|
eit.assignment[assigned] = false
|
2022-08-22 16:41:51 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2022-09-02 16:29:50 +00:00
|
|
|
type echoFunnelID uint16
|
2022-08-22 16:41:51 +00:00
|
|
|
|
2022-09-02 16:29:50 +00:00
|
|
|
func (snf echoFunnelID) Type() string {
|
2022-08-22 16:41:51 +00:00
|
|
|
return "echoID"
|
|
|
|
}
|
|
|
|
|
2022-09-02 16:29:50 +00:00
|
|
|
func (snf echoFunnelID) String() string {
|
2022-08-22 16:41:51 +00:00
|
|
|
return strconv.FormatUint(uint64(snf), 10)
|
|
|
|
}
|
|
|
|
|
2022-09-20 10:39:51 +00:00
|
|
|
func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
|
|
|
|
conn, err := newICMPConn(listenIP, zone)
|
2022-08-22 16:41:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-09-20 10:39:51 +00:00
|
|
|
logger.Info().Msgf("Created ICMP proxy listening on %s", conn.LocalAddr())
|
2022-08-22 16:41:51 +00:00
|
|
|
return &icmpProxy{
|
2022-09-02 16:29:50 +00:00
|
|
|
srcFunnelTracker: packet.NewFunnelTracker(),
|
|
|
|
echoIDTracker: newEchoIDTracker(),
|
|
|
|
encoder: packet.NewEncoder(),
|
|
|
|
conn: conn,
|
|
|
|
logger: logger,
|
|
|
|
idleTimeout: idleTimeout,
|
2022-08-22 16:41:51 +00:00
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2022-10-13 10:01:25 +00:00
|
|
|
func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
|
2024-01-15 16:49:17 +00:00
|
|
|
_, span := responder.requestSpan(ctx, pk)
|
2022-10-13 10:01:25 +00:00
|
|
|
defer responder.exportSpan()
|
|
|
|
|
2022-09-09 15:48:42 +00:00
|
|
|
originalEcho, err := getICMPEcho(pk.Message)
|
|
|
|
if err != nil {
|
2022-10-13 10:01:25 +00:00
|
|
|
tracing.EndWithErrorStatus(span, err)
|
2022-09-09 15:48:42 +00:00
|
|
|
return err
|
|
|
|
}
|
2024-01-15 16:49:17 +00:00
|
|
|
observeICMPRequest(ip.logger, span, pk.Src.String(), pk.Dst.String(), originalEcho.ID, originalEcho.Seq)
|
|
|
|
|
2022-09-09 15:48:42 +00:00
|
|
|
echoIDTrackerKey := flow3Tuple{
|
|
|
|
srcIP: pk.Src,
|
|
|
|
dstIP: pk.Dst,
|
|
|
|
originalEchoID: originalEcho.ID,
|
|
|
|
}
|
|
|
|
assignedEchoID, success := ip.echoIDTracker.getOrAssign(echoIDTrackerKey)
|
|
|
|
if !success {
|
2022-10-13 10:01:25 +00:00
|
|
|
err := fmt.Errorf("failed to assign unique echo ID")
|
|
|
|
tracing.EndWithErrorStatus(span, err)
|
|
|
|
return err
|
2022-09-09 15:48:42 +00:00
|
|
|
}
|
2022-10-13 10:01:25 +00:00
|
|
|
span.SetAttributes(attribute.Int("assignedEchoID", int(assignedEchoID)))
|
|
|
|
|
2022-11-08 23:12:33 +00:00
|
|
|
shouldReplaceFunnelFunc := createShouldReplaceFunnelFunc(ip.logger, responder.datagramMuxer, pk, originalEcho.ID)
|
2022-09-09 15:48:42 +00:00
|
|
|
newFunnelFunc := func() (packet.Funnel, error) {
|
2022-09-02 16:29:50 +00:00
|
|
|
originalEcho, err := getICMPEcho(pk.Message)
|
|
|
|
if err != nil {
|
2022-09-09 15:48:42 +00:00
|
|
|
return nil, err
|
2022-09-02 16:29:50 +00:00
|
|
|
}
|
2022-10-13 10:01:25 +00:00
|
|
|
closeCallback := func() error {
|
|
|
|
ip.echoIDTracker.release(echoIDTrackerKey, assignedEchoID)
|
|
|
|
return nil
|
2022-09-02 16:29:50 +00:00
|
|
|
}
|
2022-10-13 10:01:25 +00:00
|
|
|
icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, ip.conn, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
|
2022-09-09 15:48:42 +00:00
|
|
|
return icmpFlow, nil
|
2022-09-02 16:29:50 +00:00
|
|
|
}
|
2022-09-09 15:48:42 +00:00
|
|
|
funnelID := echoFunnelID(assignedEchoID)
|
2022-11-08 23:12:33 +00:00
|
|
|
funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, shouldReplaceFunnelFunc, newFunnelFunc)
|
2022-09-09 15:48:42 +00:00
|
|
|
if err != nil {
|
2022-10-13 10:01:25 +00:00
|
|
|
tracing.EndWithErrorStatus(span, err)
|
2022-09-09 15:48:42 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
if isNew {
|
2022-10-13 10:01:25 +00:00
|
|
|
span.SetAttributes(attribute.Bool("newFlow", true))
|
2022-09-09 15:48:42 +00:00
|
|
|
ip.logger.Debug().
|
|
|
|
Str("src", pk.Src.String()).
|
|
|
|
Str("dst", pk.Dst.String()).
|
|
|
|
Int("originalEchoID", originalEcho.ID).
|
|
|
|
Int("assignedEchoID", int(assignedEchoID)).
|
|
|
|
Msg("New flow")
|
2022-09-02 16:29:50 +00:00
|
|
|
}
|
|
|
|
icmpFlow, err := toICMPEchoFlow(funnel)
|
|
|
|
if err != nil {
|
2022-10-13 10:01:25 +00:00
|
|
|
tracing.EndWithErrorStatus(span, err)
|
|
|
|
return err
|
|
|
|
}
|
2024-01-15 16:49:17 +00:00
|
|
|
|
2022-10-13 10:01:25 +00:00
|
|
|
err = icmpFlow.sendToDst(pk.Dst, pk.Message)
|
|
|
|
if err != nil {
|
|
|
|
tracing.EndWithErrorStatus(span, err)
|
2022-09-02 16:29:50 +00:00
|
|
|
return err
|
2022-08-22 16:41:51 +00:00
|
|
|
}
|
2022-10-13 10:01:25 +00:00
|
|
|
tracing.End(span)
|
|
|
|
return nil
|
2022-08-22 16:41:51 +00:00
|
|
|
}
|
|
|
|
|
2022-08-25 11:34:19 +00:00
|
|
|
// Serve listens for responses to the requests until context is done
|
|
|
|
func (ip *icmpProxy) Serve(ctx context.Context) error {
|
2022-08-22 16:41:51 +00:00
|
|
|
go func() {
|
|
|
|
<-ctx.Done()
|
|
|
|
ip.conn.Close()
|
|
|
|
}()
|
2022-09-02 16:29:50 +00:00
|
|
|
go func() {
|
|
|
|
ip.srcFunnelTracker.ScheduleCleanup(ctx, ip.idleTimeout)
|
|
|
|
}()
|
2022-08-25 11:34:19 +00:00
|
|
|
buf := make([]byte, mtu)
|
2022-09-06 12:46:21 +00:00
|
|
|
icmpDecoder := packet.NewICMPDecoder()
|
2022-08-22 16:41:51 +00:00
|
|
|
for {
|
2022-09-06 12:46:21 +00:00
|
|
|
n, from, err := ip.conn.ReadFrom(buf)
|
2022-08-22 16:41:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-09-06 12:46:21 +00:00
|
|
|
reply, err := parseReply(from, buf[:n])
|
|
|
|
if err != nil {
|
|
|
|
ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply, continue to parse as full packet")
|
|
|
|
// In unit test, we found out when the listener listens on 0.0.0.0, the socket reads the full packet after
|
|
|
|
// the second reply
|
2022-10-14 13:44:17 +00:00
|
|
|
if err := ip.handleFullPacket(ctx, icmpDecoder, buf[:n]); err != nil {
|
2022-09-19 11:36:25 +00:00
|
|
|
ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply as full packet")
|
2022-09-06 12:46:21 +00:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if !isEchoReply(reply.msg) {
|
|
|
|
ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
|
|
|
|
continue
|
|
|
|
}
|
2022-10-14 13:44:17 +00:00
|
|
|
if err := ip.sendReply(ctx, reply); err != nil {
|
2022-11-14 11:22:38 +00:00
|
|
|
ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
|
2022-08-22 16:41:51 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-10-14 13:44:17 +00:00
|
|
|
func (ip *icmpProxy) handleFullPacket(ctx context.Context, decoder *packet.ICMPDecoder, rawPacket []byte) error {
|
2022-09-06 12:46:21 +00:00
|
|
|
icmpPacket, err := decoder.Decode(packet.RawPacket{Data: rawPacket})
|
2022-09-02 16:29:50 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2022-08-22 16:41:51 +00:00
|
|
|
}
|
2022-09-06 12:46:21 +00:00
|
|
|
echo, err := getICMPEcho(icmpPacket.Message)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
reply := echoReply{
|
|
|
|
from: icmpPacket.Src,
|
|
|
|
msg: icmpPacket.Message,
|
|
|
|
echo: echo,
|
|
|
|
}
|
2022-10-14 13:44:17 +00:00
|
|
|
if ip.sendReply(ctx, &reply); err != nil {
|
2022-09-06 12:46:21 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-10-14 13:44:17 +00:00
|
|
|
func (ip *icmpProxy) sendReply(ctx context.Context, reply *echoReply) error {
|
2022-09-09 15:48:42 +00:00
|
|
|
funnelID := echoFunnelID(reply.echo.ID)
|
|
|
|
funnel, ok := ip.srcFunnelTracker.Get(funnelID)
|
2022-09-06 12:46:21 +00:00
|
|
|
if !ok {
|
2022-09-02 16:29:50 +00:00
|
|
|
return packet.ErrFunnelNotFound
|
|
|
|
}
|
|
|
|
icmpFlow, err := toICMPEchoFlow(funnel)
|
2022-08-22 16:41:51 +00:00
|
|
|
if err != nil {
|
2022-09-02 16:29:50 +00:00
|
|
|
return err
|
2022-08-22 16:41:51 +00:00
|
|
|
}
|
2022-10-14 13:44:17 +00:00
|
|
|
|
|
|
|
_, span := icmpFlow.responder.replySpan(ctx, ip.logger)
|
|
|
|
defer icmpFlow.responder.exportSpan()
|
|
|
|
|
|
|
|
if err := icmpFlow.returnToSrc(reply); err != nil {
|
|
|
|
tracing.EndWithErrorStatus(span, err)
|
2024-01-15 16:49:17 +00:00
|
|
|
return err
|
2022-10-14 13:44:17 +00:00
|
|
|
}
|
2024-01-15 16:49:17 +00:00
|
|
|
observeICMPReply(ip.logger, span, reply.from.String(), reply.echo.ID, reply.echo.Seq)
|
|
|
|
span.SetAttributes(attribute.Int("originalEchoID", icmpFlow.originalEchoID))
|
2022-10-14 13:44:17 +00:00
|
|
|
tracing.End(span)
|
|
|
|
return nil
|
2022-09-02 16:29:50 +00:00
|
|
|
}
|