TUN-6531: Implement ICMP proxy for Windows using IcmpSendEcho

ingress/icmp_generic.go

@@ -1,4 +1,4 @@
-//go:build !darwin && !linux
+//go:build !darwin && !linux && !windows
 
 package ingress
 

ingress/icmp_linux.go

@@ -58,12 +58,11 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FlowResponder) er
 	if pk == nil {
 		return errPacketNil
 	}
-	switch body := pk.Message.Body.(type) {
+	echo, err := getICMPEcho(pk)
-	case *icmp.Echo:
+	if err != nil {
-		return ip.sendICMPEchoRequest(pk, body, responder)
+		return err
-	default:
-		return fmt.Errorf("sending ICMP %s is not implemented", pk.Type)
 	}
+	return ip.sendICMPEchoRequest(pk, echo, responder)
 }
 
 func (ip *icmpProxy) Serve(ctx context.Context) error {

ingress/icmp_windows.go

@@ -0,0 +1,317 @@
+//go:build windows
+
+package ingress
+
+/*
+#include <iphlpapi.h>
+#include <icmpapi.h>
+*/
+import "C"
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"net/netip"
+	"runtime/debug"
+	"sync"
+	"syscall"
+	"unsafe"
+
+	"github.com/google/gopacket/layers"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+const (
+	icmpEchoReplyCode = 0
+)
+
+var (
+	Iphlpapi            = syscall.NewLazyDLL("Iphlpapi.dll")
+	IcmpCreateFile_proc = Iphlpapi.NewProc("IcmpCreateFile")
+	IcmpSendEcho_proc   = Iphlpapi.NewProc("IcmpSendEcho")
+	echoReplySize       = unsafe.Sizeof(echoReply{})
+	endian              = binary.LittleEndian
+)
+
+// IP_STATUS code, see https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmp_echo_reply32#members
+// for possible values
+type ipStatus uint32
+
+const (
+	success     ipStatus = 0
+	bufTooSmall          = iota + 11000
+	destNetUnreachable
+	destHostUnreachable
+	destProtocolUnreachable
+	destPortUnreachable
+	noResources
+	badOption
+	hwError
+	packetTooBig
+	reqTimedOut
+	badReq
+	badRoute
+	ttlExpiredTransit
+	ttlExpiredReassembly
+	paramProblem
+	sourceQuench
+	optionTooBig
+	badDestination
+	// Can be returned for malformed ICMP packets
+	generalFailure = 11050
+)
+
+func (is ipStatus) String() string {
+	switch is {
+	case success:
+		return "Success"
+	case bufTooSmall:
+		return "The reply buffer too small"
+	case destNetUnreachable:
+		return "The destination network was unreachable"
+	case destHostUnreachable:
+		return "The destination host was unreachable"
+	case destProtocolUnreachable:
+		return "The destination protocol was unreachable"
+	case destPortUnreachable:
+		return "The destination port was unreachable"
+	case noResources:
+		return "Insufficient IP resources were available"
+	case badOption:
+		return "A bad IP option was specified"
+	case hwError:
+		return "A hardware error occurred"
+	case packetTooBig:
+		return "The packet was too big"
+	case reqTimedOut:
+		return "The request timed out"
+	case badReq:
+		return "Bad request"
+	case badRoute:
+		return "Bad route"
+	case ttlExpiredTransit:
+		return "The TTL expired in transit"
+	case ttlExpiredReassembly:
+		return "The TTL expired during fragment reassembly"
+	case paramProblem:
+		return "A parameter problem"
+	case sourceQuench:
+		return "Datagrams are arriving too fast to be processed and datagrams may have been discarded"
+	case optionTooBig:
+		return "The IP option was too big"
+	case badDestination:
+		return "Bad destination"
+	case generalFailure:
+		return "The ICMP packet might be malformed"
+	default:
+		return fmt.Sprintf("Unknown ip status %d", is)
+	}
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-ip_option_information
+type ipOption struct {
+	TTL         uint8
+	Tos         uint8
+	Flags       uint8
+	OptionsSize uint8
+	OptionsData uintptr
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmp_echo_reply
+type echoReply struct {
+	Address       uint32
+	Status        ipStatus
+	RoundTripTime uint32
+	DataSize      uint16
+	Reserved      uint16
+	// The pointer size defers between 32-bit and 64-bit platforms
+	DataPointer *byte
+	Options     ipOption
+}
+
+type icmpProxy struct {
+	// An open handle that can send ICMP requests https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpcreatefile
+	handle uintptr
+	logger *zerolog.Logger
+	// A pool of reusable *packet.Encoder
+	encoderPool sync.Pool
+}
+
+func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger) (ICMPProxy, error) {
+	handle, _, err := IcmpCreateFile_proc.Call()
+	// Windows procedure calls always return non-nil error constructed from the result of GetLastError.
+	// Caller need to inspect the primary returned value
+	if syscall.Handle(handle) == syscall.InvalidHandle {
+		return nil, errors.Wrap(err, "invalid ICMP handle")
+	}
+	return &icmpProxy{
+		handle: handle,
+		logger: logger,
+		encoderPool: sync.Pool{
+			New: func() any {
+				return packet.NewEncoder()
+			},
+		},
+	}, nil
+}
+
+func (ip *icmpProxy) Serve(ctx context.Context) error {
+	<-ctx.Done()
+	syscall.CloseHandle(syscall.Handle(ip.handle))
+	return ctx.Err()
+}
+
+func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FlowResponder) error {
+	if pk == nil {
+		return errPacketNil
+	}
+	defer func() {
+		if r := recover(); r != nil {
+			ip.logger.Error().Interface("error", r).Msgf("Recover panic from sending icmp request/response, error %s", debug.Stack())
+		}
+	}()
+	echo, err := getICMPEcho(pk)
+	if err != nil {
+		return err
+	}
+
+	resp, err := ip.icmpSendEcho(pk.Dst, echo)
+	if err != nil {
+		return errors.Wrap(err, "failed to send/receive ICMP echo")
+	}
+
+	err = ip.handleEchoResponse(pk, echo, resp, responder)
+	if err != nil {
+		return errors.Wrap(err, "failed to handle ICMP echo reply")
+	}
+	return nil
+}
+
+func (ip *icmpProxy) handleEchoResponse(request *packet.ICMP, echoReq *icmp.Echo, resp *echoResp, responder packet.FlowResponder) error {
+	var replyType icmp.Type
+	if request.Dst.Is4() {
+		replyType = ipv4.ICMPTypeEchoReply
+	} else {
+		replyType = ipv6.ICMPTypeEchoReply
+	}
+
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      request.Dst,
+			Dst:      request.Src,
+			Protocol: layers.IPProtocol(request.Type.Protocol()),
+		},
+		Message: &icmp.Message{
+			Type: replyType,
+			Code: icmpEchoReplyCode,
+			Body: &icmp.Echo{
+				ID:   echoReq.ID,
+				Seq:  echoReq.Seq,
+				Data: resp.data,
+			},
+		},
+	}
+
+	serializedPacket, err := ip.encodeICMPReply(&pk)
+	if err != nil {
+		return err
+	}
+	return responder.SendPacket(serializedPacket)
+}
+
+func (ip *icmpProxy) encodeICMPReply(pk *packet.ICMP) (packet.RawPacket, error) {
+	cachedEncoder := ip.encoderPool.Get()
+	defer ip.encoderPool.Put(cachedEncoder)
+	encoder, ok := cachedEncoder.(*packet.Encoder)
+	if !ok {
+		return packet.RawPacket{}, fmt.Errorf("encoderPool returned %T, expect *packet.Encoder", cachedEncoder)
+	}
+	return encoder.Encode(pk)
+}
+
+/*
+	Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
+	Parameters:
+	- IcmpHandle:
+	- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
+	- RequestData: A pointer to echo data
+	- RequestSize: Number of bytes in buffer pointed by echo data
+	- RequestOptions: IP header options
+	- ReplyBuffer: A pointer to the buffer for echoReply, options and data
+	- ReplySize: Number of bytes allocated for ReplyBuffer
+	- Timeout: Timeout in milliseconds to wait for a reply
+	Returns:
+	- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
+	To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
+	syscall function
+*/
+func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, error) {
+	dataSize := len(echo.Data)
+	replySize := echoReplySize + uintptr(dataSize)
+	replyBuf := make([]byte, replySize)
+	noIPHeaderOption := uintptr(0)
+	inAddr, err := inAddrV4(dst)
+	if err != nil {
+		return nil, err
+	}
+	replyCount, _, err := IcmpSendEcho_proc.Call(ip.handle, uintptr(inAddr), uintptr(unsafe.Pointer(&echo.Data[0])),
+		uintptr(dataSize), noIPHeaderOption, uintptr(unsafe.Pointer(&replyBuf[0])),
+		replySize, icmpTimeoutMs)
+	if replyCount == 0 {
+		// status is returned in 5th to 8th byte of reply buffer
+		if status, err := unmarshalIPStatus(replyBuf[4:8]); err == nil {
+			return nil, fmt.Errorf("received ip status: %s", status)
+		}
+		return nil, errors.Wrap(err, "did not receive ICMP echo reply")
+	} else if replyCount > 1 {
+		ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount)
+	}
+	return newEchoResp(replyBuf)
+}
+
+type echoResp struct {
+	reply *echoReply
+	data  []byte
+}
+
+func newEchoResp(replyBuf []byte) (*echoResp, error) {
+	if len(replyBuf) == 0 {
+		return nil, fmt.Errorf("reply buffer is empty")
+	}
+	// This is pattern 1 of https://pkg.go.dev/unsafe@master#Pointer, conversion of *replyBuf to *echoReply
+	// replyBuf size is larger than echoReply
+	reply := *(*echoReply)(unsafe.Pointer(&replyBuf[0]))
+	if reply.Status != success {
+		return nil, fmt.Errorf("status %d", reply.Status)
+	}
+	dataBufStart := len(replyBuf) - int(reply.DataSize)
+	if dataBufStart < int(echoReplySize) {
+		return nil, fmt.Errorf("reply buffer size %d is too small to hold data of size %d", len(replyBuf), int(reply.DataSize))
+	}
+	return &echoResp{
+		reply: &reply,
+		data:  replyBuf[dataBufStart:],
+	}, nil
+}
+
+// Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32
+func inAddrV4(ip netip.Addr) (uint32, error) {
+	if !ip.Is4() {
+		return 0, fmt.Errorf("%s is not IPv4", ip)
+	}
+	v4 := ip.As4()
+	return endian.Uint32(v4[:]), nil
+}
+
+func unmarshalIPStatus(replyBuf []byte) (ipStatus, error) {
+	if len(replyBuf) != 4 {
+		return 0, fmt.Errorf("ipStatus needs to be 4 bytes, got %d", len(replyBuf))
+	}
+	return ipStatus(endian.Uint32(replyBuf)), nil
+}

ingress/icmp_windows_test.go

@@ -0,0 +1,139 @@
+//go:build windows
+
+package ingress
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net/netip"
+	"testing"
+	"unsafe"
+
+	"golang.org/x/net/icmp"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestParseEchoReply tests parsing raw bytes from icmpSendEcho into echoResp
+func TestParseEchoReply(t *testing.T) {
+	dst, err := inAddrV4(netip.MustParseAddr("192.168.10.20"))
+	require.NoError(t, err)
+
+	validReplyData := []byte(t.Name())
+	validReply := echoReply{
+		Address:       dst,
+		Status:        success,
+		RoundTripTime: uint32(20),
+		DataSize:      uint16(len(validReplyData)),
+		DataPointer:   &validReplyData[0],
+		Options: ipOption{
+			TTL: 59,
+		},
+	}
+
+	destHostUnreachableReply := validReply
+	destHostUnreachableReply.Status = destHostUnreachable
+
+	tests := []struct {
+		testCase      string
+		replyBuf      []byte
+		expectedReply *echoReply
+		expectedData  []byte
+	}{
+		{
+			testCase: "empty buffer",
+		},
+		{
+			testCase: "status not success",
+			replyBuf: destHostUnreachableReply.marshal(t, []byte{}),
+		},
+		{
+			testCase:      "valid reply",
+			replyBuf:      validReply.marshal(t, validReplyData),
+			expectedReply: &validReply,
+			expectedData:  validReplyData,
+		},
+	}
+
+	for _, test := range tests {
+		resp, err := newEchoResp(test.replyBuf)
+		if test.expectedReply == nil {
+			require.Error(t, err)
+			require.Nil(t, resp)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, resp.reply, test.expectedReply)
+			require.True(t, bytes.Equal(resp.data, test.expectedData))
+		}
+	}
+}
+
+//  TestSendEchoErrors makes sure icmpSendEcho handles error cases
+func TestSendEchoErrors(t *testing.T) {
+	proxy, err := newICMPProxy(localhostIP, &noopLogger)
+	require.NoError(t, err)
+	winProxy := proxy.(*icmpProxy)
+
+	echo := icmp.Echo{
+		ID:   6193,
+		Seq:  25712,
+		Data: []byte(t.Name()),
+	}
+	documentIP := netip.MustParseAddr("192.0.2.200")
+	resp, err := winProxy.icmpSendEcho(documentIP, &echo)
+	require.Error(t, err)
+	require.Nil(t, resp)
+}
+
+func (er *echoReply) marshal(t *testing.T, data []byte) []byte {
+	buf := new(bytes.Buffer)
+
+	for _, field := range []any{
+		er.Address,
+		er.Status,
+		er.RoundTripTime,
+		er.DataSize,
+		er.Reserved,
+	} {
+		require.NoError(t, binary.Write(buf, endian, field))
+	}
+
+	require.NoError(t, marshalPointer(buf, uintptr(unsafe.Pointer(er.DataPointer))))
+
+	for _, field := range []any{
+		er.Options.TTL,
+		er.Options.Tos,
+		er.Options.Flags,
+		er.Options.OptionsSize,
+	} {
+		require.NoError(t, binary.Write(buf, endian, field))
+	}
+
+	require.NoError(t, marshalPointer(buf, er.Options.OptionsData))
+
+	padSize := buf.Len() % int(unsafe.Alignof(er))
+	padding := make([]byte, padSize)
+	n, err := buf.Write(padding)
+	require.NoError(t, err)
+	require.Equal(t, padSize, n)
+
+	n, err = buf.Write(data)
+	require.NoError(t, err)
+	require.Equal(t, len(data), n)
+
+	return buf.Bytes()
+}
+
+func marshalPointer(buf io.Writer, ptr uintptr) error {
+	size := unsafe.Sizeof(ptr)
+	switch size {
+	case 4:
+		return binary.Write(buf, endian, uint32(ptr))
+	case 8:
+		return binary.Write(buf, endian, uint64(ptr))
+	default:
+		return fmt.Errorf("unexpected pointer size %d", size)
+	}
+}

ingress/origin_icmp_proxy.go

@@ -15,6 +15,7 @@ import (
 const (
 	defaultCloseAfterIdle = time.Second * 15
 	mtu                   = 1500
+	icmpTimeoutMs         = 1000
 )
 
 var (
@@ -42,3 +43,11 @@ func newICMPConn(listenIP netip.Addr) (*icmp.PacketConn, error) {
 	}
 	return icmp.ListenPacket(network, listenIP.String())
 }
+
+func getICMPEcho(pk *packet.ICMP) (*icmp.Echo, error) {
+	echo, ok := pk.Message.Body.(*icmp.Echo)
+	if !ok {
+		return nil, fmt.Errorf("expect ICMP echo, got %s", pk.Type)
+	}
+	return echo, nil
+}

ingress/origin_icmp_proxy_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net/netip"
-	"runtime"
 	"testing"
 
 	"github.com/google/gopacket/layers"
@@ -28,7 +27,6 @@ var (
 // is allowed in ping_group_range. See the following gist for how to do that:
 // https://github.com/ValentinBELYN/icmplib/blob/main/docs/6-use-icmplib-without-privileges.md
 func TestICMPProxyEcho(t *testing.T) {
-	onlyDarwinOrLinux(t)
 	const (
 		echoID = 36571
 		endSeq = 100
@@ -46,7 +44,7 @@ func TestICMPProxyEcho(t *testing.T) {
 
 	responder := echoFlowResponder{
 		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+		respChan: make(chan []byte, 1),
 	}
 
 	ip := packet.IP{
@@ -76,7 +74,6 @@ func TestICMPProxyEcho(t *testing.T) {
 
 // TestICMPProxyRejectNotEcho makes sure it rejects messages other than echo
 func TestICMPProxyRejectNotEcho(t *testing.T) {
-	onlyDarwinOrLinux(t)
 	msgs := []icmp.Message{
 		{
 			Type: ipv4.ICMPTypeDestinationUnreachable,
@@ -121,12 +118,6 @@ func TestICMPProxyRejectNotEcho(t *testing.T) {
 	}
 }
 
-func onlyDarwinOrLinux(t *testing.T) {
-	if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
-		t.Skip("Cannot create non-privileged datagram-oriented ICMP endpoint on Windows")
-	}
-}
-
 type echoFlowResponder struct {
 	decoder  *packet.ICMPDecoder
 	respChan chan []byte

quic/datagramv2.go

@@ -21,7 +21,7 @@ const (
 const (
 	typeIDLen = 1
 	// Same as sessionDemuxChan capacity
-	packetChanCapacity = 16
+	packetChanCapacity = 128
 )
 
 func suffixType(b []byte, datagramType datagramV2Type) ([]byte, error) {