diff --git a/ingress/icmp_windows.go b/ingress/icmp_windows.go index f5b5ac48..c1ebb511 100644 --- a/ingress/icmp_windows.go +++ b/ingress/icmp_windows.go @@ -29,15 +29,23 @@ import ( ) const ( + // Value defined in https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsasocketw + AF_INET6 = 23 icmpEchoReplyCode = 0 + nullParameter = uintptr(0) ) var ( - Iphlpapi = syscall.NewLazyDLL("Iphlpapi.dll") - IcmpCreateFile_proc = Iphlpapi.NewProc("IcmpCreateFile") - IcmpSendEcho_proc = Iphlpapi.NewProc("IcmpSendEcho") - echoReplySize = unsafe.Sizeof(echoReply{}) - endian = binary.LittleEndian + Iphlpapi = syscall.NewLazyDLL("Iphlpapi.dll") + IcmpCreateFile_proc = Iphlpapi.NewProc("IcmpCreateFile") + Icmp6CreateFile_proc = Iphlpapi.NewProc("Icmp6CreateFile") + IcmpSendEcho_proc = Iphlpapi.NewProc("IcmpSendEcho") + Icmp6SendEcho_proc = Iphlpapi.NewProc("Icmp6SendEcho2") + echoReplySize = unsafe.Sizeof(echoReply{}) + echoV6ReplySize = unsafe.Sizeof(echoV6Reply{}) + icmpv6ErrMessageSize = 8 + ioStatusBlockSize = unsafe.Sizeof(ioStatusBlock{}) + endian = binary.LittleEndian ) // IP_STATUS code, see https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmp_echo_reply32#members @@ -68,6 +76,16 @@ const ( generalFailure = 11050 ) +// Additional IP_STATUS codes for ICMPv6 https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmpv6_echo_reply_lh#members +const ( + ipv6DestUnreachable ipStatus = iota + 11040 + ipv6TimeExceeded + ipv6BadHeader + ipv6UnrecognizedNextHeader + ipv6ICMPError + ipv6DestScopeMismatch +) + func (is ipStatus) String() string { switch is { case success: @@ -108,6 +126,18 @@ func (is ipStatus) String() string { return "The IP option was too big" case badDestination: return "Bad destination" + case ipv6DestUnreachable: + return "IPv6 destination unreachable" + case ipv6TimeExceeded: + return "IPv6 time exceeded" + case ipv6BadHeader: + return "IPv6 bad IP header" + case ipv6UnrecognizedNextHeader: + return "IPv6 unrecognized next header" + case ipv6ICMPError: + return "IPv6 ICMP error" + case ipv6DestScopeMismatch: + return "IPv6 destination scope ID mismatch" case generalFailure: return "The ICMP packet might be malformed" default: @@ -136,27 +166,88 @@ type echoReply struct { Options ipOption } +type echoV6Reply struct { + Address ipv6AddrEx + Status ipStatus + RoundTripTime uint32 +} + +// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-ipv6_address_ex +// All the fields are in network byte order. The memory alignment is 4 bytes +type ipv6AddrEx struct { + port uint16 + // flowInfo is uint32. Because of field alignment, when we cast reply buffer to ipv6AddrEx, it starts at the 5th byte + // But looking at the raw bytes, flowInfo starts at the 3rd byte. We device flowInfo into 2 uint16 so it's aligned + flowInfoUpper uint16 + flowInfoLower uint16 + addr [8]uint16 + scopeID uint32 +} + +// https://docs.microsoft.com/en-us/windows/win32/winsock/sockaddr-2 +type sockAddrIn6 struct { + family int16 + // Can't embed ipv6AddrEx, that changes the memory alignment + port uint16 + flowInfo uint32 + addr [16]byte + scopeID uint32 +} + +func newSockAddrIn6(addr netip.Addr) (*sockAddrIn6, error) { + if !addr.Is6() { + return nil, fmt.Errorf("%s is not IPv6", addr) + } + return &sockAddrIn6{ + family: AF_INET6, + port: 10, + addr: addr.As16(), + }, nil +} + +// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_io_status_block#syntax +type ioStatusBlock struct { + // The first field is an union of NTSTATUS and PVOID. NTSTATUS is int32 while PVOID depends on the platform. + // We model it as uintptr whose size depends on if the platform is 32-bit or 64-bit + // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-erref/596a1078-e883-4972-9bbc-49e60bebca55 + statusOrPointer uintptr + information uintptr +} + type icmpProxy struct { // An open handle that can send ICMP requests https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpcreatefile handle uintptr - logger *zerolog.Logger + // This is a ICMPv6 if srcSocketAddr is not nil + srcSocketAddr *sockAddrIn6 + logger *zerolog.Logger // A pool of reusable *packet.Encoder encoderPool sync.Pool } func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger, idleTimeout time.Duration) (ICMPProxy, error) { - if listenIP.Is6() { - return nil, fmt.Errorf("ICMPv6 not implemented for Windows") + var ( + srcSocketAddr *sockAddrIn6 + handle uintptr + err error + ) + if listenIP.Is4() { + handle, _, err = IcmpCreateFile_proc.Call() + } else { + srcSocketAddr, err = newSockAddrIn6(listenIP) + if err != nil { + return nil, err + } + handle, _, err = Icmp6CreateFile_proc.Call() } - handle, _, err := IcmpCreateFile_proc.Call() // Windows procedure calls always return non-nil error constructed from the result of GetLastError. // Caller need to inspect the primary returned value if syscall.Handle(handle) == syscall.InvalidHandle { return nil, errors.Wrap(err, "invalid ICMP handle") } return &icmpProxy{ - handle: handle, - logger: logger, + handle: handle, + srcSocketAddr: srcSocketAddr, + logger: logger, encoderPool: sync.Pool{ New: func() any { return packet.NewEncoder() @@ -180,24 +271,25 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er ip.logger.Error().Interface("error", r).Msgf("Recover panic from sending icmp request/response, error %s", debug.Stack()) } }() + echo, err := getICMPEcho(pk.Message) if err != nil { return err } - - resp, err := ip.icmpSendEcho(pk.Dst, echo) + respData, err := ip.icmpEchoRoundtrip(pk.Dst, echo) if err != nil { - return errors.Wrap(err, "failed to send/receive ICMP echo") + ip.logger.Err(err).Msg("ICMP echo roundtrip failed") + return err } - err = ip.handleEchoResponse(pk, echo, resp, responder) + err = ip.handleEchoReply(pk, echo, respData, responder) if err != nil { return errors.Wrap(err, "failed to handle ICMP echo reply") } return nil } -func (ip *icmpProxy) handleEchoResponse(request *packet.ICMP, echoReq *icmp.Echo, resp *echoResp, responder packet.FunnelUniPipe) error { +func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, data []byte, responder packet.FunnelUniPipe) error { var replyType icmp.Type if request.Dst.Is4() { replyType = ipv4.ICMPTypeEchoReply @@ -217,7 +309,7 @@ func (ip *icmpProxy) handleEchoResponse(request *packet.ICMP, echoReq *icmp.Echo Body: &icmp.Echo{ ID: echoReq.ID, Seq: echoReq.Seq, - Data: resp.data, + Data: data, }, }, } @@ -239,10 +331,31 @@ func (ip *icmpProxy) encodeICMPReply(pk *packet.ICMP) (packet.RawPacket, error) return encoder.Encode(pk) } +func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) ([]byte, error) { + if dst.Is6() { + if ip.srcSocketAddr == nil { + return nil, fmt.Errorf("cannot send ICMPv6 using ICMPv4 proxy") + } + resp, err := ip.icmp6SendEcho(dst, echo) + if err != nil { + return nil, errors.Wrap(err, "failed to send/receive ICMPv6 echo") + } + return resp.data, nil + } + if ip.srcSocketAddr != nil { + return nil, fmt.Errorf("cannot send ICMPv4 using ICMPv6 proxy") + } + resp, err := ip.icmpSendEcho(dst, echo) + if err != nil { + return nil, errors.Wrap(err, "failed to send/receive ICMPv4 echo") + } + return resp.data, nil +} + /* Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho Parameters: - - IcmpHandle: + - IcmpHandle: Handle created by IcmpCreateFile - DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax - RequestData: A pointer to echo data - RequestSize: Number of bytes in buffer pointed by echo data @@ -259,18 +372,25 @@ func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, e dataSize := len(echo.Data) replySize := echoReplySize + uintptr(dataSize) replyBuf := make([]byte, replySize) - noIPHeaderOption := uintptr(0) + noIPHeaderOption := nullParameter inAddr, err := inAddrV4(dst) if err != nil { return nil, err } - replyCount, _, err := IcmpSendEcho_proc.Call(ip.handle, uintptr(inAddr), uintptr(unsafe.Pointer(&echo.Data[0])), - uintptr(dataSize), noIPHeaderOption, uintptr(unsafe.Pointer(&replyBuf[0])), - replySize, icmpRequestTimeoutMs) + replyCount, _, err := IcmpSendEcho_proc.Call( + ip.handle, + uintptr(inAddr), + uintptr(unsafe.Pointer(&echo.Data[0])), + uintptr(dataSize), + noIPHeaderOption, + uintptr(unsafe.Pointer(&replyBuf[0])), + replySize, + icmpRequestTimeoutMs, + ) if replyCount == 0 { // status is returned in 5th to 8th byte of reply buffer - if status, err := unmarshalIPStatus(replyBuf[4:8]); err == nil { - return nil, fmt.Errorf("received ip status: %s", status) + if status, parseErr := unmarshalIPStatus(replyBuf[4:8]); parseErr == nil && status != success { + return nil, errors.Wrapf(err, "received ip status: %s", status) } return nil, errors.Wrap(err, "did not receive ICMP echo reply") } else if replyCount > 1 { @@ -279,6 +399,15 @@ func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, e return newEchoResp(replyBuf) } +// Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32 +func inAddrV4(ip netip.Addr) (uint32, error) { + if !ip.Is4() { + return 0, fmt.Errorf("%s is not IPv4", ip) + } + v4 := ip.As4() + return endian.Uint32(v4[:]), nil +} + type echoResp struct { reply *echoReply data []byte @@ -304,13 +433,87 @@ func newEchoResp(replyBuf []byte) (*echoResp, error) { }, nil } -// Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32 -func inAddrV4(ip netip.Addr) (uint32, error) { - if !ip.Is4() { - return 0, fmt.Errorf("%s is not IPv4", ip) +/* + Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmp6sendecho2 + Parameters: + - IcmpHandle: Handle created by Icmp6CreateFile + - Event (optional): Event object to be signaled when a reply arrives + - ApcRoutine (optional): Routine to call when the calling thread is in an alertable thread and a reply arrives + - ApcContext (optional): Optional parameter to ApcRoutine + - SourceAddress: Source address of the request + - DestinationAddress: Destination address of the request + - RequestData: A pointer to echo data + - RequestSize: Number of bytes in buffer pointed by echo data + - RequestOptions (optional): A pointer to the IPv6 header options + - ReplyBuffer: A pointer to the buffer for echoReply, options and data + - ReplySize: Number of bytes allocated for ReplyBuffer + - Timeout: Timeout in milliseconds to wait for a reply + Returns: + - the number of replies in uint32 + To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the + syscall function +*/ + +func (ip *icmpProxy) icmp6SendEcho(dst netip.Addr, echo *icmp.Echo) (*echoV6Resp, error) { + dstAddr, err := newSockAddrIn6(dst) + if err != nil { + return nil, err } - v4 := ip.As4() - return endian.Uint32(v4[:]), nil + dataSize := len(echo.Data) + // Reply buffer needs to be big enough to hold an echoV6Reply, echo data, 8 bytes for ICMP error message + // and ioStatusBlock + replySize := echoV6ReplySize + uintptr(dataSize) + uintptr(icmpv6ErrMessageSize) + ioStatusBlockSize + replyBuf := make([]byte, replySize) + noEvent := nullParameter + noApcRoutine := nullParameter + noAppCtx := nullParameter + noIPHeaderOption := nullParameter + replyCount, _, err := Icmp6SendEcho_proc.Call( + ip.handle, + noEvent, + noApcRoutine, + noAppCtx, + uintptr(unsafe.Pointer(ip.srcSocketAddr)), + uintptr(unsafe.Pointer(dstAddr)), + uintptr(unsafe.Pointer(&echo.Data[0])), + uintptr(dataSize), + noIPHeaderOption, + uintptr(unsafe.Pointer(&replyBuf[0])), + replySize, + icmpRequestTimeoutMs, + ) + if replyCount == 0 { + // status is in the 4 bytes after ipv6AddrEx. The reply buffer size is at least size of ipv6AddrEx + 4 + if status, parseErr := unmarshalIPStatus(replyBuf[unsafe.Sizeof(ipv6AddrEx{}) : unsafe.Sizeof(ipv6AddrEx{})+4]); parseErr == nil && status != success { + return nil, fmt.Errorf("received ip status: %s", status) + } + return nil, errors.Wrap(err, "did not receive ICMP echo reply") + } else if replyCount > 1 { + ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount) + } + return newEchoV6Resp(replyBuf, dataSize) +} + +type echoV6Resp struct { + reply *echoV6Reply + data []byte +} + +func newEchoV6Resp(replyBuf []byte, dataSize int) (*echoV6Resp, error) { + if len(replyBuf) == 0 { + return nil, fmt.Errorf("reply buffer is empty") + } + reply := *(*echoV6Reply)(unsafe.Pointer(&replyBuf[0])) + if reply.Status != success { + return nil, fmt.Errorf("status %d", reply.Status) + } + if uintptr(len(replyBuf)) < unsafe.Sizeof(reply)+uintptr(dataSize) { + return nil, fmt.Errorf("reply buffer size %d is too small to hold reply size %d + data size %d", len(replyBuf), echoV6ReplySize, dataSize) + } + return &echoV6Resp{ + reply: &reply, + data: replyBuf[echoV6ReplySize : echoV6ReplySize+uintptr(dataSize)], + }, nil } func unmarshalIPStatus(replyBuf []byte) (ipStatus, error) { diff --git a/ingress/icmp_windows_test.go b/ingress/icmp_windows_test.go index d6aae071..5361ac44 100644 --- a/ingress/icmp_windows_test.go +++ b/ingress/icmp_windows_test.go @@ -71,9 +71,68 @@ func TestParseEchoReply(t *testing.T) { } } +// TestParseEchoV6Reply tests parsing raw bytes from icmp6SendEcho into echoV6Resp +func TestParseEchoV6Reply(t *testing.T) { + dst := netip.MustParseAddr("2606:3600:4500::3333").As16() + var addr [8]uint16 + for i := 0; i < 8; i++ { + addr[i] = binary.BigEndian.Uint16(dst[i*2 : i*2+2]) + } + + validReplyData := []byte(t.Name()) + validReply := echoV6Reply{ + Address: ipv6AddrEx{ + addr: addr, + }, + Status: success, + RoundTripTime: 25, + } + + destHostUnreachableReply := validReply + destHostUnreachableReply.Status = ipv6DestUnreachable + + tests := []struct { + testCase string + replyBuf []byte + expectedReply *echoV6Reply + expectedData []byte + }{ + { + testCase: "empty buffer", + }, + { + testCase: "status not success", + replyBuf: destHostUnreachableReply.marshal(t, []byte{}), + }, + { + testCase: "valid reply", + replyBuf: validReply.marshal(t, validReplyData), + expectedReply: &validReply, + expectedData: validReplyData, + }, + } + + for _, test := range tests { + resp, err := newEchoV6Resp(test.replyBuf, len(test.expectedData)) + if test.expectedReply == nil { + require.Error(t, err) + require.Nil(t, resp) + } else { + require.NoError(t, err) + require.Equal(t, resp.reply, test.expectedReply) + require.True(t, bytes.Equal(resp.data, test.expectedData)) + } + } +} + // TestSendEchoErrors makes sure icmpSendEcho handles error cases func TestSendEchoErrors(t *testing.T) { - proxy, err := newICMPProxy(localhostIP, &noopLogger, time.Second) + testSendEchoErrors(t, netip.IPv4Unspecified()) + testSendEchoErrors(t, netip.IPv6Unspecified()) +} + +func testSendEchoErrors(t *testing.T, listenIP netip.Addr) { + proxy, err := newICMPProxy(listenIP, &noopLogger, time.Second) require.NoError(t, err) winProxy := proxy.(*icmpProxy) @@ -83,7 +142,10 @@ func TestSendEchoErrors(t *testing.T) { Data: []byte(t.Name()), } documentIP := netip.MustParseAddr("192.0.2.200") - resp, err := winProxy.icmpSendEcho(documentIP, &echo) + if listenIP.Is6() { + documentIP = netip.MustParseAddr("2001:db8::1") + } + resp, err := winProxy.icmpEchoRoundtrip(documentIP, &echo) require.Error(t, err) require.Nil(t, resp) } @@ -138,3 +200,36 @@ func marshalPointer(buf io.Writer, ptr uintptr) error { return fmt.Errorf("unexpected pointer size %d", size) } } + +func (er *echoV6Reply) marshal(t *testing.T, data []byte) []byte { + buf := new(bytes.Buffer) + + for _, field := range []any{ + er.Address.port, + er.Address.flowInfoUpper, + er.Address.flowInfoLower, + er.Address.addr, + er.Address.scopeID, + } { + require.NoError(t, binary.Write(buf, endian, field)) + } + + padSize := buf.Len() % int(unsafe.Alignof(er)) + padding := make([]byte, padSize) + n, err := buf.Write(padding) + require.NoError(t, err) + require.Equal(t, padSize, n) + + for _, field := range []any{ + er.Status, + er.RoundTripTime, + } { + require.NoError(t, binary.Write(buf, endian, field)) + } + + n, err = buf.Write(data) + require.NoError(t, err) + require.Equal(t, len(data), n) + + return buf.Bytes() +} diff --git a/ingress/origin_icmp_proxy_test.go b/ingress/origin_icmp_proxy_test.go index d3573894..55379dcb 100644 --- a/ingress/origin_icmp_proxy_test.go +++ b/ingress/origin_icmp_proxy_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "net/netip" - "runtime" "strings" "testing" @@ -33,9 +32,6 @@ var ( // https://github.com/ValentinBELYN/icmplib/blob/main/docs/6-use-icmplib-without-privileges.md func TestICMPProxyEcho(t *testing.T) { testICMPProxyEcho(t, true) - if runtime.GOOS == "windows" { - t.Skip("TODO: TUN-6743: test ICMPv6 on Windows") - } testICMPProxyEcho(t, false) }