TUN-5362: Adjust route ip commands to be aware of virtual networks

This commit is contained in:
Nuno Diegues 2021-11-29 12:00:31 +00:00
parent eec6b87eea
commit 571380b3f5
7 changed files with 305 additions and 104 deletions

View File

@ -1,8 +1,6 @@
package tunnel
import (
"net"
"github.com/pkg/errors"
"github.com/cloudflare/cloudflared/teamnet"
@ -26,18 +24,18 @@ func (sc *subcommandContext) addRoute(newRoute teamnet.NewRoute) (teamnet.Route,
return client.AddRoute(newRoute)
}
func (sc *subcommandContext) deleteRoute(network net.IPNet) error {
func (sc *subcommandContext) deleteRoute(params teamnet.DeleteRouteParams) error {
client, err := sc.client()
if err != nil {
return errors.Wrap(err, noClientMsg)
}
return client.DeleteRoute(network)
return client.DeleteRoute(params)
}
func (sc *subcommandContext) getRouteByIP(ip net.IP) (teamnet.DetailedRoute, error) {
func (sc *subcommandContext) getRouteByIP(params teamnet.GetRouteByIpParams) (teamnet.DetailedRoute, error) {
client, err := sc.client()
if err != nil {
return teamnet.DetailedRoute{}, errors.Wrap(err, noClientMsg)
}
return client.GetByIP(ip)
return client.GetByIP(params)
}

View File

@ -6,6 +6,7 @@ import (
"os"
"text/tabwriter"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
@ -15,26 +16,45 @@ import (
"github.com/urfave/cli/v2"
)
var (
vnetFlag = &cli.StringFlag{
Name: "virtual-network",
Aliases: []string{"vn"},
Usage: "The ID or name of the virtual network to which the route is associated to.",
}
)
func buildRouteIPSubcommand() *cli.Command {
return &cli.Command{
Name: "ip",
Usage: "Configure and query Cloudflare WARP routing to services or private networks available through this tunnel.",
Usage: "Configure and query Cloudflare WARP routing to private IP networks made available through Cloudflare Tunnels.",
UsageText: "cloudflared tunnel [--config FILEPATH] route COMMAND [arguments...]",
Description: `cloudflared can provision private routes from any IP space to origins in your corporate network.
Users enrolled in your Cloudflare for Teams organization can reach those routes through the
Cloudflare WARP client. You can also build rules to determine who can reach certain routes.`,
Description: `cloudflared can provision routes for any IP space in your corporate network. Users enrolled in
your Cloudflare for Teams organization can reach those IPs through the Cloudflare WARP
client. You can then configure L7/L4 filtering on https://dash.teams.cloudflare.com to
determine who can reach certain routes.
By default IP routes all exist within a single virtual network. If you use the same IP
space(s) in different physical private networks, all meant to be reachable via IP routes,
then you have to manage the ambiguous IP routes by associating them to virtual networks.
See "cloudflared tunnel network --help" for more information.`,
Subcommands: []*cli.Command{
{
Name: "add",
Action: cliutil.ConfiguredAction(addRouteCommand),
Usage: "Add any new network to the routing table reachable via the tunnel",
UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
Description: `Adds any network route space (represented as a CIDR) to your routing table.
That network space becomes reachable for requests egressing from a user's machine
Usage: "Add a new network to the routing table reachable via a Tunnel",
UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [flags] [CIDR] [TUNNEL] [COMMENT?]",
Description: `Adds a network IP route space (represented as a CIDR) to your routing table.
That network IP space becomes reachable for requests egressing from a user's machine
as long as it is using Cloudflare WARP client and is enrolled in the same account
that is running the tunnel chosen here. Further, those requests will be proxied to
the specified tunnel, and reach an IP in the given CIDR, as long as that IP is
reachable from the tunnel.`,
that is running the Tunnel chosen here. Further, those requests will be proxied to
the specified Tunnel, and reach an IP in the given CIDR, as long as that IP is
reachable from cloudflared.
If the CIDR exists in more than one private network, to be connected with Cloudflare
Tunnels, then you have to manage those IP routes with virtual networks (see
"cloudflared tunnel network --help)". In those cases, you then have to tell
which virtual network's routing table you want to add the route to with:
"cloudflared tunnel route ip add --virtual-network [ID/name] [CIDR] [TUNNEL]".`,
Flags: []cli.Flag{vnetFlag},
},
{
Name: "show",
@ -49,17 +69,22 @@ reachable from the tunnel.`,
Name: "delete",
Action: cliutil.ConfiguredAction(deleteRouteCommand),
Usage: "Delete a row from your organization's private routing table",
UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
Description: `Deletes the row for a given CIDR from your routing table. That portion
of your network will no longer be reachable by the WARP clients.`,
UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [CIDR]",
Description: `Deletes the row for a given CIDR from your routing table. That portion of your network
will no longer be reachable by the WARP clients. Note that if you use virtual
networks, then you have to tell which virtual network whose routing table you
have a row deleted from.`,
Flags: []cli.Flag{vnetFlag},
},
{
Name: "get",
Action: cliutil.ConfiguredAction(getRouteByIPCommand),
Usage: "Check which row of the routing table matches a given IP.",
UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
Description: `Checks which row of the routing table will be used to proxy a given IP.
This helps check and validate your config.`,
UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [flags] [IP]",
Description: `Checks which row of the routing table will be used to proxy a given IP. This helps check
and validate your config. Note that if you use virtual networks, then you have
to tell which virtual network whose routing table you want to use.`,
Flags: []cli.Flag{vnetFlag},
},
},
}
@ -112,7 +137,9 @@ func addRouteCommand(c *cli.Context) error {
if c.NArg() < 2 {
return errors.New("You must supply at least 2 arguments, first the network you wish to route (in CIDR form e.g. 1.2.3.4/32) and then the tunnel ID to proxy with")
}
args := c.Args()
_, network, err := net.ParseCIDR(args.Get(0))
if err != nil {
return errors.Wrap(err, "Invalid network CIDR")
@ -120,19 +147,32 @@ func addRouteCommand(c *cli.Context) error {
if network == nil {
return errors.New("Invalid network CIDR")
}
tunnelRef := args.Get(1)
tunnelID, err := sc.findID(tunnelRef)
if err != nil {
return errors.Wrap(err, "Invalid tunnel")
}
comment := ""
if c.NArg() >= 3 {
comment = args.Get(2)
}
var vnetId *uuid.UUID
if c.IsSet(vnetFlag.Name) {
id, err := getVnetId(sc, c.String(vnetFlag.Name))
if err != nil {
return err
}
vnetId = &id
}
_, err = sc.addRoute(teamnet.NewRoute{
Comment: comment,
Network: *network,
TunnelID: tunnelID,
VNetID: vnetId,
})
if err != nil {
return errors.Wrap(err, "API error")
@ -146,9 +186,11 @@ func deleteRouteCommand(c *cli.Context) error {
if err != nil {
return err
}
if c.NArg() != 1 {
return errors.New("You must supply exactly one argument, the network whose route you want to delete (in CIDR form e.g. 1.2.3.4/32)")
}
_, network, err := net.ParseCIDR(c.Args().First())
if err != nil {
return errors.Wrap(err, "Invalid network CIDR")
@ -156,7 +198,20 @@ func deleteRouteCommand(c *cli.Context) error {
if network == nil {
return errors.New("Invalid network CIDR")
}
if err := sc.deleteRoute(*network); err != nil {
params := teamnet.DeleteRouteParams{
Network: *network,
}
if c.IsSet(vnetFlag.Name) {
vnetId, err := getVnetId(sc, c.String(vnetFlag.Name))
if err != nil {
return err
}
params.VNetID = &vnetId
}
if err := sc.deleteRoute(params); err != nil {
return errors.Wrap(err, "API error")
}
fmt.Printf("Successfully deleted route for %s\n", network)
@ -177,7 +232,20 @@ func getRouteByIPCommand(c *cli.Context) error {
if ip == nil {
return fmt.Errorf("Invalid IP %s", ipInput)
}
route, err := sc.getRouteByIP(ip)
params := teamnet.GetRouteByIpParams{
Ip: ip,
}
if c.IsSet(vnetFlag.Name) {
vnetId, err := getVnetId(sc, c.String(vnetFlag.Name))
if err != nil {
return err
}
params.VNetID = &vnetId
}
route, err := sc.getRouteByIP(params)
if err != nil {
return errors.Wrap(err, "API error")
}
@ -202,7 +270,7 @@ func formatAndPrintRouteList(routes []*teamnet.DetailedRoute) {
defer writer.Flush()
// Print column headers with tabbed columns
_, _ = fmt.Fprintln(writer, "NETWORK\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
_, _ = fmt.Fprintln(writer, "NETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
// Loop through routes, create formatted string for each, and print using tabwriter
for _, route := range routes {

View File

@ -16,11 +16,13 @@ import (
// network, and says that eyeballs can reach that route using the corresponding
// tunnel.
type Route struct {
Network CIDR `json:"network"`
TunnelID uuid.UUID `json:"tunnel_id"`
Comment string `json:"comment"`
CreatedAt time.Time `json:"created_at"`
DeletedAt time.Time `json:"deleted_at"`
Network CIDR `json:"network"`
TunnelID uuid.UUID `json:"tunnel_id"`
// Optional field. When unset, it means the Route belongs to the default virtual network.
VNetID *uuid.UUID `json:"virtual_network_id,omitempty"`
Comment string `json:"comment"`
CreatedAt time.Time `json:"created_at"`
DeletedAt time.Time `json:"deleted_at"`
}
// CIDR is just a newtype wrapper around net.IPNet. It adds JSON unmarshalling.
@ -62,27 +64,33 @@ type NewRoute struct {
Network net.IPNet
TunnelID uuid.UUID
Comment string
// Optional field. If unset, backend will assume the default vnet for the account.
VNetID *uuid.UUID
}
// MarshalJSON handles fields with non-JSON types (e.g. net.IPNet).
func (r NewRoute) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
TunnelID uuid.UUID `json:"tunnel_id"`
Comment string `json:"comment"`
TunnelID uuid.UUID `json:"tunnel_id"`
Comment string `json:"comment"`
VNetID *uuid.UUID `json:"virtual_network_id,omitempty"`
}{
TunnelID: r.TunnelID,
Comment: r.Comment,
VNetID: r.VNetID,
})
}
// DetailedRoute is just a Route with some extra fields, e.g. TunnelName.
type DetailedRoute struct {
Network CIDR `json:"network"`
TunnelID uuid.UUID `json:"tunnel_id"`
Comment string `json:"comment"`
CreatedAt time.Time `json:"created_at"`
DeletedAt time.Time `json:"deleted_at"`
TunnelName string `json:"tunnel_name"`
Network CIDR `json:"network"`
TunnelID uuid.UUID `json:"tunnel_id"`
// Optional field. When unset, it means the DetailedRoute belongs to the default virtual network.
VNetID *uuid.UUID `json:"virtual_network_id,omitempty"`
Comment string `json:"comment"`
CreatedAt time.Time `json:"created_at"`
DeletedAt time.Time `json:"deleted_at"`
TunnelName string `json:"tunnel_name"`
}
// IsZero checks if DetailedRoute is the zero value.
@ -97,9 +105,15 @@ func (r DetailedRoute) TableString() string {
if !r.DeletedAt.IsZero() {
deletedColumn = r.DeletedAt.Format(time.RFC3339)
}
vnetColumn := "default"
if r.VNetID != nil {
vnetColumn = r.VNetID.String()
}
return fmt.Sprintf(
"%s\t%s\t%s\t%s\t%s\t%s\t",
"%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
r.Network.String(),
vnetColumn,
r.Comment,
r.TunnelID,
r.TunnelName,
@ -107,3 +121,15 @@ func (r DetailedRoute) TableString() string {
deletedColumn,
)
}
type DeleteRouteParams struct {
Network net.IPNet
// Optional field. If unset, backend will assume the default vnet for the account.
VNetID *uuid.UUID
}
type GetRouteByIpParams struct {
Ip net.IP
// Optional field. If unset, backend will assume the default vnet for the account.
VNetID *uuid.UUID
}

View File

@ -12,77 +12,154 @@ import (
)
func TestUnmarshalRoute(t *testing.T) {
// Response from the teamnet route backend
data := `{
"network":"10.1.2.40/29",
"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
"comment":"test",
"created_at":"2020-12-22T02:00:15.587008Z",
"deleted_at":null
}`
var r Route
err := json.Unmarshal([]byte(data), &r)
testCases := []struct {
Json string
HasVnet bool
}{
{
`{
"network":"10.1.2.40/29",
"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
"comment":"test",
"created_at":"2020-12-22T02:00:15.587008Z",
"deleted_at":null
}`,
false,
},
{
`{
"network":"10.1.2.40/29",
"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
"comment":"test",
"created_at":"2020-12-22T02:00:15.587008Z",
"deleted_at":null,
"virtual_network_id":"38c95083-8191-4110-8339-3f438d44fdb9"
}`,
true,
},
}
// Check everything worked
require.NoError(t, err)
require.Equal(t, uuid.MustParse("fba6ffea-807f-4e7a-a740-4184ee1b82c8"), r.TunnelID)
require.Equal(t, "test", r.Comment)
_, cidr, err := net.ParseCIDR("10.1.2.40/29")
require.NoError(t, err)
require.Equal(t, CIDR(*cidr), r.Network)
require.Equal(t, "test", r.Comment)
for _, testCase := range testCases {
data := testCase.Json
var r Route
err := json.Unmarshal([]byte(data), &r)
// Check everything worked
require.NoError(t, err)
require.Equal(t, uuid.MustParse("fba6ffea-807f-4e7a-a740-4184ee1b82c8"), r.TunnelID)
require.Equal(t, "test", r.Comment)
_, cidr, err := net.ParseCIDR("10.1.2.40/29")
require.NoError(t, err)
require.Equal(t, CIDR(*cidr), r.Network)
require.Equal(t, "test", r.Comment)
if testCase.HasVnet {
require.Equal(t, uuid.MustParse("38c95083-8191-4110-8339-3f438d44fdb9"), *r.VNetID)
} else {
require.Nil(t, r.VNetID)
}
}
}
func TestDetailedRouteJsonRoundtrip(t *testing.T) {
// Response from the teamnet route backend
data := `{
"network":"10.1.2.40/29",
"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
"comment":"test",
"created_at":"2020-12-22T02:00:15.587008Z",
"deleted_at":"2021-01-14T05:01:42.183002Z",
"tunnel_name":"Mr. Tun"
}`
var r DetailedRoute
err := json.Unmarshal([]byte(data), &r)
testCases := []struct {
Json string
HasVnet bool
}{
{
`{
"network":"10.1.2.40/29",
"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
"comment":"test",
"created_at":"2020-12-22T02:00:15.587008Z",
"deleted_at":"2021-01-14T05:01:42.183002Z",
"tunnel_name":"Mr. Tun"
}`,
false,
},
{
`{
"network":"10.1.2.40/29",
"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
"virtual_network_id":"38c95083-8191-4110-8339-3f438d44fdb9",
"comment":"test",
"created_at":"2020-12-22T02:00:15.587008Z",
"deleted_at":"2021-01-14T05:01:42.183002Z",
"tunnel_name":"Mr. Tun"
}`,
true,
},
}
// Check everything worked
require.NoError(t, err)
require.Equal(t, uuid.MustParse("fba6ffea-807f-4e7a-a740-4184ee1b82c8"), r.TunnelID)
require.Equal(t, "test", r.Comment)
_, cidr, err := net.ParseCIDR("10.1.2.40/29")
require.NoError(t, err)
require.Equal(t, CIDR(*cidr), r.Network)
require.Equal(t, "test", r.Comment)
require.Equal(t, "Mr. Tun", r.TunnelName)
for _, testCase := range testCases {
data := testCase.Json
bytes, err := json.Marshal(r)
require.NoError(t, err)
obtainedJson := string(bytes)
data = strings.Replace(data, "\t", "", -1)
data = strings.Replace(data, "\n", "", -1)
require.Equal(t, data, obtainedJson)
var r DetailedRoute
err := json.Unmarshal([]byte(data), &r)
// Check everything worked
require.NoError(t, err)
require.Equal(t, uuid.MustParse("fba6ffea-807f-4e7a-a740-4184ee1b82c8"), r.TunnelID)
require.Equal(t, "test", r.Comment)
_, cidr, err := net.ParseCIDR("10.1.2.40/29")
require.NoError(t, err)
require.Equal(t, CIDR(*cidr), r.Network)
require.Equal(t, "test", r.Comment)
require.Equal(t, "Mr. Tun", r.TunnelName)
if testCase.HasVnet {
require.Equal(t, uuid.MustParse("38c95083-8191-4110-8339-3f438d44fdb9"), *r.VNetID)
} else {
require.Nil(t, r.VNetID)
}
bytes, err := json.Marshal(r)
require.NoError(t, err)
obtainedJson := string(bytes)
data = strings.Replace(data, "\t", "", -1)
data = strings.Replace(data, "\n", "", -1)
require.Equal(t, data, obtainedJson)
}
}
func TestMarshalNewRoute(t *testing.T) {
_, network, err := net.ParseCIDR("1.2.3.4/32")
require.NoError(t, err)
require.NotNil(t, network)
newRoute := NewRoute{
Network: *network,
TunnelID: uuid.New(),
Comment: "hi",
vnetId := uuid.New()
newRoutes := []NewRoute{
{
Network: *network,
TunnelID: uuid.New(),
Comment: "hi",
},
{
Network: *network,
TunnelID: uuid.New(),
Comment: "hi",
VNetID: &vnetId,
},
}
// Test where receiver is struct
serialized, err := json.Marshal(newRoute)
require.NoError(t, err)
require.True(t, strings.Contains(string(serialized), "tunnel_id"))
for _, newRoute := range newRoutes {
// Test where receiver is struct
serialized, err := json.Marshal(newRoute)
require.NoError(t, err)
require.True(t, strings.Contains(string(serialized), "tunnel_id"))
// Test where receiver is pointer to struct
serialized, err = json.Marshal(&newRoute)
require.NoError(t, err)
require.True(t, strings.Contains(string(serialized), "tunnel_id"))
// Test where receiver is pointer to struct
serialized, err = json.Marshal(&newRoute)
require.NoError(t, err)
require.True(t, strings.Contains(string(serialized), "tunnel_id"))
if newRoute.VNetID == nil {
require.False(t, strings.Contains(string(serialized), "virtual_network_id"))
} else {
require.True(t, strings.Contains(string(serialized), "virtual_network_id"))
}
}
}
func TestRouteTableString(t *testing.T) {

View File

@ -35,6 +35,11 @@ var (
Name: "filter-comment-is",
Usage: "Show only routes with this comment.",
}
filterVnet = cli.StringFlag{
Name: "filter-virtual-network-id",
Usage: "Show only routes that are attached to the given virtual network ID.",
}
// Flags contains all filter flags.
FilterFlags = []cli.Flag{
&filterDeleted,
@ -42,6 +47,7 @@ var (
&filterSubset,
&filterSuperset,
&filterComment,
&filterVnet,
}
)
@ -82,11 +88,19 @@ func NewFromCLI(c *cli.Context) (*Filter, error) {
if tunnelID := c.String(filterTunnelID.Name); tunnelID != "" {
u, err := uuid.Parse(tunnelID)
if err != nil {
return nil, errors.Wrap(err, "Couldn't parse UUID from --filter-tunnel-id")
return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterTunnelID.Name)
}
f.tunnelID(u)
}
if vnetId := c.String(filterVnet.Name); vnetId != "" {
u, err := uuid.Parse(vnetId)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterVnet.Name)
}
f.vnetID(u)
}
if maxFetch := c.Int("max-fetch-size"); maxFetch > 0 {
f.MaxFetchSize(uint(maxFetch))
}
@ -138,6 +152,10 @@ func (f *Filter) tunnelID(id uuid.UUID) {
f.queryParams.Set("tunnel_id", id.String())
}
func (f *Filter) vnetID(id uuid.UUID) {
f.queryParams.Set("virtual_network_id", id.String())
}
func (f *Filter) MaxFetchSize(max uint) {
f.queryParams.Set("per_page", strconv.Itoa(int(max)))
}

View File

@ -236,8 +236,8 @@ type Client interface {
// Teamnet endpoints
ListRoutes(filter *teamnet.Filter) ([]*teamnet.DetailedRoute, error)
AddRoute(newRoute teamnet.NewRoute) (teamnet.Route, error)
DeleteRoute(network net.IPNet) error
GetByIP(ip net.IP) (teamnet.DetailedRoute, error)
DeleteRoute(params teamnet.DeleteRouteParams) error
GetByIP(params teamnet.GetRouteByIpParams) (teamnet.DetailedRoute, error)
// Virtual Networks endpoints
CreateVirtualNetwork(newVnet vnet.NewVirtualNetwork) (vnet.VirtualNetwork, error)

View File

@ -2,11 +2,11 @@ package tunnelstore
import (
"io"
"net"
"net/http"
"net/url"
"path"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/cloudflare/cloudflared/teamnet"
@ -47,9 +47,11 @@ func (r *RESTClient) AddRoute(newRoute teamnet.NewRoute) (teamnet.Route, error)
}
// DeleteRoute calls the Tunnelstore DELETE endpoint for a given route.
func (r *RESTClient) DeleteRoute(network net.IPNet) error {
func (r *RESTClient) DeleteRoute(params teamnet.DeleteRouteParams) error {
endpoint := r.baseEndpoints.accountRoutes
endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(network.String()))
endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(params.Network.String()))
setVnetParam(&endpoint, params.VNetID)
resp, err := r.sendRequest("DELETE", endpoint, nil)
if err != nil {
return errors.Wrap(err, "REST request failed")
@ -65,9 +67,11 @@ func (r *RESTClient) DeleteRoute(network net.IPNet) error {
}
// GetByIP checks which route will proxy a given IP.
func (r *RESTClient) GetByIP(ip net.IP) (teamnet.DetailedRoute, error) {
func (r *RESTClient) GetByIP(params teamnet.GetRouteByIpParams) (teamnet.DetailedRoute, error) {
endpoint := r.baseEndpoints.accountRoutes
endpoint.Path = path.Join(endpoint.Path, "ip", url.PathEscape(ip.String()))
endpoint.Path = path.Join(endpoint.Path, "ip", url.PathEscape(params.Ip.String()))
setVnetParam(&endpoint, params.VNetID)
resp, err := r.sendRequest("GET", endpoint, nil)
if err != nil {
return teamnet.DetailedRoute{}, errors.Wrap(err, "REST request failed")
@ -98,3 +102,13 @@ func parseDetailedRoute(body io.ReadCloser) (teamnet.DetailedRoute, error) {
err := parseResponse(body, &route)
return route, err
}
// setVnetParam overwrites the URL's query parameters with a query param to scope the Route action to a certain
// virtual network (if one is provided).
func setVnetParam(endpoint *url.URL, vnetID *uuid.UUID) {
queryParams := url.Values{}
if vnetID != nil {
queryParams.Set("virtual_network_id", vnetID.String())
}
endpoint.RawQuery = queryParams.Encode()
}