[PATCH 0/6] VXLAN fixes

[PATCH 0/6] VXLAN fixes

[Stephen Hemminger]

These fix some bugs found in followup testing with VXLAN.

[PATCH 1/6] vxlan: minor output refactoring

[Stephen Hemminger]

[-- Attachment #1: vxlan-dst.patch --]
[-- Type: text/plain, Size: 1722 bytes --]

Move code to find destination to a small function.

Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>

--- a/drivers/net/vxlan.c	2012-10-09 10:34:57.000000000 -0700
+++ b/drivers/net/vxlan.c	2012-10-09 10:48:56.546879617 -0700
@@ -621,6 +621,18 @@ static inline u8 vxlan_ecn_encap(u8 tos,
 	return INET_ECN_encapsulate(tos, inner);
 }
 
+static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
+{
+	const struct ethhdr *eth = (struct ethhdr *) skb->data;
+	struct vxlan_fdb *f;
+
+	if (is_multicast_ether_addr(eth->h_dest) ||
+	    (f = vxlan_find_mac(vxlan, eth->h_dest)) == NULL)
+		return vxlan->gaddr;
+	else
+		return f->remote_ip;
+}
+
 /* Transmit local packets over Vxlan
  *
  * Outer IP header inherits ECN and DF from inner header.
@@ -632,13 +644,11 @@ static netdev_tx_t vxlan_xmit(struct sk_
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct rtable *rt;
-	const struct ethhdr *eth;
 	const struct iphdr *old_iph;
 	struct iphdr *iph;
 	struct vxlanhdr *vxh;
 	struct udphdr *uh;
 	struct flowi4 fl4;
-	struct vxlan_fdb *f;
 	unsigned int pkt_len = skb->len;
 	u32 hash;
 	__be32 dst;
@@ -646,21 +656,16 @@ static netdev_tx_t vxlan_xmit(struct sk_
 	__u8 tos, ttl;
 	int err;
 
+	dst = vxlan_find_dst(vxlan, skb);
+	if (!dst)
+		goto drop;
+
 	/* Need space for new headers (invalidates iph ptr) */
 	if (skb_cow_head(skb, VXLAN_HEADROOM))
 		goto drop;
 
-	eth = (void *)skb->data;
 	old_iph = ip_hdr(skb);
 
-	if (!is_multicast_ether_addr(eth->h_dest) &&
-	    (f = vxlan_find_mac(vxlan, eth->h_dest)))
-		dst = f->remote_ip;
-	else if (vxlan->gaddr) {
-		dst = vxlan->gaddr;
-	} else
-		goto drop;
-
 	ttl = vxlan->ttl;
 	if (!ttl && IN_MULTICAST(ntohl(dst)))
 		ttl = 1;

[PATCH 2/6] vxlan: use ip_route_output

[Stephen Hemminger]

[-- Attachment #1: vxlan-route-output.patch --]
[-- Type: text/plain, Size: 1057 bytes --]

Select source address for VXLAN packet based on route destination
and don't lie to route code: VXLAN is not GRE.

Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>

--- a/drivers/net/vxlan.c	2012-10-09 10:48:56.546879617 -0700
+++ b/drivers/net/vxlan.c	2012-10-09 10:49:01.334832146 -0700
@@ -676,9 +676,11 @@ static netdev_tx_t vxlan_xmit(struct sk_
 
 	hash = skb_get_rxhash(skb);
 
-	rt = ip_route_output_gre(dev_net(dev), &fl4, dst,
-				 vxlan->saddr, vxlan->vni,
-				 RT_TOS(tos), vxlan->link);
+	fl4.flowi4_oif = vxlan->link;
+	fl4.flowi4_tos = RT_TOS(tos);
+	fl4.daddr = dst;
+	fl4.saddr = vxlan->saddr;
+	rt = ip_route_output_key(dev_net(dev), &fl4);
 	if (IS_ERR(rt)) {
 		netdev_dbg(dev, "no route to %pI4\n", &dst);
 		dev->stats.tx_carrier_errors++;
@@ -720,7 +722,7 @@ static netdev_tx_t vxlan_xmit(struct sk_
 	iph->frag_off	= df;
 	iph->protocol	= IPPROTO_UDP;
 	iph->tos	= vxlan_ecn_encap(tos, old_iph, skb);
-	iph->daddr	= fl4.daddr;
+	iph->daddr	= dst;
 	iph->saddr	= fl4.saddr;
 	iph->ttl	= ttl ? : ip4_dst_hoplimit(&rt->dst);
 

[PATCH 3/6] vxlan: associate with tunnel socket on xmit

[Stephen Hemminger]

[-- Attachment #1: vxlan-owner.patch --]
[-- Type: text/plain, Size: 1213 bytes --]

When tunnelling a skb, associate it with the tunnel socket.
This allows paramaters set on tunnel socket (like multicast loop
flag), to be picked up by ip_output.

Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>

--- a/drivers/net/vxlan.c	2012-10-09 10:49:01.334832146 -0700
+++ b/drivers/net/vxlan.c	2012-10-09 10:49:05.318792637 -0700
@@ -633,6 +633,23 @@ static __be32 vxlan_find_dst(struct vxla
 		return f->remote_ip;
 }
 
+static void vxlan_sock_free(struct sk_buff *skb)
+{
+	sock_put(skb->sk);
+}
+
+/* On transmit, associate with the tunnel socket */
+static void vxlan_set_owner(struct net_device *dev, struct sk_buff *skb)
+{
+	struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
+	struct sock *sk = vn->sock->sk;
+
+	skb_orphan(skb);
+	sock_hold(sk);
+	skb->sk = sk;
+	skb->destructor = vxlan_sock_free;
+}
+
 /* Transmit local packets over Vxlan
  *
  * Outer IP header inherits ECN and DF from inner header.
@@ -726,6 +743,8 @@ static netdev_tx_t vxlan_xmit(struct sk_
 	iph->saddr	= fl4.saddr;
 	iph->ttl	= ttl ? : ip4_dst_hoplimit(&rt->dst);
 
+	vxlan_set_owner(dev, skb);
+
 	/* See __IPTUNNEL_XMIT */
 	skb->ip_summed = CHECKSUM_NONE;
 	ip_select_ident(iph, &rt->dst, NULL);

[PATCH 4/6] VXLAN bases source UDP port based on flow to help the receiver to be able to load balance based on outer header flow.

[Stephen Hemminger]

[-- Attachment #1: vxlan-port-range.patch --]
[-- Type: text/plain, Size: 5757 bytes --]

This patch restricts the port range to the normal UDP local
ports, and allows overriding via configruation.

It also uses jhash of Ethernet header when looking at flows
with out know L3 header.

Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>

---
 drivers/net/vxlan.c     |   62 ++++++++++++++++++++++++++++++++++++++++++++----
 include/linux/if_link.h |    6 ++++
 2 files changed, 63 insertions(+), 5 deletions(-)

--- a/drivers/net/vxlan.c	2012-10-09 10:49:05.318792637 -0700
+++ b/drivers/net/vxlan.c	2012-10-09 10:49:08.238763697 -0700
@@ -106,6 +106,8 @@ struct vxlan_dev {
 	__be32	          gaddr;	/* multicast group */
 	__be32		  saddr;	/* source address */
 	unsigned int      link;		/* link to multicast over */
+	__u16		  port_min;	/* source port range */
+	__u16		  port_max;
 	__u8		  tos;		/* TOS override */
 	__u8		  ttl;
 	bool		  learn;
@@ -650,12 +652,29 @@ static void vxlan_set_owner(struct net_d
 	skb->destructor = vxlan_sock_free;
 }
 
+/* Compute source port for outgoing packet
+ *   first choice to use L4 flow hash since it will spread
+ *     better and maybe available from hardware
+ *   secondary choice is to use jhash on the Ethernet header
+ */
+static u16 vxlan_src_port(const struct vxlan_dev *vxlan, struct sk_buff *skb)
+{
+	unsigned int range = (vxlan->port_max - vxlan->port_min) + 1;
+	u32 hash;
+
+	hash = skb_get_rxhash(skb);
+	if (!hash)
+		hash = jhash(skb->data, 2 * ETH_ALEN,
+			     (__force u32) skb->protocol);
+
+	return (((u64) hash * range) >> 32) + vxlan->port_min;
+}
+
 /* Transmit local packets over Vxlan
  *
  * Outer IP header inherits ECN and DF from inner header.
  * Outer UDP destination is the VXLAN assigned port.
- *           source port is based on hash of flow if available
- *                       otherwise use a random value
+ *           source port is based on hash of flow
  */
 static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
 {
@@ -667,8 +686,8 @@ static netdev_tx_t vxlan_xmit(struct sk_
 	struct udphdr *uh;
 	struct flowi4 fl4;
 	unsigned int pkt_len = skb->len;
-	u32 hash;
 	__be32 dst;
+	__u16 src_port;
 	__be16 df = 0;
 	__u8 tos, ttl;
 	int err;
@@ -691,7 +710,7 @@ static netdev_tx_t vxlan_xmit(struct sk_
 	if (tos == 1)
 		tos = vxlan_get_dsfield(old_iph, skb);
 
-	hash = skb_get_rxhash(skb);
+	src_port = vxlan_src_port(vxlan, skb);
 
 	fl4.flowi4_oif = vxlan->link;
 	fl4.flowi4_tos = RT_TOS(tos);
@@ -726,7 +745,7 @@ static netdev_tx_t vxlan_xmit(struct sk_
 	uh = udp_hdr(skb);
 
 	uh->dest = htons(vxlan_port);
-	uh->source = hash ? :random32();
+	uh->source = htons(src_port);
 
 	uh->len = htons(skb->len);
 	uh->check = 0;
@@ -954,6 +973,7 @@ static void vxlan_setup(struct net_devic
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	unsigned h;
+	int low, high;
 
 	eth_hw_addr_random(dev);
 	ether_setup(dev);
@@ -973,6 +993,10 @@ static void vxlan_setup(struct net_devic
 	vxlan->age_timer.function = vxlan_cleanup;
 	vxlan->age_timer.data = (unsigned long) vxlan;
 
+	inet_get_local_port_range(&low, &high);
+	vxlan->port_min = low;
+	vxlan->port_max = high;
+
 	vxlan->dev = dev;
 
 	for (h = 0; h < FDB_HASH_SIZE; ++h)
@@ -989,6 +1013,7 @@ static const struct nla_policy vxlan_pol
 	[IFLA_VXLAN_LEARNING]	= { .type = NLA_U8 },
 	[IFLA_VXLAN_AGEING]	= { .type = NLA_U32 },
 	[IFLA_VXLAN_LIMIT]	= { .type = NLA_U32 },
+	[IFLA_VXLAN_PORT_RANGE] = { .len  = sizeof(struct ifla_vxlan_port_range) },
 };
 
 static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[])
@@ -1021,6 +1046,18 @@ static int vxlan_validate(struct nlattr
 			return -EADDRNOTAVAIL;
 		}
 	}
+
+	if (data[IFLA_VXLAN_PORT_RANGE]) {
+		const struct ifla_vxlan_port_range *p
+			= nla_data(data[IFLA_VXLAN_PORT_RANGE]);
+
+		if ((int)(ntohs(p->high) - ntohs(p->low)) < 1) {
+			pr_debug("port range %u .. %u not valid\n",
+				 ntohs(p->low), ntohs(p->high));
+			return -EINVAL;
+		}
+	}
+
 	return 0;
 }
 
@@ -1071,6 +1108,13 @@ static int vxlan_newlink(struct net *net
 	if (data[IFLA_VXLAN_LIMIT])
 		vxlan->addrmax = nla_get_u32(data[IFLA_VXLAN_LIMIT]);
 
+	if (data[IFLA_VXLAN_PORT_RANGE]) {
+		const struct ifla_vxlan_port_range *p
+			= nla_data(data[IFLA_VXLAN_PORT_RANGE]);
+		vxlan->port_min = ntohs(p->low);
+		vxlan->port_max = ntohs(p->high);
+	}
+
 	err = register_netdevice(dev);
 	if (!err)
 		hlist_add_head_rcu(&vxlan->hlist, vni_head(net, vxlan->vni));
@@ -1099,12 +1143,17 @@ static size_t vxlan_get_size(const struc
 		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_LEARNING */
 		nla_total_size(sizeof(__u32)) +	/* IFLA_VXLAN_AGEING */
 		nla_total_size(sizeof(__u32)) +	/* IFLA_VXLAN_LIMIT */
+		nla_total_size(sizeof(struct ifla_vxlan_port_range)) +
 		0;
 }
 
 static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
 	const struct vxlan_dev *vxlan = netdev_priv(dev);
+	struct ifla_vxlan_port_range ports = {
+		.low =  htons(vxlan->port_min),
+		.high = htons(vxlan->port_max),
+	};
 
 	if (nla_put_u32(skb, IFLA_VXLAN_ID, vxlan->vni))
 		goto nla_put_failure;
@@ -1125,6 +1174,9 @@ static int vxlan_fill_info(struct sk_buf
 	    nla_put_u32(skb, IFLA_VXLAN_LIMIT, vxlan->addrmax))
 		goto nla_put_failure;
 
+	if (nla_put(skb, IFLA_VXLAN_PORT_RANGE, sizeof(ports), &ports))
+		goto nla_put_failure;
+
 	return 0;
 
 nla_put_failure:
--- a/include/linux/if_link.h	2012-10-09 10:35:01.403159162 -0700
+++ b/include/linux/if_link.h	2012-10-09 10:49:08.238763697 -0700
@@ -284,10 +284,16 @@ enum {
 	IFLA_VXLAN_LEARNING,
 	IFLA_VXLAN_AGEING,
 	IFLA_VXLAN_LIMIT,
+	IFLA_VXLAN_PORT_RANGE,
 	__IFLA_VXLAN_MAX
 };
 #define IFLA_VXLAN_MAX	(__IFLA_VXLAN_MAX - 1)
 
+struct ifla_vxlan_port_range {
+	__be16	low;
+	__be16	high;
+};
+
 /* SR-IOV virtual function management section */
 
 enum {

[PATCH 5/6] vxlan: add additional headroom

[Stephen Hemminger]

[-- Attachment #1: vxlan-more-headroom.patch --]
[-- Type: text/plain, Size: 542 bytes --]

Tell upper layer protocols to allocate skb with additional headroom.
This avoids allocation and copy in local packet sends.

Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>

--- a/drivers/net/vxlan.c	2012-10-09 10:49:08.238763697 -0700
+++ b/drivers/net/vxlan.c	2012-10-09 10:49:19.830648777 -0700
@@ -977,6 +977,7 @@ static void vxlan_setup(struct net_devic
 
 	eth_hw_addr_random(dev);
 	ether_setup(dev);
+	dev->hard_header_len = ETH_HLEN + VXLAN_HEADROOM;
 
 	dev->netdev_ops = &vxlan_netdev_ops;
 	dev->destructor = vxlan_free;

[PATCH 6/6] vxlan: fix receive checksum handling

[Stephen Hemminger]

[-- Attachment #1: vxlan-checksum.patch --]
[-- Type: text/plain, Size: 1198 bytes --]

The VXLAN drive was trying to use postpull_rcsum to allow receive checksum
offload to work on drivers using CHECKSUM_COMPLETE method. But this
doesn't work correctly. Just force full receive checksum on received
packet.

Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>

--- a/drivers/net/vxlan.c	2012-10-09 10:49:19.830648777 -0700
+++ b/drivers/net/vxlan.c	2012-10-09 10:49:30.982538112 -0700
@@ -537,7 +537,6 @@ static int vxlan_udp_encap_recv(struct s
 	}
 
 	__skb_pull(skb, sizeof(struct vxlanhdr));
-	skb_postpull_rcsum(skb, eth_hdr(skb), sizeof(struct vxlanhdr));
 
 	/* Is this VNI defined? */
 	vni = ntohl(vxh->vx_vni) >> 8;
@@ -556,7 +555,6 @@ static int vxlan_udp_encap_recv(struct s
 	/* Re-examine inner Ethernet packet */
 	oip = ip_hdr(skb);
 	skb->protocol = eth_type_trans(skb, vxlan->dev);
-	skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
 
 	/* Ignore packet loops (and multicast echo) */
 	if (compare_ether_addr(eth_hdr(skb)->h_source,
@@ -568,6 +566,7 @@ static int vxlan_udp_encap_recv(struct s
 
 	__skb_tunnel_rx(skb, vxlan->dev);
 	skb_reset_network_header(skb);
+	skb->ip_summed = CHECKSUM_NONE;
 
 	err = IP_ECN_decapsulate(oip, skb);
 	if (unlikely(err)) {

Re: [PATCH 4/6] VXLAN bases source UDP port based on flow to help the receiver to be able to load balance based on outer header flow.

[Stephen Hemminger]

Commit message messed up by quilt on this one, do you want to fix
or should I resubmit?

Re: [PATCH 4/6] VXLAN bases source UDP port based on flow to help the receiver to be able to load balance based on outer header flow.

[David Miller]

From: Stephen Hemminger <shemminger@vyatta.com>
Date: Tue, 9 Oct 2012 11:07:54 -0700

> Commit message messed up by quilt on this one, do you want to fix
> or should I resubmit?

I can take care of it.

 

Re: [PATCH 1/6] vxlan: minor output refactoring

[Joe Perches]

On Tue, 2012-10-09 at 10:56 -0700, Stephen Hemminger wrote:
> +static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
> +{
> +       const struct ethhdr *eth = (struct ethhdr *) skb->data;
> +       struct vxlan_fdb *f;
> +
> +       if (is_multicast_ether_addr(eth->h_dest) ||
> +           (f = vxlan_find_mac(vxlan, eth->h_dest)) == NULL)
> +               return vxlan->gaddr;
> +       else
> +               return f->remote_ip;
> +}

Bikeshedding:

This might be simpler to read with a few more lines like:

static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
{
	const struct ethhdr *eth = (const struct ethhdr *)skb->data;
	struct vxlan_fdb *f;

	if (is_multicast_ether_addr(eth->h_dest))
		return vxlan->gaddr;

	f = vxlan_find_mac(vxlan, eth->h_dest);
	if (!f)
		return vxlan->gaddr;

	return f->remote_ip;
}

Re: [PATCH 1/6] vxlan: minor output refactoring

[Stephen Hemminger]

On Tue, 09 Oct 2012 11:27:55 -0700
Joe Perches <joe@perches.com> wrote:

> On Tue, 2012-10-09 at 10:56 -0700, Stephen Hemminger wrote:
> > +static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
> > +{
> > +       const struct ethhdr *eth = (struct ethhdr *) skb->data;
> > +       struct vxlan_fdb *f;
> > +
> > +       if (is_multicast_ether_addr(eth->h_dest) ||
> > +           (f = vxlan_find_mac(vxlan, eth->h_dest)) == NULL)
> > +               return vxlan->gaddr;
> > +       else
> > +               return f->remote_ip;
> > +}
> 
> Bikeshedding:
> 
> This might be simpler to read with a few more lines like:
> 
> static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
> {
> 	const struct ethhdr *eth = (const struct ethhdr *)skb->data;
> 	struct vxlan_fdb *f;
> 
> 	if (is_multicast_ether_addr(eth->h_dest))
> 		return vxlan->gaddr;
> 
> 	f = vxlan_find_mac(vxlan, eth->h_dest);
> 	if (!f)
> 		return vxlan->gaddr;
> 
> 	return f->remote_ip;
> }
> 

Maybe. Doesn't really matter that much.

Re: [PATCH 4/6] VXLAN bases source UDP port based on flow to help the receiver to be able to load balance based on outer header flow.

[Ben Hutchings]

On Tue, 2012-10-09 at 10:56 -0700, Stephen Hemminger wrote:
> This patch restricts the port range to the normal UDP local
> ports, and allows overriding via configruation.
> 
> It also uses jhash of Ethernet header when looking at flows
> with out know L3 header.
[...]
> +/* Compute source port for outgoing packet
> + *   first choice to use L4 flow hash since it will spread
> + *     better and maybe available from hardware
> + *   secondary choice is to use jhash on the Ethernet header
> + */
> +static u16 vxlan_src_port(const struct vxlan_dev *vxlan, struct sk_buff *skb)
> +{
> +	unsigned int range = (vxlan->port_max - vxlan->port_min) + 1;
> +	u32 hash;
> +
> +	hash = skb_get_rxhash(skb);
> +	if (!hash)
> +		hash = jhash(skb->data, 2 * ETH_ALEN,
> +			     (__force u32) skb->protocol);
> +
> +	return (((u64) hash * range) >> 32) + vxlan->port_min;
> +}
[...]
> @@ -1021,6 +1046,18 @@ static int vxlan_validate(struct nlattr
>  			return -EADDRNOTAVAIL;
>  		}
>  	}
> +
> +	if (data[IFLA_VXLAN_PORT_RANGE]) {
> +		const struct ifla_vxlan_port_range *p
> +			= nla_data(data[IFLA_VXLAN_PORT_RANGE]);
> +
> +		if ((int)(ntohs(p->high) - ntohs(p->low)) < 1) {
[...]

This seems to be off-by-one - both bounds are inclusive so they can be
equal.

Ben.

-- 
Ben Hutchings, Staff Engineer, Solarflare
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.

RE: [PATCH 1/6] vxlan: minor output refactoring

[David Laight]

> -----Original Message-----
> From: netdev-owner@vger.kernel.org [mailto:netdev-owner@vger.kernel.org] On Behalf Of Joe Perches
> Sent: 09 October 2012 19:28
> To: Stephen Hemminger
> Cc: David Miller; netdev@vger.kernel.org
> Subject: Re: [PATCH 1/6] vxlan: minor output refactoring
> 
> On Tue, 2012-10-09 at 10:56 -0700, Stephen Hemminger wrote:
> > +static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
> > +{
> > +       const struct ethhdr *eth = (struct ethhdr *) skb->data;
> > +       struct vxlan_fdb *f;
> > +
> > +       if (is_multicast_ether_addr(eth->h_dest) ||
> > +           (f = vxlan_find_mac(vxlan, eth->h_dest)) == NULL)
> > +               return vxlan->gaddr;
> > +       else
> > +               return f->remote_ip;
> > +}
> 
> Bikeshedding:
> 
> This might be simpler to read with a few more lines like:
> 
> static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
> {
> 	const struct ethhdr *eth = (const struct ethhdr *)skb->data;
> 	struct vxlan_fdb *f;
> 
> 	if (is_multicast_ether_addr(eth->h_dest))
> 		return vxlan->gaddr;
> 
> 	f = vxlan_find_mac(vxlan, eth->h_dest);
> 	if (!f)
> 		return vxlan->gaddr;
> 
> 	return f->remote_ip;
> }

or, painting it green:
static __be32 vxlan_find_dst(struct vxlan_dev *vxlan, struct sk_buff *skb)
{
       const struct ethhdr *eth = (struct ethhdr *)skb->data;
       struct vxlan_fdb *f;

       if (!is_multicast_ether_addr(eth->h_dest) &&
           (f = vxlan_find_mac(vxlan, eth->h_dest)) != NULL)
               return f->remote_ip;
       return vxlan->gaddr;
}

	David