All of lore.kernel.org
 help / color / mirror / Atom feed
From: Steffen Klassert <steffen.klassert@secunet.com>
To: <netdev@vger.kernel.org>
Cc: Steffen Klassert <steffen.klassert@secunet.com>,
	<sowmini.varadhan@oracle.com>
Subject: [PATCH RFC 08/13] esp4: Add a software GSO codepath.
Date: Thu, 4 Feb 2016 07:37:01 +0100	[thread overview]
Message-ID: <1454567826-13018-9-git-send-email-steffen.klassert@secunet.com> (raw)
In-Reply-To: <1454567826-13018-1-git-send-email-steffen.klassert@secunet.com>

This patch adds an esp4_gso_segment() callback and registers
functions for the new ESP encapsulation and crypto callbacks.

The work to get transport mode ready was done by
Sowmini Varadhan <sowmini.varadhan@oracle.com>

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 include/linux/skbuff.h  |  3 +-
 net/ipv4/af_inet.c      |  1 +
 net/ipv4/esp4.c         | 92 +++++++++++++++++++++++++++++++++++++++++++++++--
 net/ipv4/esp4_offload.c | 85 +++++++++++++++++++++++++++++++++++++++++++++
 net/ipv4/tcp_offload.c  |  1 +
 5 files changed, 178 insertions(+), 4 deletions(-)

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 4652f2c..dcc6c85 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -724,7 +724,8 @@ struct sk_buff {
 	__u8			inner_protocol_type:1;
 	__u8			remcsum_offload:1;
 	__u8			xfrm_gro:1;
-	/* 2 or 4 bit hole */
+	__u8			hw_xfrm:1;
+	/* 1 or 3 bit hole */
 
 #ifdef CONFIG_NET_SCHED
 	__u16			tc_index;	/* traffic control index */
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 5c5db66..ac6c1aa 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1220,6 +1220,7 @@ static struct sk_buff *inet_gso_segment(struct sk_buff *skb,
 		       SKB_GSO_UDP_TUNNEL |
 		       SKB_GSO_UDP_TUNNEL_CSUM |
 		       SKB_GSO_TUNNEL_REMCSUM |
+		       SKB_GSO_ESP |
 		       0)))
 		goto out;
 
diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index 4779374..550323d 100644
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -86,6 +86,15 @@ static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead,
 			     __alignof__(struct scatterlist));
 }
 
+static void esp_output_done2(struct crypto_async_request *base, int err)
+{
+	struct sk_buff *skb = base->data;
+
+	kfree(ESP_SKB_CB(skb)->tmp);
+
+	skb_dst(skb)->dev->xfrmdev_ops->xdo_dev_resume(skb, err);
+}
+
 static void esp_output_done(struct crypto_async_request *base, int err)
 {
 	struct sk_buff *skb = base->data;
@@ -118,6 +127,69 @@ static void esp_output_done_esn(struct crypto_async_request *base, int err)
 	esp_output_done(base, err);
 }
 
+static void esp4_gso_encap(struct xfrm_state *x, struct sk_buff *skb)
+{
+	struct ip_esp_hdr *esph;
+	struct iphdr *iph = ip_hdr(skb);
+	int proto = iph->protocol;
+
+	skb_push(skb, -skb_network_offset(skb));
+	esph = ip_esp_hdr(skb);
+	*skb_mac_header(skb) = IPPROTO_ESP;
+
+	esph->spi = x->id.spi;
+
+	/* save off the next_proto in seq_no to be used in
+	 * esp4_gso_encap() for invoking protocol specific
+	 * segmentation offload.
+	 */
+	esph->seq_no = proto;
+}
+
+static int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb)
+{
+	int err;
+	__be32 *seqhi;
+	int seqhilen;
+	u8 *iv;
+	struct crypto_aead *aead;
+	struct aead_request *req;
+	void *tmp;
+
+	aead = x->data;
+	tmp = ESP_SKB_CB(skb)->tmp;
+
+	seqhilen = 0;
+	if (x->props.flags & XFRM_STATE_ESN)
+		seqhilen += sizeof(__be32);
+
+	seqhi = esp_tmp_seqhi(tmp);
+	iv = esp_tmp_iv(aead, tmp, seqhilen);
+	req = esp_tmp_req(aead, iv);
+
+	aead_request_set_callback(req, 0, esp_output_done2, skb);
+
+	err = crypto_aead_encrypt(req);
+
+	switch (err) {
+	case -EINPROGRESS:
+		goto error;
+
+	case -EBUSY:
+		err = NET_XMIT_DROP;
+		break;
+
+	case 0:
+		if ((x->props.flags & XFRM_STATE_ESN))
+			esp_output_restore_header(skb);
+	}
+
+	kfree(tmp);
+
+error:
+	return err;
+}
+
 static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
 {
 	int err;
@@ -140,6 +212,7 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
 	int seqhilen;
 	__be32 *seqhi;
 	__be64 seqno;
+	int proto;
 
 	/* skb is pure payload to encrypt */
 
@@ -167,6 +240,7 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
 
 	assoclen = sizeof(*esph);
 	seqhilen = 0;
+	proto = ip_esp_hdr(skb)->seq_no;
 
 	if (x->props.flags & XFRM_STATE_ESN) {
 		seqhilen += sizeof(__be32);
@@ -196,12 +270,18 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
 			tail[i] = i + 1;
 	} while (0);
 	tail[plen - 2] = plen - 2;
-	tail[plen - 1] = *skb_mac_header(skb);
+	if (!skb->hw_xfrm)
+		tail[plen - 1] = *skb_mac_header(skb);
+	else
+		tail[plen - 1] = proto;
+
 	pskb_put(skb, trailer, clen - skb->len + alen);
 
 	skb_push(skb, -skb_network_offset(skb));
 	esph = ip_esp_hdr(skb);
-	*skb_mac_header(skb) = IPPROTO_ESP;
+
+	if (!skb->hw_xfrm)
+		*skb_mac_header(skb) = IPPROTO_ESP;
 
 	/* this is non-NULL only with UDP Encapsulation */
 	if (x->encap) {
@@ -271,6 +351,10 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
 	       min(ivlen, 8));
 
 	ESP_SKB_CB(skb)->tmp = tmp;
+
+	if (skb->hw_xfrm)
+		return 0;
+
 	err = crypto_aead_encrypt(req);
 
 	switch (err) {
@@ -735,7 +819,9 @@ static const struct xfrm_type esp_type =
 	.destructor	= esp_destroy,
 	.get_mtu	= esp4_get_mtu,
 	.input		= esp_input,
-	.output		= esp_output
+	.output		= esp_output,
+	.output_tail	= esp_output_tail,
+	.encap		= esp4_gso_encap,
 };
 
 static struct xfrm4_protocol esp4_protocol = {
diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c
index f2b0d6d..7c44c09 100644
--- a/net/ipv4/esp4_offload.c
+++ b/net/ipv4/esp4_offload.c
@@ -63,10 +63,95 @@ static int esp4_gro_complete(struct sk_buff *skb, int nhoff)
 	return err;
 }
 
+static struct sk_buff *esp4_gso_segment(struct sk_buff *skb,
+				        netdev_features_t features)
+{
+	struct ip_esp_hdr *esph;
+	struct sk_buff *skb2;
+	struct sk_buff *segs = ERR_PTR(-EINVAL);
+	struct dst_entry *dst = skb_dst(skb);
+	struct xfrm_state *x;
+	struct crypto_aead *aead;
+	int err = 0;
+	const struct net_offload *ops;
+	int proto;
+
+	if (!dst || !dst->xfrm)
+		goto out;
+
+	x = dst->xfrm;
+	aead = x->data;
+	esph = ip_esp_hdr(skb);
+
+	proto = esph->seq_no;
+	if (esph->spi != x->id.spi)
+		goto out;
+
+	if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))
+		goto out;
+
+	__skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));
+
+	skb->encap_hdr_csum = 1;
+
+	if (proto == IPPROTO_IPIP) {
+		__skb_push(skb, skb->mac_len);
+		segs = skb_mac_gso_segment(skb, features);
+	} else {
+		skb->transport_header += x->props.header_len;
+		ops = rcu_dereference(inet_offloads[proto]);
+		if (likely(ops && ops->callbacks.gso_segment))
+			segs = ops->callbacks.gso_segment(skb, features);
+	}
+	if (IS_ERR(segs))
+		goto out;
+	if (segs == NULL)
+		return ERR_PTR(-EINVAL);
+	__skb_pull(skb, skb->data - skb_mac_header(skb));
+
+	skb2 = segs;
+	do {
+		struct sk_buff *nskb = skb2->next;
+
+		if (proto == IPPROTO_IPIP) {
+			skb2->network_header = skb2->network_header - x->props.header_len;
+			skb2->transport_header = skb2->network_header + sizeof(struct iphdr);
+			skb_reset_mac_len(skb2);
+			skb_pull(skb2, skb2->mac_len + x->props.header_len);
+		} else {
+			/* skb2 mac and data are pointing at the start of
+			 * mac address. Pull data forward to point to tcp hdr
+			 */
+			 __skb_pull(skb2, skb2->transport_header - skb2->mac_header);
+
+			 /* move transport_header to point to esp header */
+			 skb2->transport_header -= x->props.header_len;
+		}
+
+		/* Set up eshp->seq_no to be used by esp_output()
+		 * for initializing trailer.
+		 */
+		ip_esp_hdr(skb2)->seq_no = proto;
+
+		err = dst->dev->xfrmdev_ops->xdo_dev_prepare(skb2);
+		if (err) {
+			kfree_skb_list(segs);
+			return ERR_PTR(err);
+		}
+
+		skb_push(skb2, skb2->mac_len);
+		skb2 = nskb;
+	} while (skb2);
+
+out:
+	return segs;
+}
+
 static const struct net_offload esp4_offload = {
 	.callbacks = {
 		.gro_receive = esp4_gro_receive,
 		.gro_complete = esp4_gro_complete,
+		.gso_segment = esp4_gso_segment,
 	},
 };
 
diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c
index 9864a2d..e67981e 100644
--- a/net/ipv4/tcp_offload.c
+++ b/net/ipv4/tcp_offload.c
@@ -97,6 +97,7 @@ struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
 			       SKB_GSO_UDP_TUNNEL |
 			       SKB_GSO_UDP_TUNNEL_CSUM |
 			       SKB_GSO_TUNNEL_REMCSUM |
+			       SKB_GSO_ESP |
 			       0) ||
 			     !(type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))))
 			goto out;
-- 
1.9.1

  parent reply	other threads:[~2016-02-04  7:05 UTC|newest]

Thread overview: 14+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2016-02-04  6:36 [PATCH RFC] IPsec performance improvements (discussion base for the IPsec performance BoF) Steffen Klassert
2016-02-04  6:36 ` [PATCH RFC 01/13] net: allow to leave the buffer fragmented in skb_cow_data() Steffen Klassert
2016-02-04  6:36 ` [PATCH RFC 02/13] gro: Partly revert "net: gro: allow to build full sized skb" Steffen Klassert
2016-02-04  6:36 ` [PATCH RFC 03/13] esp: Add a software GRO codepath Steffen Klassert
2016-02-04  6:36 ` [PATCH RFC 04/13] xfrm: Move device notifications to a sepatate file Steffen Klassert
2016-02-04  6:36 ` [PATCH RFC 05/13] xfrm: Add callbacks for IPsec GSO offloading Steffen Klassert
2016-02-04  6:36 ` [PATCH RFC 06/13] net: Add xfrm offload callbacks to struct net_device Steffen Klassert
2016-02-04  6:37 ` [PATCH RFC 07/13] net: Add ESP offload features Steffen Klassert
2016-02-04  6:37 ` Steffen Klassert [this message]
2016-02-04  6:37 ` [PATCH RFC 09/13] esp: Avoid skb_cow_data whenever possible Steffen Klassert
2016-02-04  6:37 ` [PATCH RFC 10/13] xfrm: Add basic infrastructure for IPsec device offloading Steffen Klassert
2016-02-04  6:37 ` [PATCH RFC 11/13] net: Enable IPsec software GSO Steffen Klassert
2016-02-04  6:37 ` [PATCH RFC 12/13] crypto: Make the page handling of hash walk compatible to networking Steffen Klassert
2016-02-04  6:37 ` [PATCH RFC 13/13] net: Allow IPsec GSO for locally sent traffic Steffen Klassert

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1454567826-13018-9-git-send-email-steffen.klassert@secunet.com \
    --to=steffen.klassert@secunet.com \
    --cc=netdev@vger.kernel.org \
    --cc=sowmini.varadhan@oracle.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.