[PATCHv1 0/2 net] xen-netfront: fix resume regressions in 3.16-rc1

[PATCHv1 0/2 net] xen-netfront: fix resume regressions in 3.16-rc1

[David Vrabel]

The introduction of multi-queue support to xen-netfront in 3.16-rc1,
broke resume/migration.

David

[PATCHv1 0/2 net] xen-netfront: fix resume regressions in 3.16-rc1

[David Vrabel]

The introduction of multi-queue support to xen-netfront in 3.16-rc1,
broke resume/migration.

David

[PATCH 1/2] xen-netfront: fix oops when disconnected from backend

[David Vrabel]

xennet_disconnect_backend() was not correctly iterating over all the
queues.

Signed-off-by: David Vrabel <david.vrabel@citrix.com>
---
 drivers/net/xen-netfront.c |    5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index 5a7872a..daaf1e5 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -1287,7 +1287,7 @@ static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
 
 	if (likely(netif_carrier_ok(dev) &&
 		   RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)))
-			napi_schedule(&queue->napi);
+		napi_schedule(&queue->napi);
 
 	return IRQ_HANDLED;
 }
@@ -1437,10 +1437,11 @@ static void xennet_end_access(int ref, void *page)
 static void xennet_disconnect_backend(struct netfront_info *info)
 {
 	unsigned int i = 0;
-	struct netfront_queue *queue = NULL;
 	unsigned int num_queues = info->netdev->real_num_tx_queues;
 
 	for (i = 0; i < num_queues; ++i) {
+		struct netfront_queue *queue = &info->queues[i];
+
 		/* Stop old i/f to prevent errors whilst we rebuild the state. */
 		spin_lock_bh(&queue->rx_lock);
 		spin_lock_irq(&queue->tx_lock);
-- 
1.7.10.4

[PATCH 1/2] xen-netfront: fix oops when disconnected from backend

[David Vrabel]

xennet_disconnect_backend() was not correctly iterating over all the
queues.

Signed-off-by: David Vrabel <david.vrabel@citrix.com>
---
 drivers/net/xen-netfront.c |    5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index 5a7872a..daaf1e5 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -1287,7 +1287,7 @@ static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
 
 	if (likely(netif_carrier_ok(dev) &&
 		   RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)))
-			napi_schedule(&queue->napi);
+		napi_schedule(&queue->napi);
 
 	return IRQ_HANDLED;
 }
@@ -1437,10 +1437,11 @@ static void xennet_end_access(int ref, void *page)
 static void xennet_disconnect_backend(struct netfront_info *info)
 {
 	unsigned int i = 0;
-	struct netfront_queue *queue = NULL;
 	unsigned int num_queues = info->netdev->real_num_tx_queues;
 
 	for (i = 0; i < num_queues; ++i) {
+		struct netfront_queue *queue = &info->queues[i];
+
 		/* Stop old i/f to prevent errors whilst we rebuild the state. */
 		spin_lock_bh(&queue->rx_lock);
 		spin_lock_irq(&queue->tx_lock);
-- 
1.7.10.4

[PATCH 2/2] xen-netfront: recreate queues correctly when reconnecting

[David Vrabel]

When reconnecting to the backend (after a resume/migration, for example),
a different number of queues may be required (since the guest may have
moved to a different host with different capabilities).  During the
reconnection the old queues are torn down and new ones created.

Introduce xennet_create_queues() and xennet_destroy_queues() that fixes
three bugs during the reconnection.

- The old info->queues was leaked.
- The old queue's napi instances were not deleted.
- The new queue's napi instances were left disabled (which meant no
  packets could be received).

The xennet_destroy_queues() calls is deferred until the reconnection
instead of the disconnection (in xennet_disconnect_backend()) because
napi_disable() might sleep.

Signed-off-by: David Vrabel <david.vrabel@citrix.com>
---
 drivers/net/xen-netfront.c |  104 ++++++++++++++++++++++++++++++--------------
 1 file changed, 72 insertions(+), 32 deletions(-)

diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index daaf1e5..2ccb4a0 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -1699,8 +1699,6 @@ static int xennet_init_queue(struct netfront_queue *queue)
 		goto exit_free_tx;
 	}
 
-	netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll, 64);
-
 	return 0;
 
  exit_free_tx:
@@ -1791,6 +1789,70 @@ error:
 	return err;
 }
 
+static void xennet_destroy_queues(struct netfront_info *info)
+{
+	unsigned int i;
+
+	rtnl_lock();
+
+	for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
+		struct netfront_queue *queue = &info->queues[i];
+
+		if (netif_running(info->netdev))
+			napi_disable(&queue->napi);
+		netif_napi_del(&queue->napi);
+	}
+
+	rtnl_unlock();
+
+	kfree(info->queues);
+	info->queues = NULL;
+}
+
+static int xennet_create_queues(struct netfront_info *info,
+				unsigned int num_queues)
+{
+	unsigned int i;
+	int ret;
+
+	info->queues = kcalloc(num_queues, sizeof(struct netfront_queue),
+			       GFP_KERNEL);
+	if (!info->queues)
+		return -ENOMEM;
+
+	rtnl_lock();
+
+	for (i = 0; i < num_queues; i++) {
+		struct netfront_queue *queue = &info->queues[i];
+
+		queue->id = i;
+		queue->info = info;
+
+		ret = xennet_init_queue(queue);
+		if (ret < 0) {
+			dev_warn(&info->netdev->dev, "only created %d queues\n",
+				 num_queues);
+			num_queues = i;
+			break;
+		}
+
+		netif_napi_add(queue->info->netdev, &queue->napi,
+			       xennet_poll, 64);
+		if (netif_running(info->netdev))
+			napi_enable(&queue->napi);
+	}
+
+	netif_set_real_num_tx_queues(info->netdev, num_queues);
+
+	rtnl_unlock();
+
+	if (num_queues == 0) {
+		dev_err(&info->netdev->dev, "no queues\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
 /* Common code used when first setting up, and when resuming. */
 static int talk_to_netback(struct xenbus_device *dev,
 			   struct netfront_info *info)
@@ -1827,42 +1889,20 @@ static int talk_to_netback(struct xenbus_device *dev,
 		goto out;
 	}
 
-	/* Allocate array of queues */
-	info->queues = kcalloc(num_queues, sizeof(struct netfront_queue), GFP_KERNEL);
-	if (!info->queues) {
-		err = -ENOMEM;
-		goto out;
-	}
-	rtnl_lock();
-	netif_set_real_num_tx_queues(info->netdev, num_queues);
-	rtnl_unlock();
+	if (info->queues)
+		xennet_destroy_queues(info);
+
+	err = xennet_create_queues(info, num_queues);
+	if (err < 0)
+		goto destroy_ring;
 
 	/* Create shared ring, alloc event channel -- for each queue */
 	for (i = 0; i < num_queues; ++i) {
 		queue = &info->queues[i];
-		queue->id = i;
-		queue->info = info;
-		err = xennet_init_queue(queue);
-		if (err) {
-			/* xennet_init_queue() cleans up after itself on failure,
-			 * but we still have to clean up any previously initialised
-			 * queues. If i > 0, set num_queues to i, then goto
-			 * destroy_ring, which calls xennet_disconnect_backend()
-			 * to tidy up.
-			 */
-			if (i > 0) {
-				rtnl_lock();
-				netif_set_real_num_tx_queues(info->netdev, i);
-				rtnl_unlock();
-				goto destroy_ring;
-			} else {
-				goto out;
-			}
-		}
 		err = setup_netfront(dev, queue, feature_split_evtchn);
 		if (err) {
-			/* As for xennet_init_queue(), setup_netfront() will tidy
-			 * up the current queue on error, but we need to clean up
+			/* setup_netfront() will tidy up the current
+			 * queue on error, but we need to clean up
 			 * those already allocated.
 			 */
 			if (i > 0) {
-- 
1.7.10.4

[PATCH 2/2] xen-netfront: recreate queues correctly when reconnecting

[David Vrabel]

When reconnecting to the backend (after a resume/migration, for example),
a different number of queues may be required (since the guest may have
moved to a different host with different capabilities).  During the
reconnection the old queues are torn down and new ones created.

Introduce xennet_create_queues() and xennet_destroy_queues() that fixes
three bugs during the reconnection.

- The old info->queues was leaked.
- The old queue's napi instances were not deleted.
- The new queue's napi instances were left disabled (which meant no
  packets could be received).

The xennet_destroy_queues() calls is deferred until the reconnection
instead of the disconnection (in xennet_disconnect_backend()) because
napi_disable() might sleep.

Signed-off-by: David Vrabel <david.vrabel@citrix.com>
---
 drivers/net/xen-netfront.c |  104 ++++++++++++++++++++++++++++++--------------
 1 file changed, 72 insertions(+), 32 deletions(-)

diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index daaf1e5..2ccb4a0 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -1699,8 +1699,6 @@ static int xennet_init_queue(struct netfront_queue *queue)
 		goto exit_free_tx;
 	}
 
-	netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll, 64);
-
 	return 0;
 
  exit_free_tx:
@@ -1791,6 +1789,70 @@ error:
 	return err;
 }
 
+static void xennet_destroy_queues(struct netfront_info *info)
+{
+	unsigned int i;
+
+	rtnl_lock();
+
+	for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
+		struct netfront_queue *queue = &info->queues[i];
+
+		if (netif_running(info->netdev))
+			napi_disable(&queue->napi);
+		netif_napi_del(&queue->napi);
+	}
+
+	rtnl_unlock();
+
+	kfree(info->queues);
+	info->queues = NULL;
+}
+
+static int xennet_create_queues(struct netfront_info *info,
+				unsigned int num_queues)
+{
+	unsigned int i;
+	int ret;
+
+	info->queues = kcalloc(num_queues, sizeof(struct netfront_queue),
+			       GFP_KERNEL);
+	if (!info->queues)
+		return -ENOMEM;
+
+	rtnl_lock();
+
+	for (i = 0; i < num_queues; i++) {
+		struct netfront_queue *queue = &info->queues[i];
+
+		queue->id = i;
+		queue->info = info;
+
+		ret = xennet_init_queue(queue);
+		if (ret < 0) {
+			dev_warn(&info->netdev->dev, "only created %d queues\n",
+				 num_queues);
+			num_queues = i;
+			break;
+		}
+
+		netif_napi_add(queue->info->netdev, &queue->napi,
+			       xennet_poll, 64);
+		if (netif_running(info->netdev))
+			napi_enable(&queue->napi);
+	}
+
+	netif_set_real_num_tx_queues(info->netdev, num_queues);
+
+	rtnl_unlock();
+
+	if (num_queues == 0) {
+		dev_err(&info->netdev->dev, "no queues\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
 /* Common code used when first setting up, and when resuming. */
 static int talk_to_netback(struct xenbus_device *dev,
 			   struct netfront_info *info)
@@ -1827,42 +1889,20 @@ static int talk_to_netback(struct xenbus_device *dev,
 		goto out;
 	}
 
-	/* Allocate array of queues */
-	info->queues = kcalloc(num_queues, sizeof(struct netfront_queue), GFP_KERNEL);
-	if (!info->queues) {
-		err = -ENOMEM;
-		goto out;
-	}
-	rtnl_lock();
-	netif_set_real_num_tx_queues(info->netdev, num_queues);
-	rtnl_unlock();
+	if (info->queues)
+		xennet_destroy_queues(info);
+
+	err = xennet_create_queues(info, num_queues);
+	if (err < 0)
+		goto destroy_ring;
 
 	/* Create shared ring, alloc event channel -- for each queue */
 	for (i = 0; i < num_queues; ++i) {
 		queue = &info->queues[i];
-		queue->id = i;
-		queue->info = info;
-		err = xennet_init_queue(queue);
-		if (err) {
-			/* xennet_init_queue() cleans up after itself on failure,
-			 * but we still have to clean up any previously initialised
-			 * queues. If i > 0, set num_queues to i, then goto
-			 * destroy_ring, which calls xennet_disconnect_backend()
-			 * to tidy up.
-			 */
-			if (i > 0) {
-				rtnl_lock();
-				netif_set_real_num_tx_queues(info->netdev, i);
-				rtnl_unlock();
-				goto destroy_ring;
-			} else {
-				goto out;
-			}
-		}
 		err = setup_netfront(dev, queue, feature_split_evtchn);
 		if (err) {
-			/* As for xennet_init_queue(), setup_netfront() will tidy
-			 * up the current queue on error, but we need to clean up
+			/* setup_netfront() will tidy up the current
+			 * queue on error, but we need to clean up
 			 * those already allocated.
 			 */
 			if (i > 0) {
-- 
1.7.10.4

Re: [Xen-devel] [PATCH 1/2] xen-netfront: fix oops when disconnected from backend

[Wei Liu]

On Wed, Jun 18, 2014 at 10:47:27AM +0100, David Vrabel wrote:
> xennet_disconnect_backend() was not correctly iterating over all the
> queues.
> 
> Signed-off-by: David Vrabel <david.vrabel@citrix.com>

Reviewed-by: Wei Liu <wei.liu2@citrix.com>

> ---
>  drivers/net/xen-netfront.c |    5 +++--
>  1 file changed, 3 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
> index 5a7872a..daaf1e5 100644
> --- a/drivers/net/xen-netfront.c
> +++ b/drivers/net/xen-netfront.c
> @@ -1287,7 +1287,7 @@ static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
>  
>  	if (likely(netif_carrier_ok(dev) &&
>  		   RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)))
> -			napi_schedule(&queue->napi);
> +		napi_schedule(&queue->napi);
>  

This indentation fix is not actually related to this bug. IMHO it should
either be removed or noted in commit message.

Wei.

Re: [PATCH 1/2] xen-netfront: fix oops when disconnected from backend

[Wei Liu]

On Wed, Jun 18, 2014 at 10:47:27AM +0100, David Vrabel wrote:
> xennet_disconnect_backend() was not correctly iterating over all the
> queues.
> 
> Signed-off-by: David Vrabel <david.vrabel@citrix.com>

Reviewed-by: Wei Liu <wei.liu2@citrix.com>

> ---
>  drivers/net/xen-netfront.c |    5 +++--
>  1 file changed, 3 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
> index 5a7872a..daaf1e5 100644
> --- a/drivers/net/xen-netfront.c
> +++ b/drivers/net/xen-netfront.c
> @@ -1287,7 +1287,7 @@ static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
>  
>  	if (likely(netif_carrier_ok(dev) &&
>  		   RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)))
> -			napi_schedule(&queue->napi);
> +		napi_schedule(&queue->napi);
>  

This indentation fix is not actually related to this bug. IMHO it should
either be removed or noted in commit message.

Wei.

Re: [Xen-devel] [PATCH 2/2] xen-netfront: recreate queues correctly when reconnecting

[Wei Liu]

On Wed, Jun 18, 2014 at 10:47:28AM +0100, David Vrabel wrote:
> When reconnecting to the backend (after a resume/migration, for example),
> a different number of queues may be required (since the guest may have
> moved to a different host with different capabilities).  During the
> reconnection the old queues are torn down and new ones created.
> 
> Introduce xennet_create_queues() and xennet_destroy_queues() that fixes
> three bugs during the reconnection.
> 
> - The old info->queues was leaked.
> - The old queue's napi instances were not deleted.
> - The new queue's napi instances were left disabled (which meant no
>   packets could be received).
> 
> The xennet_destroy_queues() calls is deferred until the reconnection
> instead of the disconnection (in xennet_disconnect_backend()) because
> napi_disable() might sleep.
> 
> Signed-off-by: David Vrabel <david.vrabel@citrix.com>

Reviewed-by: Wei Liu <wei.liu2@citrix.com>

> ---
>  drivers/net/xen-netfront.c |  104 ++++++++++++++++++++++++++++++--------------
>  1 file changed, 72 insertions(+), 32 deletions(-)

Re: [PATCH 2/2] xen-netfront: recreate queues correctly when reconnecting

[Wei Liu]

On Wed, Jun 18, 2014 at 10:47:28AM +0100, David Vrabel wrote:
> When reconnecting to the backend (after a resume/migration, for example),
> a different number of queues may be required (since the guest may have
> moved to a different host with different capabilities).  During the
> reconnection the old queues are torn down and new ones created.
> 
> Introduce xennet_create_queues() and xennet_destroy_queues() that fixes
> three bugs during the reconnection.
> 
> - The old info->queues was leaked.
> - The old queue's napi instances were not deleted.
> - The new queue's napi instances were left disabled (which meant no
>   packets could be received).
> 
> The xennet_destroy_queues() calls is deferred until the reconnection
> instead of the disconnection (in xennet_disconnect_backend()) because
> napi_disable() might sleep.
> 
> Signed-off-by: David Vrabel <david.vrabel@citrix.com>

Reviewed-by: Wei Liu <wei.liu2@citrix.com>

> ---
>  drivers/net/xen-netfront.c |  104 ++++++++++++++++++++++++++++++--------------
>  1 file changed, 72 insertions(+), 32 deletions(-)

Re: [PATCHv1 0/2 net] xen-netfront: fix resume regressions in 3.16-rc1

[David Miller]

From: David Vrabel <david.vrabel@citrix.com>
Date: Wed, 18 Jun 2014 10:47:26 +0100

> The introduction of multi-queue support to xen-netfront in 3.16-rc1,
> broke resume/migration.

Series applied, thanks.

Re: [PATCHv1 0/2 net] xen-netfront: fix resume regressions in 3.16-rc1

[David Miller]

From: David Vrabel <david.vrabel@citrix.com>
Date: Wed, 18 Jun 2014 10:47:26 +0100

> The introduction of multi-queue support to xen-netfront in 3.16-rc1,
> broke resume/migration.

Series applied, thanks.