From mboxrd@z Thu Jan 1 00:00:00 1970 From: Wei Liu Subject: [RFC PATCH V4 12/13] netfront: multi page ring support. Date: Thu, 2 Feb 2012 16:49:22 +0000 Message-ID: <1328201363-13915-13-git-send-email-wei.liu2@citrix.com> References: <1328201363-13915-1-git-send-email-wei.liu2@citrix.com> Mime-Version: 1.0 Content-Type: text/plain Cc: ian.campbell@citrix.com, konrad.wilk@oracle.com, Wei Liu To: netdev@vger.kernel.org, xen-devel@lists.xensource.com Return-path: Received: from smtp02.citrix.com ([66.165.176.63]:45019 "EHLO SMTP02.CITRIX.COM" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754609Ab2BBQtz (ORCPT ); Thu, 2 Feb 2012 11:49:55 -0500 In-Reply-To: <1328201363-13915-1-git-send-email-wei.liu2@citrix.com> Sender: netdev-owner@vger.kernel.org List-ID: Signed-off-by: Wei Liu --- drivers/net/xen-netfront.c | 228 ++++++++++++++++++++++++++++++-------------- 1 files changed, 156 insertions(+), 72 deletions(-) diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index b7ff815..a1cfb24 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -66,9 +66,18 @@ struct netfront_cb { #define GRANT_INVALID_REF 0 -#define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, PAGE_SIZE) -#define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, PAGE_SIZE) -#define TX_MAX_TARGET min_t(int, NET_TX_RING_SIZE, 256) +#define XENNET_MAX_RING_PAGE_ORDER 2 +#define XENNET_MAX_RING_PAGES (1U << XENNET_MAX_RING_PAGE_ORDER) + +#define NET_TX_RING_SIZE(_nr_pages) \ + __CONST_RING_SIZE(xen_netif_tx, PAGE_SIZE * (_nr_pages)) +#define NET_RX_RING_SIZE(_nr_pages) \ + __CONST_RING_SIZE(xen_netif_rx, PAGE_SIZE * (_nr_pages)) + +#define XENNET_MAX_TX_RING_SIZE NET_TX_RING_SIZE(XENNET_MAX_RING_PAGES) +#define XENNET_MAX_RX_RING_SIZE NET_RX_RING_SIZE(XENNET_MAX_RING_PAGES) + +#define TX_MAX_TARGET XENNET_MAX_TX_RING_SIZE struct netfront_stats { u64 rx_packets; @@ -84,12 +93,19 @@ struct netfront_info { struct napi_struct napi; + /* Statistics */ + struct netfront_stats __percpu *stats; + + unsigned long rx_gso_checksum_fixup; + unsigned int evtchn; struct xenbus_device *xbdev; spinlock_t tx_lock; struct xen_netif_tx_front_ring tx; - int tx_ring_ref; + int tx_ring_ref[XENNET_MAX_RING_PAGES]; + int tx_ring_page_order; + int tx_ring_pages; /* * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries @@ -103,36 +119,33 @@ struct netfront_info { union skb_entry { struct sk_buff *skb; unsigned long link; - } tx_skbs[NET_TX_RING_SIZE]; + } tx_skbs[XENNET_MAX_TX_RING_SIZE]; grant_ref_t gref_tx_head; - grant_ref_t grant_tx_ref[NET_TX_RING_SIZE]; + grant_ref_t grant_tx_ref[XENNET_MAX_TX_RING_SIZE]; unsigned tx_skb_freelist; spinlock_t rx_lock ____cacheline_aligned_in_smp; struct xen_netif_rx_front_ring rx; - int rx_ring_ref; + int rx_ring_ref[XENNET_MAX_RING_PAGES]; + int rx_ring_page_order; + int rx_ring_pages; /* Receive-ring batched refills. */ #define RX_MIN_TARGET 8 #define RX_DFL_MIN_TARGET 64 -#define RX_MAX_TARGET min_t(int, NET_RX_RING_SIZE, 256) +#define RX_MAX_TARGET XENNET_MAX_RX_RING_SIZE unsigned rx_min_target, rx_max_target, rx_target; struct sk_buff_head rx_batch; struct timer_list rx_refill_timer; - struct sk_buff *rx_skbs[NET_RX_RING_SIZE]; + struct sk_buff *rx_skbs[XENNET_MAX_RX_RING_SIZE]; grant_ref_t gref_rx_head; - grant_ref_t grant_rx_ref[NET_RX_RING_SIZE]; - - unsigned long rx_pfn_array[NET_RX_RING_SIZE]; - struct multicall_entry rx_mcl[NET_RX_RING_SIZE+1]; - struct mmu_update rx_mmu[NET_RX_RING_SIZE]; - - /* Statistics */ - struct netfront_stats __percpu *stats; + grant_ref_t grant_rx_ref[XENNET_MAX_RX_RING_SIZE]; - unsigned long rx_gso_checksum_fixup; + unsigned long rx_pfn_array[XENNET_MAX_RX_RING_SIZE]; + struct multicall_entry rx_mcl[XENNET_MAX_RX_RING_SIZE+1]; + struct mmu_update rx_mmu[XENNET_MAX_RX_RING_SIZE]; }; struct netfront_rx_info { @@ -170,15 +183,15 @@ static unsigned short get_id_from_freelist(unsigned *head, return id; } -static int xennet_rxidx(RING_IDX idx) +static int xennet_rxidx(RING_IDX idx, struct netfront_info *info) { - return idx & (NET_RX_RING_SIZE - 1); + return idx & (NET_RX_RING_SIZE(info->rx_ring_pages) - 1); } static struct sk_buff *xennet_get_rx_skb(struct netfront_info *np, RING_IDX ri) { - int i = xennet_rxidx(ri); + int i = xennet_rxidx(ri, np); struct sk_buff *skb = np->rx_skbs[i]; np->rx_skbs[i] = NULL; return skb; @@ -187,7 +200,7 @@ static struct sk_buff *xennet_get_rx_skb(struct netfront_info *np, static grant_ref_t xennet_get_rx_ref(struct netfront_info *np, RING_IDX ri) { - int i = xennet_rxidx(ri); + int i = xennet_rxidx(ri, np); grant_ref_t ref = np->grant_rx_ref[i]; np->grant_rx_ref[i] = GRANT_INVALID_REF; return ref; @@ -300,7 +313,7 @@ no_skb: skb->dev = dev; - id = xennet_rxidx(req_prod + i); + id = xennet_rxidx(req_prod + i, np); BUG_ON(np->rx_skbs[id]); np->rx_skbs[id] = skb; @@ -596,7 +609,7 @@ static int xennet_close(struct net_device *dev) static void xennet_move_rx_slot(struct netfront_info *np, struct sk_buff *skb, grant_ref_t ref) { - int new = xennet_rxidx(np->rx.req_prod_pvt); + int new = xennet_rxidx(np->rx.req_prod_pvt, np); BUG_ON(np->rx_skbs[new]); np->rx_skbs[new] = skb; @@ -1089,7 +1102,7 @@ static void xennet_release_tx_bufs(struct netfront_info *np) struct sk_buff *skb; int i; - for (i = 0; i < NET_TX_RING_SIZE; i++) { + for (i = 0; i < NET_TX_RING_SIZE(np->tx_ring_pages); i++) { /* Skip over entries which are actually freelist references */ if (skb_entry_is_link(&np->tx_skbs[i])) continue; @@ -1123,7 +1136,7 @@ static void xennet_release_rx_bufs(struct netfront_info *np) spin_lock_bh(&np->rx_lock); - for (id = 0; id < NET_RX_RING_SIZE; id++) { + for (id = 0; id < NET_RX_RING_SIZE(np->rx_ring_pages); id++) { ref = np->grant_rx_ref[id]; if (ref == GRANT_INVALID_REF) { unused++; @@ -1305,13 +1318,13 @@ static struct net_device * __devinit xennet_create_dev(struct xenbus_device *dev /* Initialise tx_skbs as a free chain containing every entry. */ np->tx_skb_freelist = 0; - for (i = 0; i < NET_TX_RING_SIZE; i++) { + for (i = 0; i < XENNET_MAX_TX_RING_SIZE; i++) { skb_entry_set_link(&np->tx_skbs[i], i+1); np->grant_tx_ref[i] = GRANT_INVALID_REF; } /* Clear out rx_skbs */ - for (i = 0; i < NET_RX_RING_SIZE; i++) { + for (i = 0; i < XENNET_MAX_RX_RING_SIZE; i++) { np->rx_skbs[i] = NULL; np->grant_rx_ref[i] = GRANT_INVALID_REF; } @@ -1409,13 +1422,6 @@ static int __devinit netfront_probe(struct xenbus_device *dev, return err; } -static void xennet_end_access(int ref, void *page) -{ - /* This frees the page as a side-effect */ - if (ref != GRANT_INVALID_REF) - gnttab_end_foreign_access(ref, 0, (unsigned long)page); -} - static void xennet_disconnect_backend(struct netfront_info *info) { /* Stop old i/f to prevent errors whilst we rebuild the state. */ @@ -1429,12 +1435,12 @@ static void xennet_disconnect_backend(struct netfront_info *info) unbind_from_irqhandler(info->netdev->irq, info->netdev); info->evtchn = info->netdev->irq = 0; - /* End access and free the pages */ - xennet_end_access(info->tx_ring_ref, info->tx.sring); - xennet_end_access(info->rx_ring_ref, info->rx.sring); + xenbus_unmap_ring_vfree(info->xbdev, (void *)info->tx.sring); + free_pages((unsigned long)info->tx.sring, info->tx_ring_page_order); + + xenbus_unmap_ring_vfree(info->xbdev, (void *)info->rx.sring); + free_pages((unsigned long)info->rx.sring, info->rx_ring_page_order); - info->tx_ring_ref = GRANT_INVALID_REF; - info->rx_ring_ref = GRANT_INVALID_REF; info->tx.sring = NULL; info->rx.sring = NULL; } @@ -1482,11 +1488,14 @@ static int setup_netfront(struct xenbus_device *dev, struct netfront_info *info) struct xen_netif_tx_sring *txs; struct xen_netif_rx_sring *rxs; int err; - int grefs[1]; struct net_device *netdev = info->netdev; + unsigned int max_tx_ring_page_order, max_rx_ring_page_order; + int i; - info->tx_ring_ref = GRANT_INVALID_REF; - info->rx_ring_ref = GRANT_INVALID_REF; + for (i = 0; i < XENNET_MAX_RING_PAGES; i++) { + info->tx_ring_ref[i] = GRANT_INVALID_REF; + info->rx_ring_ref[i] = GRANT_INVALID_REF; + } info->rx.sring = NULL; info->tx.sring = NULL; netdev->irq = 0; @@ -1497,50 +1506,91 @@ static int setup_netfront(struct xenbus_device *dev, struct netfront_info *info) goto fail; } - txs = (struct xen_netif_tx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "max-tx-ring-page-order", "%u", + &max_tx_ring_page_order); + if (err < 0) { + info->tx_ring_page_order = 0; + dev_info(&dev->dev, "single tx ring\n"); + } else { + info->tx_ring_page_order = max_tx_ring_page_order; + dev_info(&dev->dev, "multi page tx ring, order = %d\n", + max_tx_ring_page_order); + } + info->tx_ring_pages = (1U << info->tx_ring_page_order); + + txs = (struct xen_netif_tx_sring *) + __get_free_pages(__GFP_ZERO | GFP_NOIO | __GFP_HIGH, + info->tx_ring_page_order); if (!txs) { err = -ENOMEM; xenbus_dev_fatal(dev, err, "allocating tx ring page"); goto fail; } SHARED_RING_INIT(txs); - FRONT_RING_INIT(&info->tx, txs, PAGE_SIZE); + FRONT_RING_INIT(&info->tx, txs, PAGE_SIZE * info->tx_ring_pages); + + err = xenbus_grant_ring(dev, txs, info->tx_ring_pages, + info->tx_ring_ref); + + if (err < 0) + goto grant_tx_ring_fail; - err = xenbus_grant_ring(dev, txs, 1, grefs); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "max-rx-ring-page-order", "%u", + &max_rx_ring_page_order); if (err < 0) { - free_page((unsigned long)txs); - goto fail; + info->rx_ring_page_order = 0; + dev_info(&dev->dev, "single rx ring\n"); + } else { + info->rx_ring_page_order = max_rx_ring_page_order; + dev_info(&dev->dev, "multi page rx ring, order = %d\n", + max_rx_ring_page_order); } + info->rx_ring_pages = (1U << info->rx_ring_page_order); - info->tx_ring_ref = grefs[0]; - rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); + rxs = (struct xen_netif_rx_sring *) + __get_free_pages(__GFP_ZERO | GFP_NOIO | __GFP_HIGH, + info->rx_ring_page_order); if (!rxs) { err = -ENOMEM; xenbus_dev_fatal(dev, err, "allocating rx ring page"); - goto fail; + goto alloc_rx_ring_fail; } SHARED_RING_INIT(rxs); - FRONT_RING_INIT(&info->rx, rxs, PAGE_SIZE); + FRONT_RING_INIT(&info->rx, rxs, PAGE_SIZE * info->rx_ring_pages); - err = xenbus_grant_ring(dev, rxs, 1, grefs); - if (err < 0) { - free_page((unsigned long)rxs); - goto fail; - } - info->rx_ring_ref = grefs[0]; + err = xenbus_grant_ring(dev, rxs, info->rx_ring_pages, + info->rx_ring_ref); + + if (err < 0) + goto grant_rx_ring_fail; + + FRONT_RING_INIT(&info->rx, rxs, PAGE_SIZE * info->rx_ring_pages); err = xenbus_alloc_evtchn(dev, &info->evtchn); if (err) - goto fail; + goto alloc_evtchn_fail; err = bind_evtchn_to_irqhandler(info->evtchn, xennet_interrupt, 0, netdev->name, netdev); if (err < 0) - goto fail; + goto bind_fail; netdev->irq = err; + return 0; - fail: +bind_fail: + xenbus_free_evtchn(dev, info->evtchn); +alloc_evtchn_fail: + xenbus_unmap_ring_vfree(info->xbdev, (void *)info->rx.sring); +grant_rx_ring_fail: + free_pages((unsigned long)info->rx.sring, info->rx_ring_page_order); +alloc_rx_ring_fail: + xenbus_unmap_ring_vfree(info->xbdev, (void *)info->tx.sring); +grant_tx_ring_fail: + free_pages((unsigned long)info->tx.sring, info->tx_ring_page_order); +fail: return err; } @@ -1551,6 +1601,7 @@ static int talk_to_netback(struct xenbus_device *dev, const char *message; struct xenbus_transaction xbt; int err; + int i; /* Create shared ring, alloc event channel. */ err = setup_netfront(dev, info); @@ -1564,18 +1615,50 @@ again: goto destroy_ring; } - err = xenbus_printf(xbt, dev->nodename, "tx-ring-ref", "%u", - info->tx_ring_ref); - if (err) { - message = "writing tx ring-ref"; - goto abort_transaction; + if (info->tx_ring_page_order == 0) + err = xenbus_printf(xbt, dev->nodename, "tx-ring-ref", "%u", + info->tx_ring_ref[0]); + else { + err = xenbus_printf(xbt, dev->nodename, "tx-ring-order", "%u", + info->tx_ring_page_order); + if (err) { + message = "writing tx ring-ref"; + goto abort_transaction; + } + for (i = 0; i < info->tx_ring_pages; i++) { + char name[sizeof("tx-ring-ref")+2]; + snprintf(name, sizeof(name), "tx-ring-ref%u", i); + err = xenbus_printf(xbt, dev->nodename, name, "%u", + info->tx_ring_ref[i]); + if (err) { + message = "writing tx ring-ref"; + goto abort_transaction; + } + } } - err = xenbus_printf(xbt, dev->nodename, "rx-ring-ref", "%u", - info->rx_ring_ref); - if (err) { - message = "writing rx ring-ref"; - goto abort_transaction; + + if (info->rx_ring_page_order == 0) + err = xenbus_printf(xbt, dev->nodename, "rx-ring-ref", "%u", + info->rx_ring_ref[0]); + else { + err = xenbus_printf(xbt, dev->nodename, "rx-ring-order", "%u", + info->rx_ring_page_order); + if (err) { + message = "writing tx ring-ref"; + goto abort_transaction; + } + for (i = 0; i < info->rx_ring_pages; i++) { + char name[sizeof("rx-ring-ref")+2]; + snprintf(name, sizeof(name), "rx-ring-ref%u", i); + err = xenbus_printf(xbt, dev->nodename, name, "%u", + info->rx_ring_ref[i]); + if (err) { + message = "writing rx ring-ref"; + goto abort_transaction; + } + } } + err = xenbus_printf(xbt, dev->nodename, "event-channel", "%u", info->evtchn); if (err) { @@ -1662,7 +1745,8 @@ static int xennet_connect(struct net_device *dev) xennet_release_tx_bufs(np); /* Step 2: Rebuild the RX buffer freelist and the RX ring itself. */ - for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE; i++) { + for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE(np->rx_ring_pages); + i++) { skb_frag_t *frag; const struct page *page; if (!np->rx_skbs[i]) -- 1.7.2.5