* [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors
@ 2012-08-30 11:21 Sasha Levin
2012-08-30 11:21 ` [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible Sasha Levin
2012-08-30 14:14 ` [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors Michael S. Tsirkin
0 siblings, 2 replies; 11+ messages in thread
From: Sasha Levin @ 2012-08-30 11:21 UTC (permalink / raw)
To: mst, rusty; +Cc: virtualization, linux-kernel, avi, kvm, Sasha Levin
Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will use indirect
descriptors even if we have plenty of space in the ring. This means that
we take a performance hit at all times due to the overhead of creating
indirect descriptors.
Instead, use it only after we're below a configurable offset.
Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
---
drivers/block/virtio_blk.c | 4 ++++
drivers/char/hw_random/virtio-rng.c | 4 ++++
drivers/char/virtio_console.c | 4 ++++
drivers/net/virtio_net.c | 4 ++++
drivers/virtio/virtio_balloon.c | 4 ++++
drivers/virtio/virtio_ring.c | 21 +++++++++++++++------
include/linux/virtio.h | 1 +
net/9p/trans_virtio.c | 4 ++++
8 files changed, 40 insertions(+), 6 deletions(-)
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 2edfb5c..7c63065 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -22,6 +22,9 @@ static DEFINE_IDA(vd_index_ida);
struct workqueue_struct *virtblk_wq;
+static unsigned int indirect_thresh;
+module_param(indirect_thresh, uint, S_IRUGO);
+
struct virtio_blk
{
struct virtio_device *vdev;
@@ -735,6 +738,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev)
INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
vblk->config_enable = true;
+ vdev->indirect_thresh = indirect_thresh;
err = init_vq(vblk);
if (err)
diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c
index 5708299..3a644f1 100644
--- a/drivers/char/hw_random/virtio-rng.c
+++ b/drivers/char/hw_random/virtio-rng.c
@@ -25,6 +25,9 @@
#include <linux/virtio_rng.h>
#include <linux/module.h>
+static unsigned int indirect_thresh;
+module_param(indirect_thresh, uint, S_IRUGO);
+
static struct virtqueue *vq;
static unsigned int data_avail;
static DECLARE_COMPLETION(have_data);
@@ -93,6 +96,7 @@ static int probe_common(struct virtio_device *vdev)
int err;
/* We expect a single virtqueue. */
+ vdev->indirect_thresh = indirect_thresh;
vq = virtio_find_single_vq(vdev, random_recv_done, "input");
if (IS_ERR(vq))
return PTR_ERR(vq);
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index e88f843..da2e44c 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -39,6 +39,9 @@
#include <linux/module.h>
#include "../tty/hvc/hvc_console.h"
+static unsigned int indirect_thresh;
+module_param(indirect_thresh, uint, S_IRUGO);
+
/*
* This is a global struct for storing common data for all the devices
* this driver handles.
@@ -1887,6 +1890,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev)
max_nr_ports),
&portdev->config.max_nr_ports) == 0)
multiport = true;
+ vdev->indirect_thresh = indirect_thresh;
err = init_vqs(portdev);
if (err < 0) {
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index cbf8b06..949c89e 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -34,6 +34,9 @@ static bool csum = true, gso = true;
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);
+static unsigned int indirect_thresh = 16;
+module_param(indirect_thresh, uint, S_IRUGO);
+
/* FIXME: MTU in config. */
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
#define GOOD_COPY_LEN 128
@@ -1128,6 +1131,7 @@ static int virtnet_probe(struct virtio_device *vdev)
if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
vi->mergeable_rx_bufs = true;
+ vdev->indirect_thresh = indirect_thresh;
err = init_vqs(vi);
if (err)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 0908e60..ca5ae7a 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -35,6 +35,9 @@
*/
#define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
+static unsigned int indirect_thresh;
+module_param(indirect_thresh, uint, S_IRUGO);
+
struct virtio_balloon
{
struct virtio_device *vdev;
@@ -356,6 +359,7 @@ static int virtballoon_probe(struct virtio_device *vdev)
init_waitqueue_head(&vb->acked);
vb->vdev = vdev;
vb->need_stats_update = 0;
+ vdev->indirect_thresh = indirect_thresh;
err = init_vqs(vb);
if (err)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 5aa43c3..4063e03 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -87,8 +87,11 @@ struct vring_virtqueue
/* Other side has made a mess, don't try any more. */
bool broken;
- /* Host supports indirect buffers */
- bool indirect;
+ /*
+ * Min. number of free space in the ring to trigger direct
+ * descriptor use
+ */
+ unsigned int indirect_thresh;
/* Host publishes avail event idx */
bool event;
@@ -216,9 +219,12 @@ int virtqueue_add_buf(struct virtqueue *_vq,
}
#endif
- /* If the host supports indirect descriptor tables, and we have multiple
- * buffers, then go indirect. FIXME: tune this threshold */
- if (vq->indirect && (out + in) > 1 && vq->num_free) {
+ /*
+ * If the host supports indirect descriptor tables, and we have multiple
+ * buffers, then go indirect.
+ */
+ if ((out + in) > 1 && vq->num_free &&
+ (vq->num_free < vq->indirect_thresh)) {
head = vring_add_indirect(vq, sg, out, in, gfp);
if (likely(head >= 0))
goto add_head;
@@ -647,13 +653,16 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
vq->broken = false;
vq->last_used_idx = 0;
vq->num_added = 0;
+ vq->indirect_thresh = 0;
list_add_tail(&vq->vq.list, &vdev->vqs);
#ifdef DEBUG
vq->in_use = false;
vq->last_add_time_valid = false;
#endif
- vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
+ if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
+ vq->indirect_thresh = vdev->indirect_thresh;
+
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
/* No callback? Tell other side not to bother us. */
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index a1ba8bb..48bc457 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -69,6 +69,7 @@ struct virtio_device {
/* Note that this is a Linux set_bit-style bitmap. */
unsigned long features[1];
void *priv;
+ unsigned int indirect_thresh;
};
#define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev)
diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c
index 35b8911..418f933 100644
--- a/net/9p/trans_virtio.c
+++ b/net/9p/trans_virtio.c
@@ -52,6 +52,9 @@
#define VIRTQUEUE_NUM 128
+static unsigned int indirect_thresh;
+module_param(indirect_thresh, uint, S_IRUGO);
+
/* a single mutex to manage channel initialization and attachment */
static DEFINE_MUTEX(virtio_9p_lock);
static DECLARE_WAIT_QUEUE_HEAD(vp_wq);
@@ -501,6 +504,7 @@ static int p9_virtio_probe(struct virtio_device *vdev)
chan->vdev = vdev;
/* We expect one virtqueue, for requests. */
+ vdev->indirect_thresh = indirect_thresh;
chan->vq = virtio_find_single_vq(vdev, req_done, "requests");
if (IS_ERR(chan->vq)) {
err = PTR_ERR(chan->vq);
--
1.7.12
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-08-30 11:21 [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors Sasha Levin
@ 2012-08-30 11:21 ` Sasha Levin
2012-08-30 13:38 ` Michael S. Tsirkin
2012-08-30 14:14 ` [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors Michael S. Tsirkin
1 sibling, 1 reply; 11+ messages in thread
From: Sasha Levin @ 2012-08-30 11:21 UTC (permalink / raw)
To: mst, rusty; +Cc: virtualization, linux-kernel, avi, kvm, Sasha Levin
Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will
use indirect descriptors and allocate them using a simple
kmalloc().
This patch adds a cache which will allow indirect buffers under
a configurable size to be allocated from that cache instead.
Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
---
drivers/block/virtio_blk.c | 4 ++++
drivers/char/hw_random/virtio-rng.c | 4 ++++
drivers/char/virtio_console.c | 4 ++++
drivers/net/virtio_net.c | 4 ++++
drivers/virtio/virtio_balloon.c | 4 ++++
drivers/virtio/virtio_ring.c | 34 ++++++++++++++++++++++++++++++----
include/linux/virtio.h | 1 +
net/9p/trans_virtio.c | 5 +++++
8 files changed, 56 insertions(+), 4 deletions(-)
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 7c63065..e4c6c42 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -25,6 +25,9 @@ struct workqueue_struct *virtblk_wq;
static unsigned int indirect_thresh;
module_param(indirect_thresh, uint, S_IRUGO);
+static unsigned int indirect_alloc_thresh;
+module_param(indirect_alloc_thresh, uint, S_IRUGO);
+
struct virtio_blk
{
struct virtio_device *vdev;
@@ -739,6 +742,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev)
INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
vblk->config_enable = true;
vdev->indirect_thresh = indirect_thresh;
+ vdev->indirect_alloc_thresh = indirect_alloc_thresh;
err = init_vq(vblk);
if (err)
diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c
index 3a644f1..ed22db8 100644
--- a/drivers/char/hw_random/virtio-rng.c
+++ b/drivers/char/hw_random/virtio-rng.c
@@ -28,6 +28,9 @@
static unsigned int indirect_thresh;
module_param(indirect_thresh, uint, S_IRUGO);
+static unsigned int indirect_alloc_thresh;
+module_param(indirect_alloc_thresh, uint, S_IRUGO);
+
static struct virtqueue *vq;
static unsigned int data_avail;
static DECLARE_COMPLETION(have_data);
@@ -97,6 +100,7 @@ static int probe_common(struct virtio_device *vdev)
/* We expect a single virtqueue. */
vdev->indirect_thresh = indirect_thresh;
+ vdev->indirect_alloc_thresh = indirect_alloc_thresh;
vq = virtio_find_single_vq(vdev, random_recv_done, "input");
if (IS_ERR(vq))
return PTR_ERR(vq);
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index da2e44c..8f30732 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -42,6 +42,9 @@
static unsigned int indirect_thresh;
module_param(indirect_thresh, uint, S_IRUGO);
+static unsigned int indirect_alloc_thresh;
+module_param(indirect_alloc_thresh, uint, S_IRUGO);
+
/*
* This is a global struct for storing common data for all the devices
* this driver handles.
@@ -1891,6 +1894,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev)
&portdev->config.max_nr_ports) == 0)
multiport = true;
vdev->indirect_thresh = indirect_thresh;
+ vdev->indirect_alloc_thresh = indirect_alloc_thresh;
err = init_vqs(portdev);
if (err < 0) {
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 949c89e..a00e19d 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -37,6 +37,9 @@ module_param(gso, bool, 0444);
static unsigned int indirect_thresh = 16;
module_param(indirect_thresh, uint, S_IRUGO);
+static unsigned int indirect_alloc_thresh = 16;
+module_param(indirect_alloc_thresh, uint, S_IRUGO);
+
/* FIXME: MTU in config. */
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
#define GOOD_COPY_LEN 128
@@ -1132,6 +1135,7 @@ static int virtnet_probe(struct virtio_device *vdev)
if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
vi->mergeable_rx_bufs = true;
vdev->indirect_thresh = indirect_thresh;
+ vdev->indirect_alloc_thresh = indirect_alloc_thresh;
err = init_vqs(vi);
if (err)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index ca5ae7a..039c4a6 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -38,6 +38,9 @@
static unsigned int indirect_thresh;
module_param(indirect_thresh, uint, S_IRUGO);
+static unsigned int indirect_alloc_thresh;
+module_param(indirect_alloc_thresh, uint, S_IRUGO);
+
struct virtio_balloon
{
struct virtio_device *vdev;
@@ -360,6 +363,7 @@ static int virtballoon_probe(struct virtio_device *vdev)
vb->vdev = vdev;
vb->need_stats_update = 0;
vdev->indirect_thresh = indirect_thresh;
+ vdev->indirect_alloc_thresh = indirect_alloc_thresh;
err = init_vqs(vb);
if (err)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 4063e03..dde867b 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -93,6 +93,10 @@ struct vring_virtqueue
*/
unsigned int indirect_thresh;
+ /* Buffers below this size will be allocated from cache */
+ unsigned int indirect_alloc_thresh;
+ struct kmem_cache *indirect_cache;
+
/* Host publishes avail event idx */
bool event;
@@ -135,7 +139,10 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
unsigned head;
int i;
- desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
+ if ((out + in) <= vq->indirect_alloc_thresh)
+ desc = kmem_cache_alloc(vq->indirect_cache, gfp);
+ else
+ desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
if (!desc)
return -ENOMEM;
@@ -384,8 +391,14 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
i = head;
/* Free the indirect table */
- if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
- kfree(phys_to_virt(vq->vring.desc[i].addr));
+ if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) {
+ u32 descs = vq->vring.desc[i].len / sizeof(struct vring_desc);
+ if (descs > vq->indirect_alloc_thresh)
+ kfree(phys_to_virt(vq->vring.desc[i].addr));
+ else
+ kmem_cache_free(vq->indirect_cache,
+ phys_to_virt(vq->vring.desc[i].addr));
+ }
while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
i = vq->vring.desc[i].next;
@@ -654,14 +667,25 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
vq->last_used_idx = 0;
vq->num_added = 0;
vq->indirect_thresh = 0;
+ vq->indirect_alloc_thresh = 0;
+ vq->indirect_cache = NULL;
list_add_tail(&vq->vq.list, &vdev->vqs);
#ifdef DEBUG
vq->in_use = false;
vq->last_add_time_valid = false;
#endif
- if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
+ if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) {
vq->indirect_thresh = vdev->indirect_thresh;
+ vq->indirect_alloc_thresh = vdev->indirect_alloc_thresh;
+ if (vq->indirect_alloc_thresh) {
+ vq->indirect_cache =
+ KMEM_CACHE(vring_desc[vq->indirect_alloc_thresh], 0);
+
+ if (vq->indirect_cache == NULL)
+ vq->indirect_alloc_thresh = 0;
+ }
+ }
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
@@ -685,6 +709,8 @@ EXPORT_SYMBOL_GPL(vring_new_virtqueue);
void vring_del_virtqueue(struct virtqueue *vq)
{
list_del(&vq->list);
+ if (to_vvq(vq)->indirect_cache)
+ kmem_cache_destroy(to_vvq(vq)->indirect_cache);
kfree(to_vvq(vq));
}
EXPORT_SYMBOL_GPL(vring_del_virtqueue);
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index 48bc457..3261c02 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -70,6 +70,7 @@ struct virtio_device {
unsigned long features[1];
void *priv;
unsigned int indirect_thresh;
+ unsigned int indirect_alloc_thresh;
};
#define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev)
diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c
index 418f933..058b6dd 100644
--- a/net/9p/trans_virtio.c
+++ b/net/9p/trans_virtio.c
@@ -55,6 +55,9 @@
static unsigned int indirect_thresh;
module_param(indirect_thresh, uint, S_IRUGO);
+static unsigned int indirect_alloc_thresh;
+module_param(indirect_alloc_thresh, uint, S_IRUGO);
+
/* a single mutex to manage channel initialization and attachment */
static DEFINE_MUTEX(virtio_9p_lock);
static DECLARE_WAIT_QUEUE_HEAD(vp_wq);
@@ -505,6 +508,8 @@ static int p9_virtio_probe(struct virtio_device *vdev)
/* We expect one virtqueue, for requests. */
vdev->indirect_thresh = indirect_thresh;
+ vdev->indirect_alloc_thresh = indirect_alloc_thresh;
+
chan->vq = virtio_find_single_vq(vdev, req_done, "requests");
if (IS_ERR(chan->vq)) {
err = PTR_ERR(chan->vq);
--
1.7.12
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-08-30 11:21 ` [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible Sasha Levin
@ 2012-08-30 13:38 ` Michael S. Tsirkin
2012-08-31 9:36 ` Sasha Levin
0 siblings, 1 reply; 11+ messages in thread
From: Michael S. Tsirkin @ 2012-08-30 13:38 UTC (permalink / raw)
To: Sasha Levin; +Cc: rusty, virtualization, linux-kernel, avi, kvm
On Thu, Aug 30, 2012 at 01:21:58PM +0200, Sasha Levin wrote:
> Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will
> use indirect descriptors and allocate them using a simple
> kmalloc().
>
> This patch adds a cache which will allow indirect buffers under
> a configurable size to be allocated from that cache instead.
>
> Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
I am not sure we need these module params.
But assuming we keep them, we need to validate values - they come from
user.
> ---
> drivers/block/virtio_blk.c | 4 ++++
> drivers/char/hw_random/virtio-rng.c | 4 ++++
> drivers/char/virtio_console.c | 4 ++++
> drivers/net/virtio_net.c | 4 ++++
> drivers/virtio/virtio_balloon.c | 4 ++++
> drivers/virtio/virtio_ring.c | 34 ++++++++++++++++++++++++++++++----
> include/linux/virtio.h | 1 +
> net/9p/trans_virtio.c | 5 +++++
> 8 files changed, 56 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
> index 7c63065..e4c6c42 100644
> --- a/drivers/block/virtio_blk.c
> +++ b/drivers/block/virtio_blk.c
> @@ -25,6 +25,9 @@ struct workqueue_struct *virtblk_wq;
> static unsigned int indirect_thresh;
> module_param(indirect_thresh, uint, S_IRUGO);
>
> +static unsigned int indirect_alloc_thresh;
> +module_param(indirect_alloc_thresh, uint, S_IRUGO);
> +
> struct virtio_blk
> {
> struct virtio_device *vdev;
> @@ -739,6 +742,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev)
> INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
> vblk->config_enable = true;
> vdev->indirect_thresh = indirect_thresh;
> + vdev->indirect_alloc_thresh = indirect_alloc_thresh;
>
> err = init_vq(vblk);
> if (err)
> diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c
> index 3a644f1..ed22db8 100644
> --- a/drivers/char/hw_random/virtio-rng.c
> +++ b/drivers/char/hw_random/virtio-rng.c
> @@ -28,6 +28,9 @@
> static unsigned int indirect_thresh;
> module_param(indirect_thresh, uint, S_IRUGO);
>
> +static unsigned int indirect_alloc_thresh;
> +module_param(indirect_alloc_thresh, uint, S_IRUGO);
> +
> static struct virtqueue *vq;
> static unsigned int data_avail;
> static DECLARE_COMPLETION(have_data);
> @@ -97,6 +100,7 @@ static int probe_common(struct virtio_device *vdev)
>
> /* We expect a single virtqueue. */
> vdev->indirect_thresh = indirect_thresh;
> + vdev->indirect_alloc_thresh = indirect_alloc_thresh;
> vq = virtio_find_single_vq(vdev, random_recv_done, "input");
> if (IS_ERR(vq))
> return PTR_ERR(vq);
> diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
> index da2e44c..8f30732 100644
> --- a/drivers/char/virtio_console.c
> +++ b/drivers/char/virtio_console.c
> @@ -42,6 +42,9 @@
> static unsigned int indirect_thresh;
> module_param(indirect_thresh, uint, S_IRUGO);
>
> +static unsigned int indirect_alloc_thresh;
> +module_param(indirect_alloc_thresh, uint, S_IRUGO);
> +
> /*
> * This is a global struct for storing common data for all the devices
> * this driver handles.
> @@ -1891,6 +1894,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev)
> &portdev->config.max_nr_ports) == 0)
> multiport = true;
> vdev->indirect_thresh = indirect_thresh;
> + vdev->indirect_alloc_thresh = indirect_alloc_thresh;
>
> err = init_vqs(portdev);
> if (err < 0) {
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index 949c89e..a00e19d 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -37,6 +37,9 @@ module_param(gso, bool, 0444);
> static unsigned int indirect_thresh = 16;
> module_param(indirect_thresh, uint, S_IRUGO);
>
> +static unsigned int indirect_alloc_thresh = 16;
Why 16? Please make is MAX_SG + 1 this makes some sense.
> +module_param(indirect_alloc_thresh, uint, S_IRUGO);
> +
> /* FIXME: MTU in config. */
> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
> #define GOOD_COPY_LEN 128
> @@ -1132,6 +1135,7 @@ static int virtnet_probe(struct virtio_device *vdev)
> if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
> vi->mergeable_rx_bufs = true;
> vdev->indirect_thresh = indirect_thresh;
> + vdev->indirect_alloc_thresh = indirect_alloc_thresh;
>
> err = init_vqs(vi);
> if (err)
> diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
> index ca5ae7a..039c4a6 100644
> --- a/drivers/virtio/virtio_balloon.c
> +++ b/drivers/virtio/virtio_balloon.c
> @@ -38,6 +38,9 @@
> static unsigned int indirect_thresh;
> module_param(indirect_thresh, uint, S_IRUGO);
>
> +static unsigned int indirect_alloc_thresh;
> +module_param(indirect_alloc_thresh, uint, S_IRUGO);
> +
> struct virtio_balloon
> {
> struct virtio_device *vdev;
> @@ -360,6 +363,7 @@ static int virtballoon_probe(struct virtio_device *vdev)
> vb->vdev = vdev;
> vb->need_stats_update = 0;
> vdev->indirect_thresh = indirect_thresh;
> + vdev->indirect_alloc_thresh = indirect_alloc_thresh;
>
> err = init_vqs(vb);
> if (err)
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 4063e03..dde867b 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -93,6 +93,10 @@ struct vring_virtqueue
> */
> unsigned int indirect_thresh;
>
> + /* Buffers below this size will be allocated from cache */
> + unsigned int indirect_alloc_thresh;
> + struct kmem_cache *indirect_cache;
> +
> /* Host publishes avail event idx */
> bool event;
>
> @@ -135,7 +139,10 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
> unsigned head;
> int i;
>
> - desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
> + if ((out + in) <= vq->indirect_alloc_thresh)
> + desc = kmem_cache_alloc(vq->indirect_cache, gfp);
> + else
> + desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
> if (!desc)
> return -ENOMEM;
>
> @@ -384,8 +391,14 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
> i = head;
>
> /* Free the indirect table */
> - if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
> - kfree(phys_to_virt(vq->vring.desc[i].addr));
> + if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) {
> + u32 descs = vq->vring.desc[i].len / sizeof(struct vring_desc);
> + if (descs > vq->indirect_alloc_thresh)
> + kfree(phys_to_virt(vq->vring.desc[i].addr));
> + else
> + kmem_cache_free(vq->indirect_cache,
> + phys_to_virt(vq->vring.desc[i].addr));
> + }
If logic in two chunks above does not match it all
blows up. So let's add a helper is_cache(vq, buf)
and call from both places.
>
> while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
> i = vq->vring.desc[i].next;
> @@ -654,14 +667,25 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
> vq->last_used_idx = 0;
> vq->num_added = 0;
> vq->indirect_thresh = 0;
> + vq->indirect_alloc_thresh = 0;
> + vq->indirect_cache = NULL;
> list_add_tail(&vq->vq.list, &vdev->vqs);
> #ifdef DEBUG
> vq->in_use = false;
> vq->last_add_time_valid = false;
> #endif
>
> - if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
> + if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) {
> vq->indirect_thresh = vdev->indirect_thresh;
> + vq->indirect_alloc_thresh = vdev->indirect_alloc_thresh;
This means for virtio-net we still create a cache for both
TX and RX, but it's later unused for RX unless
big packet mode is set.
Pls make this flag per vq not per device.
> + if (vq->indirect_alloc_thresh) {
> + vq->indirect_cache =
> + KMEM_CACHE(vring_desc[vq->indirect_alloc_thresh], 0);
> +
> + if (vq->indirect_cache == NULL)
> + vq->indirect_alloc_thresh = 0;
> + }
> + }
>
> vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
>
> @@ -685,6 +709,8 @@ EXPORT_SYMBOL_GPL(vring_new_virtqueue);
> void vring_del_virtqueue(struct virtqueue *vq)
> {
> list_del(&vq->list);
> + if (to_vvq(vq)->indirect_cache)
> + kmem_cache_destroy(to_vvq(vq)->indirect_cache);
> kfree(to_vvq(vq));
> }
> EXPORT_SYMBOL_GPL(vring_del_virtqueue);
> diff --git a/include/linux/virtio.h b/include/linux/virtio.h
> index 48bc457..3261c02 100644
> --- a/include/linux/virtio.h
> +++ b/include/linux/virtio.h
> @@ -70,6 +70,7 @@ struct virtio_device {
> unsigned long features[1];
> void *priv;
> unsigned int indirect_thresh;
> + unsigned int indirect_alloc_thresh;
> };
>
> #define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev)
So what is a reasonable value?
It would be such that most bufs have # of s/g below it.
So I think 'expected_sg' would be a better name,
add documentation explaining what it is.
> diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c
> index 418f933..058b6dd 100644
> --- a/net/9p/trans_virtio.c
> +++ b/net/9p/trans_virtio.c
> @@ -55,6 +55,9 @@
> static unsigned int indirect_thresh;
> module_param(indirect_thresh, uint, S_IRUGO);
>
> +static unsigned int indirect_alloc_thresh;
> +module_param(indirect_alloc_thresh, uint, S_IRUGO);
> +
> /* a single mutex to manage channel initialization and attachment */
> static DEFINE_MUTEX(virtio_9p_lock);
> static DECLARE_WAIT_QUEUE_HEAD(vp_wq);
> @@ -505,6 +508,8 @@ static int p9_virtio_probe(struct virtio_device *vdev)
>
> /* We expect one virtqueue, for requests. */
> vdev->indirect_thresh = indirect_thresh;
> + vdev->indirect_alloc_thresh = indirect_alloc_thresh;
> +
> chan->vq = virtio_find_single_vq(vdev, req_done, "requests");
> if (IS_ERR(chan->vq)) {
> err = PTR_ERR(chan->vq);
> --
> 1.7.12
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors
2012-08-30 11:21 [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors Sasha Levin
2012-08-30 11:21 ` [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible Sasha Levin
@ 2012-08-30 14:14 ` Michael S. Tsirkin
1 sibling, 0 replies; 11+ messages in thread
From: Michael S. Tsirkin @ 2012-08-30 14:14 UTC (permalink / raw)
To: Sasha Levin; +Cc: rusty, virtualization, linux-kernel, avi, kvm
On Thu, Aug 30, 2012 at 01:21:57PM +0200, Sasha Levin wrote:
> Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will use indirect
> descriptors even if we have plenty of space in the ring. This means that
> we take a performance hit at all times due to the overhead of creating
> indirect descriptors.
>
> Instead, use it only after we're below a configurable offset.
>
> Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
Some minor comments here and for the next patch.
And I think I see a bug. Otherwise looks good to me.
Let's add MODULE_PARM_DESC clearly explaining
that this is a debugging option and users should
not touch it.
We can also add this text:
Min. number of free space in the ring to trigger direct
descriptor use. Set to 1 to always use direct
descriptors.
Also maybe direct_thresh is a better name.
> ---
> drivers/block/virtio_blk.c | 4 ++++
> drivers/char/hw_random/virtio-rng.c | 4 ++++
> drivers/char/virtio_console.c | 4 ++++
> drivers/net/virtio_net.c | 4 ++++
> drivers/virtio/virtio_balloon.c | 4 ++++
> drivers/virtio/virtio_ring.c | 21 +++++++++++++++------
> include/linux/virtio.h | 1 +
> net/9p/trans_virtio.c | 4 ++++
> 8 files changed, 40 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
> index 2edfb5c..7c63065 100644
> --- a/drivers/block/virtio_blk.c
> +++ b/drivers/block/virtio_blk.c
> @@ -22,6 +22,9 @@ static DEFINE_IDA(vd_index_ida);
>
> struct workqueue_struct *virtblk_wq;
>
> +static unsigned int indirect_thresh;
> +module_param(indirect_thresh, uint, S_IRUGO);
> +
> struct virtio_blk
> {
> struct virtio_device *vdev;
> @@ -735,6 +738,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev)
>
> INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
> vblk->config_enable = true;
> + vdev->indirect_thresh = indirect_thresh;
>
> err = init_vq(vblk);
> if (err)
> diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c
> index 5708299..3a644f1 100644
> --- a/drivers/char/hw_random/virtio-rng.c
> +++ b/drivers/char/hw_random/virtio-rng.c
> @@ -25,6 +25,9 @@
> #include <linux/virtio_rng.h>
> #include <linux/module.h>
>
> +static unsigned int indirect_thresh;
> +module_param(indirect_thresh, uint, S_IRUGO);
> +
> static struct virtqueue *vq;
> static unsigned int data_avail;
> static DECLARE_COMPLETION(have_data);
> @@ -93,6 +96,7 @@ static int probe_common(struct virtio_device *vdev)
> int err;
>
> /* We expect a single virtqueue. */
> + vdev->indirect_thresh = indirect_thresh;
> vq = virtio_find_single_vq(vdev, random_recv_done, "input");
> if (IS_ERR(vq))
> return PTR_ERR(vq);
> diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
> index e88f843..da2e44c 100644
> --- a/drivers/char/virtio_console.c
> +++ b/drivers/char/virtio_console.c
> @@ -39,6 +39,9 @@
> #include <linux/module.h>
> #include "../tty/hvc/hvc_console.h"
>
> +static unsigned int indirect_thresh;
> +module_param(indirect_thresh, uint, S_IRUGO);
> +
> /*
> * This is a global struct for storing common data for all the devices
> * this driver handles.
> @@ -1887,6 +1890,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev)
> max_nr_ports),
> &portdev->config.max_nr_ports) == 0)
> multiport = true;
> + vdev->indirect_thresh = indirect_thresh;
>
> err = init_vqs(portdev);
> if (err < 0) {
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index cbf8b06..949c89e 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -34,6 +34,9 @@ static bool csum = true, gso = true;
> module_param(csum, bool, 0444);
> module_param(gso, bool, 0444);
>
> +static unsigned int indirect_thresh = 16;
> +module_param(indirect_thresh, uint, S_IRUGO);
> +
> /* FIXME: MTU in config. */
> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
> #define GOOD_COPY_LEN 128
> @@ -1128,6 +1131,7 @@ static int virtnet_probe(struct virtio_device *vdev)
>
> if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
> vi->mergeable_rx_bufs = true;
> + vdev->indirect_thresh = indirect_thresh;
>
> err = init_vqs(vi);
> if (err)
> diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
> index 0908e60..ca5ae7a 100644
> --- a/drivers/virtio/virtio_balloon.c
> +++ b/drivers/virtio/virtio_balloon.c
> @@ -35,6 +35,9 @@
> */
> #define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
>
> +static unsigned int indirect_thresh;
> +module_param(indirect_thresh, uint, S_IRUGO);
> +
> struct virtio_balloon
> {
> struct virtio_device *vdev;
> @@ -356,6 +359,7 @@ static int virtballoon_probe(struct virtio_device *vdev)
> init_waitqueue_head(&vb->acked);
> vb->vdev = vdev;
> vb->need_stats_update = 0;
> + vdev->indirect_thresh = indirect_thresh;
>
> err = init_vqs(vb);
> if (err)
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 5aa43c3..4063e03 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -87,8 +87,11 @@ struct vring_virtqueue
> /* Other side has made a mess, don't try any more. */
> bool broken;
>
> - /* Host supports indirect buffers */
> - bool indirect;
> + /*
> + * Min. number of free space in the ring to trigger direct
> + * descriptor use
> + */
> + unsigned int indirect_thresh;
>
> /* Host publishes avail event idx */
> bool event;
> @@ -216,9 +219,12 @@ int virtqueue_add_buf(struct virtqueue *_vq,
> }
> #endif
>
> - /* If the host supports indirect descriptor tables, and we have multiple
> - * buffers, then go indirect. FIXME: tune this threshold */
> - if (vq->indirect && (out + in) > 1 && vq->num_free) {
> + /*
> + * If the host supports indirect descriptor tables, and we have multiple
> + * buffers, then go indirect.
> + */
Fix comments to mention threshold.
> + if ((out + in) > 1 && vq->num_free &&
> + (vq->num_free < vq->indirect_thresh)) {
Please do not add () around conditions like this.
> head = vring_add_indirect(vq, sg, out, in, gfp);
> if (likely(head >= 0))
> goto add_head;
> @@ -647,13 +653,16 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
> vq->broken = false;
> vq->last_used_idx = 0;
> vq->num_added = 0;
> + vq->indirect_thresh = 0;
> list_add_tail(&vq->vq.list, &vdev->vqs);
> #ifdef DEBUG
> vq->in_use = false;
> vq->last_add_time_valid = false;
> #endif
>
> - vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
> + if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
> + vq->indirect_thresh = vdev->indirect_thresh;
> +
cleaner
else
` vq->indirect_thresh = 0;
So when indirect_thresh is 0 (default) this
disables indirect completely. Likely not what you wanted.
Maybe the right thing to do is something like
if (!indirect_thresh)
vq->indirect_thresh = vq->vring.num + 1
?
> vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
>
> /* No callback? Tell other side not to bother us. */
> diff --git a/include/linux/virtio.h b/include/linux/virtio.h
> index a1ba8bb..48bc457 100644
> --- a/include/linux/virtio.h
> +++ b/include/linux/virtio.h
> @@ -69,6 +69,7 @@ struct virtio_device {
> /* Note that this is a Linux set_bit-style bitmap. */
> unsigned long features[1];
> void *priv;
> + unsigned int indirect_thresh;
> };
>
> #define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev)
> diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c
> index 35b8911..418f933 100644
> --- a/net/9p/trans_virtio.c
> +++ b/net/9p/trans_virtio.c
> @@ -52,6 +52,9 @@
>
> #define VIRTQUEUE_NUM 128
>
> +static unsigned int indirect_thresh;
> +module_param(indirect_thresh, uint, S_IRUGO);
> +
> /* a single mutex to manage channel initialization and attachment */
> static DEFINE_MUTEX(virtio_9p_lock);
> static DECLARE_WAIT_QUEUE_HEAD(vp_wq);
> @@ -501,6 +504,7 @@ static int p9_virtio_probe(struct virtio_device *vdev)
> chan->vdev = vdev;
>
> /* We expect one virtqueue, for requests. */
> + vdev->indirect_thresh = indirect_thresh;
> chan->vq = virtio_find_single_vq(vdev, req_done, "requests");
> if (IS_ERR(chan->vq)) {
> err = PTR_ERR(chan->vq);
> --
> 1.7.12
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-08-30 13:38 ` Michael S. Tsirkin
@ 2012-08-31 9:36 ` Sasha Levin
2012-08-31 9:56 ` Michael S. Tsirkin
0 siblings, 1 reply; 11+ messages in thread
From: Sasha Levin @ 2012-08-31 9:36 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: rusty, virtualization, linux-kernel, avi, kvm
On 08/30/2012 03:38 PM, Michael S. Tsirkin wrote:
>> +static unsigned int indirect_alloc_thresh = 16;
> Why 16? Please make is MAX_SG + 1 this makes some sense.
Wouldn't MAX_SG mean we always allocate from the cache? Isn't the memory waste
too big in this case?
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-08-31 9:36 ` Sasha Levin
@ 2012-08-31 9:56 ` Michael S. Tsirkin
2012-09-04 16:34 ` Avi Kivity
0 siblings, 1 reply; 11+ messages in thread
From: Michael S. Tsirkin @ 2012-08-31 9:56 UTC (permalink / raw)
To: Sasha Levin; +Cc: rusty, virtualization, linux-kernel, avi, kvm
On Fri, Aug 31, 2012 at 11:36:07AM +0200, Sasha Levin wrote:
> On 08/30/2012 03:38 PM, Michael S. Tsirkin wrote:
> >> +static unsigned int indirect_alloc_thresh = 16;
> > Why 16? Please make is MAX_SG + 1 this makes some sense.
>
> Wouldn't MAX_SG mean we always allocate from the cache? Isn't the memory waste
> too big in this case?
Sorry. I really meant MAX_SKB_FRAGS + 1. MAX_SKB_FRAGS is 17 so gets us
threshold of 18. It is less than the size of an skb+shinfo itself so -
does it look too big to you? Also why do you think 16 is not too big but
18 is? If there's a reason then I am fine with 16 too but then please
put it in code comment near where the value is set.
Yes this means virtio net always allocates from cache
but this is a good thing, isn't it? Gets us more consistent
performance.
--
MST
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-08-31 9:56 ` Michael S. Tsirkin
@ 2012-09-04 16:34 ` Avi Kivity
2012-09-04 16:36 ` Avi Kivity
2012-09-04 18:41 ` Michael S. Tsirkin
0 siblings, 2 replies; 11+ messages in thread
From: Avi Kivity @ 2012-09-04 16:34 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: Sasha Levin, rusty, virtualization, linux-kernel, kvm
On 08/31/2012 12:56 PM, Michael S. Tsirkin wrote:
> On Fri, Aug 31, 2012 at 11:36:07AM +0200, Sasha Levin wrote:
>> On 08/30/2012 03:38 PM, Michael S. Tsirkin wrote:
>> >> +static unsigned int indirect_alloc_thresh = 16;
>> > Why 16? Please make is MAX_SG + 1 this makes some sense.
>>
>> Wouldn't MAX_SG mean we always allocate from the cache? Isn't the memory waste
>> too big in this case?
>
> Sorry. I really meant MAX_SKB_FRAGS + 1. MAX_SKB_FRAGS is 17 so gets us
> threshold of 18. It is less than the size of an skb+shinfo itself so -
> does it look too big to you? Also why do you think 16 is not too big but
> 18 is? If there's a reason then I am fine with 16 too but then please
> put it in code comment near where the value is set.
>
> Yes this means virtio net always allocates from cache
> but this is a good thing, isn't it? Gets us more consistent
> performance.
kmalloc() also goes to a cache. Is there a measurable difference?
Ugh, there's an ugly loop in __find_general_cachep(), which really wants
to be replaced with fls().
--
error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-09-04 16:34 ` Avi Kivity
@ 2012-09-04 16:36 ` Avi Kivity
2012-09-04 18:41 ` Michael S. Tsirkin
1 sibling, 0 replies; 11+ messages in thread
From: Avi Kivity @ 2012-09-04 16:36 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: Sasha Levin, rusty, virtualization, linux-kernel, kvm
On 09/04/2012 07:34 PM, Avi Kivity wrote:
> On 08/31/2012 12:56 PM, Michael S. Tsirkin wrote:
>> On Fri, Aug 31, 2012 at 11:36:07AM +0200, Sasha Levin wrote:
>>> On 08/30/2012 03:38 PM, Michael S. Tsirkin wrote:
>>> >> +static unsigned int indirect_alloc_thresh = 16;
>>> > Why 16? Please make is MAX_SG + 1 this makes some sense.
>>>
>>> Wouldn't MAX_SG mean we always allocate from the cache? Isn't the memory waste
>>> too big in this case?
>>
>> Sorry. I really meant MAX_SKB_FRAGS + 1. MAX_SKB_FRAGS is 17 so gets us
>> threshold of 18. It is less than the size of an skb+shinfo itself so -
>> does it look too big to you? Also why do you think 16 is not too big but
>> 18 is? If there's a reason then I am fine with 16 too but then please
>> put it in code comment near where the value is set.
>>
>> Yes this means virtio net always allocates from cache
>> but this is a good thing, isn't it? Gets us more consistent
>> performance.
>
> kmalloc() also goes to a cache. Is there a measurable difference?
>
> Ugh, there's an ugly loop in __find_general_cachep(), which really wants
> to be replaced with fls().
>
Actually, not, as the loop will be very short for small sizes.
--
error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-09-04 16:34 ` Avi Kivity
2012-09-04 16:36 ` Avi Kivity
@ 2012-09-04 18:41 ` Michael S. Tsirkin
2012-09-05 14:21 ` Avi Kivity
1 sibling, 1 reply; 11+ messages in thread
From: Michael S. Tsirkin @ 2012-09-04 18:41 UTC (permalink / raw)
To: Avi Kivity; +Cc: Sasha Levin, rusty, virtualization, linux-kernel, kvm
On Tue, Sep 04, 2012 at 07:34:19PM +0300, Avi Kivity wrote:
> On 08/31/2012 12:56 PM, Michael S. Tsirkin wrote:
> > On Fri, Aug 31, 2012 at 11:36:07AM +0200, Sasha Levin wrote:
> >> On 08/30/2012 03:38 PM, Michael S. Tsirkin wrote:
> >> >> +static unsigned int indirect_alloc_thresh = 16;
> >> > Why 16? Please make is MAX_SG + 1 this makes some sense.
> >>
> >> Wouldn't MAX_SG mean we always allocate from the cache? Isn't the memory waste
> >> too big in this case?
> >
> > Sorry. I really meant MAX_SKB_FRAGS + 1. MAX_SKB_FRAGS is 17 so gets us
> > threshold of 18. It is less than the size of an skb+shinfo itself so -
> > does it look too big to you? Also why do you think 16 is not too big but
> > 18 is? If there's a reason then I am fine with 16 too but then please
> > put it in code comment near where the value is set.
> >
> > Yes this means virtio net always allocates from cache
> > but this is a good thing, isn't it? Gets us more consistent
> > performance.
>
> kmalloc() also goes to a cache. Is there a measurable difference?
Yes see 0/2 and followup discussion.
> Ugh, there's an ugly loop in __find_general_cachep(), which really wants
> to be replaced with fls().
>
> --
> error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-09-04 18:41 ` Michael S. Tsirkin
@ 2012-09-05 14:21 ` Avi Kivity
2012-09-05 14:27 ` Michael S. Tsirkin
0 siblings, 1 reply; 11+ messages in thread
From: Avi Kivity @ 2012-09-05 14:21 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: Sasha Levin, rusty, virtualization, linux-kernel, kvm
On 09/04/2012 09:41 PM, Michael S. Tsirkin wrote:
> On Tue, Sep 04, 2012 at 07:34:19PM +0300, Avi Kivity wrote:
>> On 08/31/2012 12:56 PM, Michael S. Tsirkin wrote:
>> > On Fri, Aug 31, 2012 at 11:36:07AM +0200, Sasha Levin wrote:
>> >> On 08/30/2012 03:38 PM, Michael S. Tsirkin wrote:
>> >> >> +static unsigned int indirect_alloc_thresh = 16;
>> >> > Why 16? Please make is MAX_SG + 1 this makes some sense.
>> >>
>> >> Wouldn't MAX_SG mean we always allocate from the cache? Isn't the memory waste
>> >> too big in this case?
>> >
>> > Sorry. I really meant MAX_SKB_FRAGS + 1. MAX_SKB_FRAGS is 17 so gets us
>> > threshold of 18. It is less than the size of an skb+shinfo itself so -
>> > does it look too big to you? Also why do you think 16 is not too big but
>> > 18 is? If there's a reason then I am fine with 16 too but then please
>> > put it in code comment near where the value is set.
>> >
>> > Yes this means virtio net always allocates from cache
>> > but this is a good thing, isn't it? Gets us more consistent
>> > performance.
>>
>> kmalloc() also goes to a cache. Is there a measurable difference?
>
> Yes see 0/2 and followup discussion.
I don't see 0/2, looks like this was not threaded properly. What was
the subject line?
--
error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible
2012-09-05 14:21 ` Avi Kivity
@ 2012-09-05 14:27 ` Michael S. Tsirkin
0 siblings, 0 replies; 11+ messages in thread
From: Michael S. Tsirkin @ 2012-09-05 14:27 UTC (permalink / raw)
To: Avi Kivity; +Cc: Sasha Levin, rusty, virtualization, linux-kernel, kvm
On Wed, Sep 05, 2012 at 05:21:12PM +0300, Avi Kivity wrote:
> On 09/04/2012 09:41 PM, Michael S. Tsirkin wrote:
> > On Tue, Sep 04, 2012 at 07:34:19PM +0300, Avi Kivity wrote:
> >> On 08/31/2012 12:56 PM, Michael S. Tsirkin wrote:
> >> > On Fri, Aug 31, 2012 at 11:36:07AM +0200, Sasha Levin wrote:
> >> >> On 08/30/2012 03:38 PM, Michael S. Tsirkin wrote:
> >> >> >> +static unsigned int indirect_alloc_thresh = 16;
> >> >> > Why 16? Please make is MAX_SG + 1 this makes some sense.
> >> >>
> >> >> Wouldn't MAX_SG mean we always allocate from the cache? Isn't the memory waste
> >> >> too big in this case?
> >> >
> >> > Sorry. I really meant MAX_SKB_FRAGS + 1. MAX_SKB_FRAGS is 17 so gets us
> >> > threshold of 18. It is less than the size of an skb+shinfo itself so -
> >> > does it look too big to you? Also why do you think 16 is not too big but
> >> > 18 is? If there's a reason then I am fine with 16 too but then please
> >> > put it in code comment near where the value is set.
> >> >
> >> > Yes this means virtio net always allocates from cache
> >> > but this is a good thing, isn't it? Gets us more consistent
> >> > performance.
> >>
> >> kmalloc() also goes to a cache. Is there a measurable difference?
> >
> > Yes see 0/2 and followup discussion.
>
> I don't see 0/2, looks like this was not threaded properly. What was
> the subject line?
My mistake, there is no 0/2, the resolts where in the followup thread
of the previous version:
[PATCH v2 2/2] virtio-ring: Allocate indirect buffers from cache when possible
Sasha, could you please accompany the next version
with a cover letter 0/2) including performance results?
>
> --
> error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 11+ messages in thread
end of thread, other threads:[~2012-09-05 14:26 UTC | newest]
Thread overview: 11+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2012-08-30 11:21 [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors Sasha Levin
2012-08-30 11:21 ` [PATCH v3 2/2] virtio-ring: Allocate indirect buffers from cache when possible Sasha Levin
2012-08-30 13:38 ` Michael S. Tsirkin
2012-08-31 9:36 ` Sasha Levin
2012-08-31 9:56 ` Michael S. Tsirkin
2012-09-04 16:34 ` Avi Kivity
2012-09-04 16:36 ` Avi Kivity
2012-09-04 18:41 ` Michael S. Tsirkin
2012-09-05 14:21 ` Avi Kivity
2012-09-05 14:27 ` Michael S. Tsirkin
2012-08-30 14:14 ` [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors Michael S. Tsirkin
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).