[PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Michael Dalton]

skb_page_frag_refill currently permits only order-0 page allocs
unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
higher-order page allocations whether or not GFP_WAIT is used. If
memory cannot be allocated, the allocator will fall back to
successively smaller page allocs (down to order-0 page allocs).

This change brings skb_page_frag_refill in line with the existing
page allocation strategy employed by netdev_alloc_frag, which attempts
higher-order page allocations whether or not GFP_WAIT is set, falling
back to successively lower-order page allocations on failure. Part
of migration of virtio-net to per-receive queue page frag allocators.

Signed-off-by: Michael Dalton <mwdalton@google.com>
---
 net/core/sock.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/net/core/sock.c b/net/core/sock.c
index ab20ed9..7383d23 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
 		put_page(pfrag->page);
 	}
 
-	/* We restrict high order allocations to users that can afford to wait */
-	order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
-
+	order = SKB_FRAG_PAGE_ORDER;
 	do {
 		gfp_t gfp = prio;
 
-- 
1.8.5.1

[PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Michael Dalton]

The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
mergeable rx buffer allocations. This commit migrates virtio-net to use
per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
mergeable rx buffer memory allocation, which now will use skb_refill_frag()
for both atomic and GFP-WAIT buffer allocations.

To address fragmentation concerns, if after buffer allocation there
is too little space left in the page frag to allocate a subsequent
buffer, the remaining space is added to the current allocated buffer
so that the remaining space can be used to store packet data.

Signed-off-by: Michael Dalton <mwdalton@google.com>
---
 drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 31 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a988..d38d130 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -78,6 +78,9 @@ struct receive_queue {
 	/* Chain pages by the private ptr. */
 	struct page *pages;
 
+	/* Page frag for GFP_ATOMIC packet buffer allocation. */
+	struct page_frag atomic_frag;
+
 	/* RX: fragments + linear part + virtio header */
 	struct scatterlist sg[MAX_SKB_FRAGS + 2];
 
@@ -127,9 +130,9 @@ struct virtnet_info {
 	struct mutex config_lock;
 
 	/* Page_frag for GFP_KERNEL packet buffer allocation when we run
-	 * low on memory.
+	 * low on memory. May sleep.
 	 */
-	struct page_frag alloc_frag;
+	struct page_frag sleep_frag;
 
 	/* Does the affinity hint is set for virtqueues? */
 	bool affinity_hint_set;
@@ -336,8 +339,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	int num_buf = hdr->mhdr.num_buffers;
 	struct page *page = virt_to_head_page(buf);
 	int offset = buf - page_address(page);
-	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len,
-					       MERGE_BUFFER_LEN);
+	int truesize = max_t(int, len, MERGE_BUFFER_LEN);
+	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
 	struct sk_buff *curr_skb = head_skb;
 
 	if (unlikely(!curr_skb))
@@ -353,11 +356,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 			dev->stats.rx_length_errors++;
 			goto err_buf;
 		}
-		if (unlikely(len > MERGE_BUFFER_LEN)) {
-			pr_debug("%s: rx error: merge buffer too long\n",
-				 dev->name);
-			len = MERGE_BUFFER_LEN;
-		}
 
 		page = virt_to_head_page(buf);
 		--rq->num;
@@ -376,19 +374,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 			head_skb->truesize += nskb->truesize;
 			num_skb_frags = 0;
 		}
+		truesize = max_t(int, len, MERGE_BUFFER_LEN);
 		if (curr_skb != head_skb) {
 			head_skb->data_len += len;
 			head_skb->len += len;
-			head_skb->truesize += MERGE_BUFFER_LEN;
+			head_skb->truesize += truesize;
 		}
 		offset = buf - page_address(page);
 		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
 			put_page(page);
 			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
-					     len, MERGE_BUFFER_LEN);
+					     len, truesize);
 		} else {
 			skb_add_rx_frag(curr_skb, num_skb_frags, page,
-					offset, len, MERGE_BUFFER_LEN);
+					offset, len, truesize);
 		}
 	}
 
@@ -579,24 +578,24 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
 static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
 {
 	struct virtnet_info *vi = rq->vq->vdev->priv;
-	char *buf = NULL;
-	int err;
+	struct page_frag *alloc_frag;
+	char *buf;
+	int err, len, hole;
 
-	if (gfp & __GFP_WAIT) {
-		if (skb_page_frag_refill(MERGE_BUFFER_LEN, &vi->alloc_frag,
-					 gfp)) {
-			buf = (char *)page_address(vi->alloc_frag.page) +
-			      vi->alloc_frag.offset;
-			get_page(vi->alloc_frag.page);
-			vi->alloc_frag.offset += MERGE_BUFFER_LEN;
-		}
-	} else {
-		buf = netdev_alloc_frag(MERGE_BUFFER_LEN);
-	}
-	if (!buf)
+	alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
+	if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
 		return -ENOMEM;
+	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
+	get_page(alloc_frag->page);
+	len = MERGE_BUFFER_LEN;
+	alloc_frag->offset += len;
+	hole = alloc_frag->size - alloc_frag->offset;
+	if (hole < MERGE_BUFFER_LEN) {
+		len += hole;
+		alloc_frag->offset += hole;
+	}
 
-	sg_init_one(rq->sg, buf, MERGE_BUFFER_LEN);
+	sg_init_one(rq->sg, buf, len);
 	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
 	if (err < 0)
 		put_page(virt_to_head_page(buf));
@@ -1377,6 +1376,16 @@ static void free_receive_bufs(struct virtnet_info *vi)
 	}
 }
 
+static void free_receive_page_frags(struct virtnet_info *vi)
+{
+	int i;
+	for (i = 0; i < vi->max_queue_pairs; i++)
+		if (vi->rq[i].atomic_frag.page)
+			put_page(vi->rq[i].atomic_frag.page);
+	if (vi->sleep_frag.page)
+		put_page(vi->sleep_frag.page);
+}
+
 static void free_unused_bufs(struct virtnet_info *vi)
 {
 	void *buf;
@@ -1706,8 +1715,7 @@ free_recv_bufs:
 free_vqs:
 	cancel_delayed_work_sync(&vi->refill);
 	virtnet_del_vqs(vi);
-	if (vi->alloc_frag.page)
-		put_page(vi->alloc_frag.page);
+	free_receive_page_frags(vi);
 free_stats:
 	free_percpu(vi->stats);
 free:
@@ -1741,8 +1749,7 @@ static void virtnet_remove(struct virtio_device *vdev)
 	unregister_netdev(vi->dev);
 
 	remove_vq_common(vi);
-	if (vi->alloc_frag.page)
-		put_page(vi->alloc_frag.page);
+	free_receive_page_frags(vi);
 
 	flush_work(&vi->config_work);
 
-- 
1.8.5.1

[PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael Dalton]

Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads.  For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.

This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.

These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.

Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.

net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs):  13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s

Signed-off-by: Michael Dalton <mwdalton@google.com>
---
 drivers/net/virtio_net.c | 63 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 46 insertions(+), 17 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index d38d130..904af37 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -26,6 +26,7 @@
 #include <linux/if_vlan.h>
 #include <linux/slab.h>
 #include <linux/cpu.h>
+#include <linux/average.h>
 
 static int napi_weight = NAPI_POLL_WEIGHT;
 module_param(napi_weight, int, 0444);
@@ -36,11 +37,15 @@ module_param(gso, bool, 0444);
 
 /* FIXME: MTU in config. */
 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
-#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \
-                                sizeof(struct virtio_net_hdr_mrg_rxbuf), \
-                                L1_CACHE_BYTES))
 #define GOOD_COPY_LEN	128
 
+/* Weight used for the RX packet size EWMA. The average packet size is used to
+ * determine the packet buffer size when refilling RX rings. As the entire RX
+ * ring may be refilled at once, the weight is chosen so that the EWMA will be
+ * insensitive to short-term, transient changes in packet size.
+ */
+#define RECEIVE_AVG_WEIGHT 64
+
 #define VIRTNET_DRIVER_VERSION "1.0.0"
 
 struct virtnet_stats {
@@ -78,6 +83,9 @@ struct receive_queue {
 	/* Chain pages by the private ptr. */
 	struct page *pages;
 
+	/* Average packet length for mergeable receive buffers. */
+	struct ewma mrg_avg_pkt_len;
+
 	/* Page frag for GFP_ATOMIC packet buffer allocation. */
 	struct page_frag atomic_frag;
 
@@ -339,13 +347,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	int num_buf = hdr->mhdr.num_buffers;
 	struct page *page = virt_to_head_page(buf);
 	int offset = buf - page_address(page);
-	int truesize = max_t(int, len, MERGE_BUFFER_LEN);
-	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
+	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, len);
 	struct sk_buff *curr_skb = head_skb;
 
 	if (unlikely(!curr_skb))
 		goto err_skb;
-
 	while (--num_buf) {
 		int num_skb_frags;
 
@@ -374,23 +380,40 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 			head_skb->truesize += nskb->truesize;
 			num_skb_frags = 0;
 		}
-		truesize = max_t(int, len, MERGE_BUFFER_LEN);
 		if (curr_skb != head_skb) {
 			head_skb->data_len += len;
 			head_skb->len += len;
-			head_skb->truesize += truesize;
+			head_skb->truesize += len;
 		}
 		offset = buf - page_address(page);
 		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
 			put_page(page);
 			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
-					     len, truesize);
+					     len, len);
 		} else {
 			skb_add_rx_frag(curr_skb, num_skb_frags, page,
-					offset, len, truesize);
+					offset, len, len);
 		}
 	}
 
+	/* All frags before the last frag are fully used -- for those frags,
+	 * truesize = len. Use the size of the most recent buffer allocation
+	 * from the last frag's page to estimate the truesize of the last frag.
+	 * EWMA with a weight of 64 makes the size adjustments quite small in
+	 * the frags allocated on one page (even a order-3 one), and truesize
+	 * doesn't need to be 100% accurate.
+	 */
+	if (skb_is_nonlinear(head_skb)) {
+		u32 est_buffer_len = page_private(page);
+		if (est_buffer_len > len) {
+			u32 truesize_delta = est_buffer_len - len;
+
+			curr_skb->truesize += truesize_delta;
+			if (curr_skb != head_skb)
+				head_skb->truesize += truesize_delta;
+		}
+	}
+	ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
 	return head_skb;
 
 err_skb:
@@ -578,24 +601,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
 static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
 {
 	struct virtnet_info *vi = rq->vq->vdev->priv;
+	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 	struct page_frag *alloc_frag;
 	char *buf;
-	int err, len, hole;
+	int err, hole;
+	u32 buflen;
 
+	buflen = hdr_len + clamp_t(u32, ewma_read(&rq->mrg_avg_pkt_len),
+				   GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
+	buflen = ALIGN(buflen, L1_CACHE_BYTES);
 	alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
-	if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
+	if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, gfp)))
 		return -ENOMEM;
 	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
 	get_page(alloc_frag->page);
-	len = MERGE_BUFFER_LEN;
-	alloc_frag->offset += len;
+	alloc_frag->offset += buflen;
+	set_page_private(alloc_frag->page, buflen);
 	hole = alloc_frag->size - alloc_frag->offset;
-	if (hole < MERGE_BUFFER_LEN) {
-		len += hole;
+	if (hole < buflen) {
+		buflen += hole;
 		alloc_frag->offset += hole;
 	}
 
-	sg_init_one(rq->sg, buf, len);
+	sg_init_one(rq->sg, buf, buflen);
 	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
 	if (err < 0)
 		put_page(virt_to_head_page(buf));
@@ -1516,6 +1544,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
 			       napi_weight);
 
 		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
+		ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
 		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
 	}
 
-- 
1.8.5.1

[PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael Dalton]

Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads.  For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.

This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.

These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.

Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.

net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs):  13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s

Signed-off-by: Michael Dalton <mwdalton@google.com>
---
 drivers/net/virtio_net.c | 63 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 46 insertions(+), 17 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index d38d130..904af37 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -26,6 +26,7 @@
 #include <linux/if_vlan.h>
 #include <linux/slab.h>
 #include <linux/cpu.h>
+#include <linux/average.h>
 
 static int napi_weight = NAPI_POLL_WEIGHT;
 module_param(napi_weight, int, 0444);
@@ -36,11 +37,15 @@ module_param(gso, bool, 0444);
 
 /* FIXME: MTU in config. */
 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
-#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \
-                                sizeof(struct virtio_net_hdr_mrg_rxbuf), \
-                                L1_CACHE_BYTES))
 #define GOOD_COPY_LEN	128
 
+/* Weight used for the RX packet size EWMA. The average packet size is used to
+ * determine the packet buffer size when refilling RX rings. As the entire RX
+ * ring may be refilled at once, the weight is chosen so that the EWMA will be
+ * insensitive to short-term, transient changes in packet size.
+ */
+#define RECEIVE_AVG_WEIGHT 64
+
 #define VIRTNET_DRIVER_VERSION "1.0.0"
 
 struct virtnet_stats {
@@ -78,6 +83,9 @@ struct receive_queue {
 	/* Chain pages by the private ptr. */
 	struct page *pages;
 
+	/* Average packet length for mergeable receive buffers. */
+	struct ewma mrg_avg_pkt_len;
+
 	/* Page frag for GFP_ATOMIC packet buffer allocation. */
 	struct page_frag atomic_frag;
 
@@ -339,13 +347,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	int num_buf = hdr->mhdr.num_buffers;
 	struct page *page = virt_to_head_page(buf);
 	int offset = buf - page_address(page);
-	int truesize = max_t(int, len, MERGE_BUFFER_LEN);
-	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
+	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, len);
 	struct sk_buff *curr_skb = head_skb;
 
 	if (unlikely(!curr_skb))
 		goto err_skb;
-
 	while (--num_buf) {
 		int num_skb_frags;
 
@@ -374,23 +380,40 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 			head_skb->truesize += nskb->truesize;
 			num_skb_frags = 0;
 		}
-		truesize = max_t(int, len, MERGE_BUFFER_LEN);
 		if (curr_skb != head_skb) {
 			head_skb->data_len += len;
 			head_skb->len += len;
-			head_skb->truesize += truesize;
+			head_skb->truesize += len;
 		}
 		offset = buf - page_address(page);
 		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
 			put_page(page);
 			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
-					     len, truesize);
+					     len, len);
 		} else {
 			skb_add_rx_frag(curr_skb, num_skb_frags, page,
-					offset, len, truesize);
+					offset, len, len);
 		}
 	}
 
+	/* All frags before the last frag are fully used -- for those frags,
+	 * truesize = len. Use the size of the most recent buffer allocation
+	 * from the last frag's page to estimate the truesize of the last frag.
+	 * EWMA with a weight of 64 makes the size adjustments quite small in
+	 * the frags allocated on one page (even a order-3 one), and truesize
+	 * doesn't need to be 100% accurate.
+	 */
+	if (skb_is_nonlinear(head_skb)) {
+		u32 est_buffer_len = page_private(page);
+		if (est_buffer_len > len) {
+			u32 truesize_delta = est_buffer_len - len;
+
+			curr_skb->truesize += truesize_delta;
+			if (curr_skb != head_skb)
+				head_skb->truesize += truesize_delta;
+		}
+	}
+	ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
 	return head_skb;
 
 err_skb:
@@ -578,24 +601,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
 static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
 {
 	struct virtnet_info *vi = rq->vq->vdev->priv;
+	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 	struct page_frag *alloc_frag;
 	char *buf;
-	int err, len, hole;
+	int err, hole;
+	u32 buflen;
 
+	buflen = hdr_len + clamp_t(u32, ewma_read(&rq->mrg_avg_pkt_len),
+				   GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
+	buflen = ALIGN(buflen, L1_CACHE_BYTES);
 	alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
-	if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
+	if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, gfp)))
 		return -ENOMEM;
 	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
 	get_page(alloc_frag->page);
-	len = MERGE_BUFFER_LEN;
-	alloc_frag->offset += len;
+	alloc_frag->offset += buflen;
+	set_page_private(alloc_frag->page, buflen);
 	hole = alloc_frag->size - alloc_frag->offset;
-	if (hole < MERGE_BUFFER_LEN) {
-		len += hole;
+	if (hole < buflen) {
+		buflen += hole;
 		alloc_frag->offset += hole;
 	}
 
-	sg_init_one(rq->sg, buf, len);
+	sg_init_one(rq->sg, buf, buflen);
 	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
 	if (err < 0)
 		put_page(virt_to_head_page(buf));
@@ -1516,6 +1544,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
 			       napi_weight);
 
 		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
+		ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
 		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
 	}
 
-- 
1.8.5.1

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[David Miller]

Can I get some reviews of this series from virtio folks?

Thanks.

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Jason Wang]

On 12/17/2013 08:16 AM, Michael Dalton wrote:
> skb_page_frag_refill currently permits only order-0 page allocs
> unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
> higher-order page allocations whether or not GFP_WAIT is used. If
> memory cannot be allocated, the allocator will fall back to
> successively smaller page allocs (down to order-0 page allocs).
>
> This change brings skb_page_frag_refill in line with the existing
> page allocation strategy employed by netdev_alloc_frag, which attempts
> higher-order page allocations whether or not GFP_WAIT is set, falling
> back to successively lower-order page allocations on failure. Part
> of migration of virtio-net to per-receive queue page frag allocators.
>
> Signed-off-by: Michael Dalton <mwdalton@google.com>
> ---
>  net/core/sock.c | 4 +---
>  1 file changed, 1 insertion(+), 3 deletions(-)
>
> diff --git a/net/core/sock.c b/net/core/sock.c
> index ab20ed9..7383d23 100644
> --- a/net/core/sock.c
> +++ b/net/core/sock.c
> @@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
>  		put_page(pfrag->page);
>  	}
>  
> -	/* We restrict high order allocations to users that can afford to wait */
> -	order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
> -
> +	order = SKB_FRAG_PAGE_ORDER;
>  	do {
>  		gfp_t gfp = prio;
>  

The original code seems try to avoid the high order allocation for
atomic allocation. This patch changes this, and looks like it will
introduce some extra cost when the memory is highly fragmented.

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Jason Wang]

On 12/17/2013 08:16 AM, Michael Dalton wrote:
> The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
> mergeable rx buffer allocations. This commit migrates virtio-net to use
> per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
> mergeable rx buffer memory allocation, which now will use skb_refill_frag()
> for both atomic and GFP-WAIT buffer allocations.
>
> To address fragmentation concerns, if after buffer allocation there
> is too little space left in the page frag to allocate a subsequent
> buffer, the remaining space is added to the current allocated buffer
> so that the remaining space can be used to store packet data.
>
> Signed-off-by: Michael Dalton <mwdalton@google.com>
> ---
>  drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
>  1 file changed, 38 insertions(+), 31 deletions(-)
>
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index c51a988..d38d130 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -78,6 +78,9 @@ struct receive_queue {
>  	/* Chain pages by the private ptr. */
>  	struct page *pages;
>  
> +	/* Page frag for GFP_ATOMIC packet buffer allocation. */
> +	struct page_frag atomic_frag;
> +
>  	/* RX: fragments + linear part + virtio header */
>  	struct scatterlist sg[MAX_SKB_FRAGS + 2];
>  
> @@ -127,9 +130,9 @@ struct virtnet_info {
>  	struct mutex config_lock;
>  
>  	/* Page_frag for GFP_KERNEL packet buffer allocation when we run
> -	 * low on memory.
> +	 * low on memory. May sleep.
>  	 */
> -	struct page_frag alloc_frag;
> +	struct page_frag sleep_frag;

Any reason to use two different page_frag consider only
skb_page_frag_refill() is used?
>  
>  	/* Does the affinity hint is set for virtqueues? */
>  	bool affinity_hint_set;
> @@ -336,8 +339,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  	int num_buf = hdr->mhdr.num_buffers;
>  	struct page *page = virt_to_head_page(buf);
>  	int offset = buf - page_address(page);
> -	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len,
> -					       MERGE_BUFFER_LEN);
> +	int truesize = max_t(int, len, MERGE_BUFFER_LEN);
> +	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
>  	struct sk_buff *curr_skb = head_skb;
>  
>  	if (unlikely(!curr_skb))
> @@ -353,11 +356,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  			dev->stats.rx_length_errors++;
>  			goto err_buf;
>  		}
> -		if (unlikely(len > MERGE_BUFFER_LEN)) {
> -			pr_debug("%s: rx error: merge buffer too long\n",
> -				 dev->name);
> -			len = MERGE_BUFFER_LEN;
> -		}
>  
>  		page = virt_to_head_page(buf);
>  		--rq->num;
> @@ -376,19 +374,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  			head_skb->truesize += nskb->truesize;
>  			num_skb_frags = 0;
>  		}
> +		truesize = max_t(int, len, MERGE_BUFFER_LEN);
>  		if (curr_skb != head_skb) {
>  			head_skb->data_len += len;
>  			head_skb->len += len;
> -			head_skb->truesize += MERGE_BUFFER_LEN;
> +			head_skb->truesize += truesize;
>  		}
>  		offset = buf - page_address(page);
>  		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
>  			put_page(page);
>  			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
> -					     len, MERGE_BUFFER_LEN);
> +					     len, truesize);
>  		} else {
>  			skb_add_rx_frag(curr_skb, num_skb_frags, page,
> -					offset, len, MERGE_BUFFER_LEN);
> +					offset, len, truesize);
>  		}
>  	}
>  
> @@ -579,24 +578,24 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
>  static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
>  {
>  	struct virtnet_info *vi = rq->vq->vdev->priv;
> -	char *buf = NULL;
> -	int err;
> +	struct page_frag *alloc_frag;
> +	char *buf;
> +	int err, len, hole;
>  
> -	if (gfp & __GFP_WAIT) {
> -		if (skb_page_frag_refill(MERGE_BUFFER_LEN, &vi->alloc_frag,
> -					 gfp)) {
> -			buf = (char *)page_address(vi->alloc_frag.page) +
> -			      vi->alloc_frag.offset;
> -			get_page(vi->alloc_frag.page);
> -			vi->alloc_frag.offset += MERGE_BUFFER_LEN;
> -		}
> -	} else {
> -		buf = netdev_alloc_frag(MERGE_BUFFER_LEN);
> -	}
> -	if (!buf)
> +	alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
> +	if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
>  		return -ENOMEM;
> +	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
> +	get_page(alloc_frag->page);
> +	len = MERGE_BUFFER_LEN;
> +	alloc_frag->offset += len;
> +	hole = alloc_frag->size - alloc_frag->offset;
> +	if (hole < MERGE_BUFFER_LEN) {
> +		len += hole;
> +		alloc_frag->offset += hole;
> +	}
>  
> -	sg_init_one(rq->sg, buf, MERGE_BUFFER_LEN);
> +	sg_init_one(rq->sg, buf, len);

I wonder whether we can use get_a_page() and give_pages() to recycle the
pages like before which may help the performance. We can also do some
optimizations for this in vhost.
>  	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
>  	if (err < 0)
>  		put_page(virt_to_head_page(buf));
> @@ -1377,6 +1376,16 @@ static void free_receive_bufs(struct virtnet_info *vi)
>  	}
>  }
>  
> +static void free_receive_page_frags(struct virtnet_info *vi)
> +{
> +	int i;
> +	for (i = 0; i < vi->max_queue_pairs; i++)
> +		if (vi->rq[i].atomic_frag.page)
> +			put_page(vi->rq[i].atomic_frag.page);
> +	if (vi->sleep_frag.page)
> +		put_page(vi->sleep_frag.page);
> +}
> +
>  static void free_unused_bufs(struct virtnet_info *vi)
>  {
>  	void *buf;
> @@ -1706,8 +1715,7 @@ free_recv_bufs:
>  free_vqs:
>  	cancel_delayed_work_sync(&vi->refill);
>  	virtnet_del_vqs(vi);
> -	if (vi->alloc_frag.page)
> -		put_page(vi->alloc_frag.page);
> +	free_receive_page_frags(vi);
>  free_stats:
>  	free_percpu(vi->stats);
>  free:
> @@ -1741,8 +1749,7 @@ static void virtnet_remove(struct virtio_device *vdev)
>  	unregister_netdev(vi->dev);
>  
>  	remove_vq_common(vi);
> -	if (vi->alloc_frag.page)
> -		put_page(vi->alloc_frag.page);
> +	free_receive_page_frags(vi);
>  
>  	flush_work(&vi->config_work);
>  

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael S. Tsirkin]

On Mon, Dec 16, 2013 at 04:16:29PM -0800, Michael Dalton wrote:
> Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
> allocators") changed the mergeable receive buffer size from PAGE_SIZE to
> MTU-size, introducing a single-stream regression for benchmarks with large
> average packet size. There is no single optimal buffer size for all
> workloads.  For workloads with packet size <= MTU bytes, MTU + virtio-net
> header-sized buffers are preferred as larger buffers reduce the TCP window
> due to SKB truesize. However, single-stream workloads with large average
> packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
> are used.
> 
> This commit auto-tunes the mergeable receiver buffer packet size by
> choosing the packet buffer size based on an EWMA of the recent packet
> sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
> virtio-net header len to PAGE_SIZE. This improves throughput for
> large packet workloads, as any workload with average packet size >=
> PAGE_SIZE will use PAGE_SIZE buffers.
> 
> These optimizations interact positively with recent commit
> ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
> which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
> optimizations benefit buffers of any size.
> 
> Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
> between two QEMU VMs on a single physical machine. Each VM has two VCPUs
> with all offloads & vhost enabled. All VMs and vhost threads run in a
> single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
> in the system will not be scheduled on the benchmark CPUs. Trunk includes
> SKB rx frag coalescing.
> 
> net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
> net-next (MTU-size bufs):  13170.01Gb/s
> net-next + auto-tune: 14555.94Gb/s
> 
> Signed-off-by: Michael Dalton <mwdalton@google.com>

OK so a high level benchmark shows it's worth it,
but how well does the logic work?
I think we should make the buffer size accessible in sysfs
or debugfs, and look at it, otherwise we don't really know.

> ---
>  drivers/net/virtio_net.c | 63 +++++++++++++++++++++++++++++++++++-------------
>  1 file changed, 46 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index d38d130..904af37 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -26,6 +26,7 @@
>  #include <linux/if_vlan.h>
>  #include <linux/slab.h>
>  #include <linux/cpu.h>
> +#include <linux/average.h>
>  
>  static int napi_weight = NAPI_POLL_WEIGHT;
>  module_param(napi_weight, int, 0444);
> @@ -36,11 +37,15 @@ module_param(gso, bool, 0444);
>  
>  /* FIXME: MTU in config. */
>  #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
> -#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \
> -                                sizeof(struct virtio_net_hdr_mrg_rxbuf), \
> -                                L1_CACHE_BYTES))
>  #define GOOD_COPY_LEN	128
>  
> +/* Weight used for the RX packet size EWMA. The average packet size is used to
> + * determine the packet buffer size when refilling RX rings. As the entire RX
> + * ring may be refilled at once, the weight is chosen so that the EWMA will be
> + * insensitive to short-term, transient changes in packet size.
> + */
> +#define RECEIVE_AVG_WEIGHT 64
> +
>  #define VIRTNET_DRIVER_VERSION "1.0.0"
>  
>  struct virtnet_stats {
> @@ -78,6 +83,9 @@ struct receive_queue {
>  	/* Chain pages by the private ptr. */
>  	struct page *pages;
>  
> +	/* Average packet length for mergeable receive buffers. */
> +	struct ewma mrg_avg_pkt_len;
> +
>  	/* Page frag for GFP_ATOMIC packet buffer allocation. */
>  	struct page_frag atomic_frag;
>  
> @@ -339,13 +347,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  	int num_buf = hdr->mhdr.num_buffers;
>  	struct page *page = virt_to_head_page(buf);
>  	int offset = buf - page_address(page);
> -	int truesize = max_t(int, len, MERGE_BUFFER_LEN);
> -	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
> +	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, len);
>  	struct sk_buff *curr_skb = head_skb;
>  
>  	if (unlikely(!curr_skb))
>  		goto err_skb;
> -


Don't like this chunk :)

>  	while (--num_buf) {
>  		int num_skb_frags;
>  
> @@ -374,23 +380,40 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  			head_skb->truesize += nskb->truesize;
>  			num_skb_frags = 0;
>  		}
> -		truesize = max_t(int, len, MERGE_BUFFER_LEN);
>  		if (curr_skb != head_skb) {
>  			head_skb->data_len += len;
>  			head_skb->len += len;
> -			head_skb->truesize += truesize;
> +			head_skb->truesize += len;
>  		}
>  		offset = buf - page_address(page);
>  		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
>  			put_page(page);
>  			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
> -					     len, truesize);
> +					     len, len);
>  		} else {
>  			skb_add_rx_frag(curr_skb, num_skb_frags, page,
> -					offset, len, truesize);
> +					offset, len, len);
>  		}
>  	}
>  
> +	/* All frags before the last frag are fully used -- for those frags,
> +	 * truesize = len. Use the size of the most recent buffer allocation
> +	 * from the last frag's page to estimate the truesize of the last frag.

I don't get the real motivation for this.

We have skbs A,B,C sharing a page, with chunk D being unused.
This randomly charges chunk D to an skb that ended up last
in the page.
Correct?
Why does this make sense?

> +	 * EWMA with a weight of 64 makes the size adjustments quite small in
> +	 * the frags allocated on one page (even a order-3 one), and truesize
> +	 * doesn't need to be 100% accurate.

If the explanation for the above is that we don't care where D is
charged, let's not charge it to any skbs.

> +	 */
> +	if (skb_is_nonlinear(head_skb)) {
> +		u32 est_buffer_len = page_private(page);
> +		if (est_buffer_len > len) {
> +			u32 truesize_delta = est_buffer_len - len;
> +
> +			curr_skb->truesize += truesize_delta;
> +			if (curr_skb != head_skb)
> +				head_skb->truesize += truesize_delta;
> +		}
> +	}
> +	ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);

Why head_skb only? Why not full buffer size that comes from host?
This is simply len.


>  	return head_skb;
>  
>  err_skb:
> @@ -578,24 +601,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
>  static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
>  {
>  	struct virtnet_info *vi = rq->vq->vdev->priv;
> +	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
>  	struct page_frag *alloc_frag;
>  	char *buf;
> -	int err, len, hole;
> +	int err, hole;
> +	u32 buflen;
>  
> +	buflen = hdr_len + clamp_t(u32, ewma_read(&rq->mrg_avg_pkt_len),
> +				   GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
> +	buflen = ALIGN(buflen, L1_CACHE_BYTES);
>  	alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
> -	if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
> +	if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, gfp)))
>  		return -ENOMEM;
>  	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
>  	get_page(alloc_frag->page);
> -	len = MERGE_BUFFER_LEN;
> -	alloc_frag->offset += len;
> +	alloc_frag->offset += buflen;
> +	set_page_private(alloc_frag->page, buflen);
>  	hole = alloc_frag->size - alloc_frag->offset;
> -	if (hole < MERGE_BUFFER_LEN) {
> -		len += hole;
> +	if (hole < buflen) {
> +		buflen += hole;
>  		alloc_frag->offset += hole;
>  	}
>  
> -	sg_init_one(rq->sg, buf, len);
> +	sg_init_one(rq->sg, buf, buflen);
>  	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
>  	if (err < 0)
>  		put_page(virt_to_head_page(buf));
> @@ -1516,6 +1544,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
>  			       napi_weight);
>  
>  		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
> +		ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
>  		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
>  	}
>  
> -- 
> 1.8.5.1

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Michael S. Tsirkin]

On Mon, Dec 16, 2013 at 04:16:27PM -0800, Michael Dalton wrote:
> skb_page_frag_refill currently permits only order-0 page allocs
> unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
> higher-order page allocations whether or not GFP_WAIT is used. If
> memory cannot be allocated, the allocator will fall back to
> successively smaller page allocs (down to order-0 page allocs).
> 
> This change brings skb_page_frag_refill in line with the existing
> page allocation strategy employed by netdev_alloc_frag, which attempts
> higher-order page allocations whether or not GFP_WAIT is set, falling
> back to successively lower-order page allocations on failure. Part
> of migration of virtio-net to per-receive queue page frag allocators.
> 
> Signed-off-by: Michael Dalton <mwdalton@google.com>

I don't get how this is related to patch 3/3 exactly.
That one seems to clamp the allocations from ewma to at most
PAGE_SIZE, so how to we get higher-order allocations here?
Could you clarify please?

> ---
>  net/core/sock.c | 4 +---
>  1 file changed, 1 insertion(+), 3 deletions(-)
> 
> diff --git a/net/core/sock.c b/net/core/sock.c
> index ab20ed9..7383d23 100644
> --- a/net/core/sock.c
> +++ b/net/core/sock.c
> @@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
>  		put_page(pfrag->page);
>  	}
>  
> -	/* We restrict high order allocations to users that can afford to wait */
> -	order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
> -
> +	order = SKB_FRAG_PAGE_ORDER;
>  	do {
>  		gfp_t gfp = prio;
>  
> -- 
> 1.8.5.1

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Michael S. Tsirkin]

On Mon, Dec 16, 2013 at 04:16:28PM -0800, Michael Dalton wrote:
> The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
> mergeable rx buffer allocations. This commit migrates virtio-net to use
> per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
> mergeable rx buffer memory allocation, which now will use skb_refill_frag()
> for both atomic and GFP-WAIT buffer allocations.

OK so just to clarify, this is intended as a cleanup
and preparation for 3/3, not as an optimization?
Some notes below.

> 
> To address fragmentation concerns, if after buffer allocation there
> is too little space left in the page frag to allocate a subsequent
> buffer, the remaining space is added to the current allocated buffer
> so that the remaining space can be used to store packet data.
> 
> Signed-off-by: Michael Dalton <mwdalton@google.com>
> ---
>  drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
>  1 file changed, 38 insertions(+), 31 deletions(-)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index c51a988..d38d130 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -78,6 +78,9 @@ struct receive_queue {
>  	/* Chain pages by the private ptr. */
>  	struct page *pages;
>  
> +	/* Page frag for GFP_ATOMIC packet buffer allocation. */
> +	struct page_frag atomic_frag;
> +
>  	/* RX: fragments + linear part + virtio header */
>  	struct scatterlist sg[MAX_SKB_FRAGS + 2];
>  
> @@ -127,9 +130,9 @@ struct virtnet_info {
>  	struct mutex config_lock;
>  
>  	/* Page_frag for GFP_KERNEL packet buffer allocation when we run
> -	 * low on memory.
> +	 * low on memory. May sleep.
>  	 */
> -	struct page_frag alloc_frag;
> +	struct page_frag sleep_frag;
>  
>  	/* Does the affinity hint is set for virtqueues? */
>  	bool affinity_hint_set;
> @@ -336,8 +339,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  	int num_buf = hdr->mhdr.num_buffers;
>  	struct page *page = virt_to_head_page(buf);
>  	int offset = buf - page_address(page);
> -	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len,
> -					       MERGE_BUFFER_LEN);
> +	int truesize = max_t(int, len, MERGE_BUFFER_LEN);
> +	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
>  	struct sk_buff *curr_skb = head_skb;
>  
>  	if (unlikely(!curr_skb))
> @@ -353,11 +356,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  			dev->stats.rx_length_errors++;
>  			goto err_buf;
>  		}
> -		if (unlikely(len > MERGE_BUFFER_LEN)) {
> -			pr_debug("%s: rx error: merge buffer too long\n",
> -				 dev->name);
> -			len = MERGE_BUFFER_LEN;
> -		}
>  
>  		page = virt_to_head_page(buf);
>  		--rq->num;
> @@ -376,19 +374,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  			head_skb->truesize += nskb->truesize;
>  			num_skb_frags = 0;
>  		}
> +		truesize = max_t(int, len, MERGE_BUFFER_LEN);
>  		if (curr_skb != head_skb) {
>  			head_skb->data_len += len;
>  			head_skb->len += len;
> -			head_skb->truesize += MERGE_BUFFER_LEN;
> +			head_skb->truesize += truesize;
>  		}
>  		offset = buf - page_address(page);
>  		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
>  			put_page(page);
>  			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
> -					     len, MERGE_BUFFER_LEN);
> +					     len, truesize);
>  		} else {
>  			skb_add_rx_frag(curr_skb, num_skb_frags, page,
> -					offset, len, MERGE_BUFFER_LEN);
> +					offset, len, truesize);
>  		}
>  	}
>  
> @@ -579,24 +578,24 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
>  static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
>  {
>  	struct virtnet_info *vi = rq->vq->vdev->priv;
> -	char *buf = NULL;
> -	int err;
> +	struct page_frag *alloc_frag;
> +	char *buf;
> +	int err, len, hole;
>  
> -	if (gfp & __GFP_WAIT) {
> -		if (skb_page_frag_refill(MERGE_BUFFER_LEN, &vi->alloc_frag,
> -					 gfp)) {
> -			buf = (char *)page_address(vi->alloc_frag.page) +
> -			      vi->alloc_frag.offset;
> -			get_page(vi->alloc_frag.page);
> -			vi->alloc_frag.offset += MERGE_BUFFER_LEN;
> -		}
> -	} else {
> -		buf = netdev_alloc_frag(MERGE_BUFFER_LEN);
> -	}
> -	if (!buf)
> +	alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
> +	if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
>  		return -ENOMEM;

I note that netdev_alloc_frag sets __GFP_COLD which kind of
makes sense.

I also note that netdev_alloc_frag uses some tricks to
reduce the amount of cache lines dirtied - need to look
at whether they actually apply here.

> +	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
> +	get_page(alloc_frag->page);
> +	len = MERGE_BUFFER_LEN;
> +	alloc_frag->offset += len;
> +	hole = alloc_frag->size - alloc_frag->offset;
> +	if (hole < MERGE_BUFFER_LEN) {
> +		len += hole;
> +		alloc_frag->offset += hole;
> +	}
>  
> -	sg_init_one(rq->sg, buf, MERGE_BUFFER_LEN);
> +	sg_init_one(rq->sg, buf, len);
>  	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
>  	if (err < 0)
>  		put_page(virt_to_head_page(buf));
> @@ -1377,6 +1376,16 @@ static void free_receive_bufs(struct virtnet_info *vi)
>  	}
>  }
>  
> +static void free_receive_page_frags(struct virtnet_info *vi)
> +{
> +	int i;
> +	for (i = 0; i < vi->max_queue_pairs; i++)
> +		if (vi->rq[i].atomic_frag.page)
> +			put_page(vi->rq[i].atomic_frag.page);
> +	if (vi->sleep_frag.page)
> +		put_page(vi->sleep_frag.page);
> +}
> +
>  static void free_unused_bufs(struct virtnet_info *vi)
>  {
>  	void *buf;
> @@ -1706,8 +1715,7 @@ free_recv_bufs:
>  free_vqs:
>  	cancel_delayed_work_sync(&vi->refill);
>  	virtnet_del_vqs(vi);
> -	if (vi->alloc_frag.page)
> -		put_page(vi->alloc_frag.page);
> +	free_receive_page_frags(vi);
>  free_stats:
>  	free_percpu(vi->stats);
>  free:
> @@ -1741,8 +1749,7 @@ static void virtnet_remove(struct virtio_device *vdev)
>  	unregister_netdev(vi->dev);
>  
>  	remove_vq_common(vi);
> -	if (vi->alloc_frag.page)
> -		put_page(vi->alloc_frag.page);
> +	free_receive_page_frags(vi);
>  
>  	flush_work(&vi->config_work);
>  
> -- 
> 1.8.5.1

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael S. Tsirkin]

On Mon, Dec 16, 2013 at 04:16:29PM -0800, Michael Dalton wrote:
> Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
> allocators") changed the mergeable receive buffer size from PAGE_SIZE to
> MTU-size, introducing a single-stream regression for benchmarks with large
> average packet size. There is no single optimal buffer size for all
> workloads.  For workloads with packet size <= MTU bytes, MTU + virtio-net
> header-sized buffers are preferred as larger buffers reduce the TCP window
> due to SKB truesize. However, single-stream workloads with large average
> packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
> are used.
> 
> This commit auto-tunes the mergeable receiver buffer packet size by
> choosing the packet buffer size based on an EWMA of the recent packet
> sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
> virtio-net header len to PAGE_SIZE. This improves throughput for
> large packet workloads, as any workload with average packet size >=
> PAGE_SIZE will use PAGE_SIZE buffers.
> 
> These optimizations interact positively with recent commit
> ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
> which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
> optimizations benefit buffers of any size.
> 
> Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
> between two QEMU VMs on a single physical machine. Each VM has two VCPUs
> with all offloads & vhost enabled. All VMs and vhost threads run in a
> single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
> in the system will not be scheduled on the benchmark CPUs. Trunk includes
> SKB rx frag coalescing.
> 
> net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
> net-next (MTU-size bufs):  13170.01Gb/s
> net-next + auto-tune: 14555.94Gb/s

Also I guess this 1% difference is in the noise, right?
Could you share data about host CPU utilization during
these runs please?

> Signed-off-by: Michael Dalton <mwdalton@google.com>
> ---
>  drivers/net/virtio_net.c | 63 +++++++++++++++++++++++++++++++++++-------------
>  1 file changed, 46 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index d38d130..904af37 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -26,6 +26,7 @@
>  #include <linux/if_vlan.h>
>  #include <linux/slab.h>
>  #include <linux/cpu.h>
> +#include <linux/average.h>
>  
>  static int napi_weight = NAPI_POLL_WEIGHT;
>  module_param(napi_weight, int, 0444);
> @@ -36,11 +37,15 @@ module_param(gso, bool, 0444);
>  
>  /* FIXME: MTU in config. */
>  #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
> -#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \
> -                                sizeof(struct virtio_net_hdr_mrg_rxbuf), \
> -                                L1_CACHE_BYTES))
>  #define GOOD_COPY_LEN	128
>  
> +/* Weight used for the RX packet size EWMA. The average packet size is used to
> + * determine the packet buffer size when refilling RX rings. As the entire RX
> + * ring may be refilled at once, the weight is chosen so that the EWMA will be
> + * insensitive to short-term, transient changes in packet size.
> + */
> +#define RECEIVE_AVG_WEIGHT 64
> +
>  #define VIRTNET_DRIVER_VERSION "1.0.0"
>  
>  struct virtnet_stats {
> @@ -78,6 +83,9 @@ struct receive_queue {
>  	/* Chain pages by the private ptr. */
>  	struct page *pages;
>  
> +	/* Average packet length for mergeable receive buffers. */
> +	struct ewma mrg_avg_pkt_len;
> +
>  	/* Page frag for GFP_ATOMIC packet buffer allocation. */
>  	struct page_frag atomic_frag;
>  
> @@ -339,13 +347,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  	int num_buf = hdr->mhdr.num_buffers;
>  	struct page *page = virt_to_head_page(buf);
>  	int offset = buf - page_address(page);
> -	int truesize = max_t(int, len, MERGE_BUFFER_LEN);
> -	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
> +	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, len);
>  	struct sk_buff *curr_skb = head_skb;
>  
>  	if (unlikely(!curr_skb))
>  		goto err_skb;
> -
>  	while (--num_buf) {
>  		int num_skb_frags;
>  
> @@ -374,23 +380,40 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
>  			head_skb->truesize += nskb->truesize;
>  			num_skb_frags = 0;
>  		}
> -		truesize = max_t(int, len, MERGE_BUFFER_LEN);
>  		if (curr_skb != head_skb) {
>  			head_skb->data_len += len;
>  			head_skb->len += len;
> -			head_skb->truesize += truesize;
> +			head_skb->truesize += len;
>  		}
>  		offset = buf - page_address(page);
>  		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
>  			put_page(page);
>  			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
> -					     len, truesize);
> +					     len, len);
>  		} else {
>  			skb_add_rx_frag(curr_skb, num_skb_frags, page,
> -					offset, len, truesize);
> +					offset, len, len);
>  		}
>  	}
>  
> +	/* All frags before the last frag are fully used -- for those frags,
> +	 * truesize = len. Use the size of the most recent buffer allocation
> +	 * from the last frag's page to estimate the truesize of the last frag.
> +	 * EWMA with a weight of 64 makes the size adjustments quite small in
> +	 * the frags allocated on one page (even a order-3 one), and truesize
> +	 * doesn't need to be 100% accurate.
> +	 */
> +	if (skb_is_nonlinear(head_skb)) {
> +		u32 est_buffer_len = page_private(page);
> +		if (est_buffer_len > len) {
> +			u32 truesize_delta = est_buffer_len - len;
> +
> +			curr_skb->truesize += truesize_delta;
> +			if (curr_skb != head_skb)
> +				head_skb->truesize += truesize_delta;
> +		}
> +	}
> +	ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
>  	return head_skb;
>  
>  err_skb:
> @@ -578,24 +601,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
>  static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
>  {
>  	struct virtnet_info *vi = rq->vq->vdev->priv;
> +	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
>  	struct page_frag *alloc_frag;
>  	char *buf;
> -	int err, len, hole;
> +	int err, hole;
> +	u32 buflen;
>  
> +	buflen = hdr_len + clamp_t(u32, ewma_read(&rq->mrg_avg_pkt_len),
> +				   GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
> +	buflen = ALIGN(buflen, L1_CACHE_BYTES);
>  	alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
> -	if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
> +	if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, gfp)))
>  		return -ENOMEM;
>  	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
>  	get_page(alloc_frag->page);
> -	len = MERGE_BUFFER_LEN;
> -	alloc_frag->offset += len;
> +	alloc_frag->offset += buflen;
> +	set_page_private(alloc_frag->page, buflen);
>  	hole = alloc_frag->size - alloc_frag->offset;
> -	if (hole < MERGE_BUFFER_LEN) {
> -		len += hole;
> +	if (hole < buflen) {
> +		buflen += hole;
>  		alloc_frag->offset += hole;
>  	}
>  
> -	sg_init_one(rq->sg, buf, len);
> +	sg_init_one(rq->sg, buf, buflen);
>  	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
>  	if (err < 0)
>  		put_page(virt_to_head_page(buf));
> @@ -1516,6 +1544,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
>  			       napi_weight);
>  
>  		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
> +		ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
>  		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
>  	}
>  
> -- 
> 1.8.5.1

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Michael S. Tsirkin]

On Thu, Dec 19, 2013 at 02:58:15PM -0500, David Miller wrote:
> 
> Can I get some reviews of this series from virtio folks?
> 
> Thanks.

I think it's a good idea, certainly the best we can do
short term.
Would like a couple of comments addressed before it's applied.

Longer term one approach to consider is to make device
look aheads in the VQ until it finds a large
enough buffer.

-- 
MST

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Eric Dumazet]

On Mon, 2013-12-23 at 15:52 +0800, Jason Wang wrote:
> On 12/17/2013 08:16 AM, Michael Dalton wrote:
> > skb_page_frag_refill currently permits only order-0 page allocs
> > unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
> > higher-order page allocations whether or not GFP_WAIT is used. If
> > memory cannot be allocated, the allocator will fall back to
> > successively smaller page allocs (down to order-0 page allocs).
> >
> > This change brings skb_page_frag_refill in line with the existing
> > page allocation strategy employed by netdev_alloc_frag, which attempts
> > higher-order page allocations whether or not GFP_WAIT is set, falling
> > back to successively lower-order page allocations on failure. Part
> > of migration of virtio-net to per-receive queue page frag allocators.
> >
> > Signed-off-by: Michael Dalton <mwdalton@google.com>
> > ---
> >  net/core/sock.c | 4 +---
> >  1 file changed, 1 insertion(+), 3 deletions(-)
> >
> > diff --git a/net/core/sock.c b/net/core/sock.c
> > index ab20ed9..7383d23 100644
> > --- a/net/core/sock.c
> > +++ b/net/core/sock.c
> > @@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
> >  		put_page(pfrag->page);
> >  	}
> >  
> > -	/* We restrict high order allocations to users that can afford to wait */
> > -	order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
> > -
> > +	order = SKB_FRAG_PAGE_ORDER;
> >  	do {
> >  		gfp_t gfp = prio;
> >  
> 
> The original code seems try to avoid the high order allocation for
> atomic allocation. This patch changes this, and looks like it will
> introduce some extra cost when the memory is highly fragmented.

No noticeable extra cost that I could measure.

We use the same strategy in RX path nowadays.

Acked-by: Eric Dumazet <edumazet@google.com>

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Eric Dumazet]

On Mon, 2013-12-23 at 15:52 +0800, Jason Wang wrote:
> On 12/17/2013 08:16 AM, Michael Dalton wrote:
> > skb_page_frag_refill currently permits only order-0 page allocs
> > unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
> > higher-order page allocations whether or not GFP_WAIT is used. If
> > memory cannot be allocated, the allocator will fall back to
> > successively smaller page allocs (down to order-0 page allocs).
> >
> > This change brings skb_page_frag_refill in line with the existing
> > page allocation strategy employed by netdev_alloc_frag, which attempts
> > higher-order page allocations whether or not GFP_WAIT is set, falling
> > back to successively lower-order page allocations on failure. Part
> > of migration of virtio-net to per-receive queue page frag allocators.
> >
> > Signed-off-by: Michael Dalton <mwdalton@google.com>
> > ---
> >  net/core/sock.c | 4 +---
> >  1 file changed, 1 insertion(+), 3 deletions(-)
> >
> > diff --git a/net/core/sock.c b/net/core/sock.c
> > index ab20ed9..7383d23 100644
> > --- a/net/core/sock.c
> > +++ b/net/core/sock.c
> > @@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
> >  		put_page(pfrag->page);
> >  	}
> >  
> > -	/* We restrict high order allocations to users that can afford to wait */
> > -	order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
> > -
> > +	order = SKB_FRAG_PAGE_ORDER;
> >  	do {
> >  		gfp_t gfp = prio;
> >  
> 
> The original code seems try to avoid the high order allocation for
> atomic allocation. This patch changes this, and looks like it will
> introduce some extra cost when the memory is highly fragmented.

No noticeable extra cost that I could measure.

We use the same strategy in RX path nowadays.

Acked-by: Eric Dumazet <edumazet@google.com>

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Eric Dumazet]

On Mon, 2013-12-23 at 16:12 +0800, Jason Wang wrote:
> On 12/17/2013 08:16 AM, Michael Dalton wrote:
> > The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
> > mergeable rx buffer allocations. This commit migrates virtio-net to use
> > per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
> > mergeable rx buffer memory allocation, which now will use skb_refill_frag()
> > for both atomic and GFP-WAIT buffer allocations.
> >
> > To address fragmentation concerns, if after buffer allocation there
> > is too little space left in the page frag to allocate a subsequent
> > buffer, the remaining space is added to the current allocated buffer
> > so that the remaining space can be used to store packet data.
> >
> > Signed-off-by: Michael Dalton <mwdalton@google.com>
> > ---
> >  drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
> >  1 file changed, 38 insertions(+), 31 deletions(-)
> >
> > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> > index c51a988..d38d130 100644
> > --- a/drivers/net/virtio_net.c
> > +++ b/drivers/net/virtio_net.c
> > @@ -78,6 +78,9 @@ struct receive_queue {
> >  	/* Chain pages by the private ptr. */
> >  	struct page *pages;
> >  
> > +	/* Page frag for GFP_ATOMIC packet buffer allocation. */
> > +	struct page_frag atomic_frag;
> > +
> >  	/* RX: fragments + linear part + virtio header */
> >  	struct scatterlist sg[MAX_SKB_FRAGS + 2];
> >  
> > @@ -127,9 +130,9 @@ struct virtnet_info {
> >  	struct mutex config_lock;
> >  
> >  	/* Page_frag for GFP_KERNEL packet buffer allocation when we run
> > -	 * low on memory.
> > +	 * low on memory. May sleep.
> >  	 */
> > -	struct page_frag alloc_frag;
> > +	struct page_frag sleep_frag;
> 
> Any reason to use two different page_frag consider only
> skb_page_frag_refill() is used?

One is used under process context, where preemption and GFP_KERNEL are
allowed.

One is used from softirq context and GFP_ATOMIC. You cant share a common
page_frag.

Acked-by: Eric Dumazet <edumazet@google.com>

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Eric Dumazet]

On Mon, 2013-12-23 at 14:53 +0200, Michael S. Tsirkin wrote:
> On Mon, Dec 16, 2013 at 04:16:27PM -0800, Michael Dalton wrote:
> > skb_page_frag_refill currently permits only order-0 page allocs
> > unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
> > higher-order page allocations whether or not GFP_WAIT is used. If
> > memory cannot be allocated, the allocator will fall back to
> > successively smaller page allocs (down to order-0 page allocs).
> > 
> > This change brings skb_page_frag_refill in line with the existing
> > page allocation strategy employed by netdev_alloc_frag, which attempts
> > higher-order page allocations whether or not GFP_WAIT is set, falling
> > back to successively lower-order page allocations on failure. Part
> > of migration of virtio-net to per-receive queue page frag allocators.
> > 
> > Signed-off-by: Michael Dalton <mwdalton@google.com>
> 
> I don't get how this is related to patch 3/3 exactly.
> That one seems to clamp the allocations from ewma to at most
> PAGE_SIZE, so how to we get higher-order allocations here?
> Could you clarify please?

If your ewma stabilizes at 2050 bytes per frag, using order-0 page will
waste ~50% of memory.

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Michael S. Tsirkin]

On Mon, Dec 23, 2013 at 09:30:38AM -0800, Eric Dumazet wrote:
> On Mon, 2013-12-23 at 14:53 +0200, Michael S. Tsirkin wrote:
> > On Mon, Dec 16, 2013 at 04:16:27PM -0800, Michael Dalton wrote:
> > > skb_page_frag_refill currently permits only order-0 page allocs
> > > unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
> > > higher-order page allocations whether or not GFP_WAIT is used. If
> > > memory cannot be allocated, the allocator will fall back to
> > > successively smaller page allocs (down to order-0 page allocs).
> > > 
> > > This change brings skb_page_frag_refill in line with the existing
> > > page allocation strategy employed by netdev_alloc_frag, which attempts
> > > higher-order page allocations whether or not GFP_WAIT is set, falling
> > > back to successively lower-order page allocations on failure. Part
> > > of migration of virtio-net to per-receive queue page frag allocators.
> > > 
> > > Signed-off-by: Michael Dalton <mwdalton@google.com>
> > 
> > I don't get how this is related to patch 3/3 exactly.
> > That one seems to clamp the allocations from ewma to at most
> > PAGE_SIZE, so how to we get higher-order allocations here?
> > Could you clarify please?
> 
> If your ewma stabilizes at 2050 bytes per frag, using order-0 page will
> waste ~50% of memory.
> 

Aha, got it. FWIW
Acked-by: Michael S. Tsirkin <mst@redhat.com>



-- 
MST

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Michael S. Tsirkin]

On Mon, Dec 23, 2013 at 09:27:07AM -0800, Eric Dumazet wrote:
> On Mon, 2013-12-23 at 16:12 +0800, Jason Wang wrote:
> > On 12/17/2013 08:16 AM, Michael Dalton wrote:
> > > The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
> > > mergeable rx buffer allocations. This commit migrates virtio-net to use
> > > per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
> > > mergeable rx buffer memory allocation, which now will use skb_refill_frag()
> > > for both atomic and GFP-WAIT buffer allocations.
> > >
> > > To address fragmentation concerns, if after buffer allocation there
> > > is too little space left in the page frag to allocate a subsequent
> > > buffer, the remaining space is added to the current allocated buffer
> > > so that the remaining space can be used to store packet data.
> > >
> > > Signed-off-by: Michael Dalton <mwdalton@google.com>
> > > ---
> > >  drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
> > >  1 file changed, 38 insertions(+), 31 deletions(-)
> > >
> > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> > > index c51a988..d38d130 100644
> > > --- a/drivers/net/virtio_net.c
> > > +++ b/drivers/net/virtio_net.c
> > > @@ -78,6 +78,9 @@ struct receive_queue {
> > >  	/* Chain pages by the private ptr. */
> > >  	struct page *pages;
> > >  
> > > +	/* Page frag for GFP_ATOMIC packet buffer allocation. */
> > > +	struct page_frag atomic_frag;
> > > +
> > >  	/* RX: fragments + linear part + virtio header */
> > >  	struct scatterlist sg[MAX_SKB_FRAGS + 2];
> > >  
> > > @@ -127,9 +130,9 @@ struct virtnet_info {
> > >  	struct mutex config_lock;
> > >  
> > >  	/* Page_frag for GFP_KERNEL packet buffer allocation when we run
> > > -	 * low on memory.
> > > +	 * low on memory. May sleep.
> > >  	 */
> > > -	struct page_frag alloc_frag;
> > > +	struct page_frag sleep_frag;
> > 
> > Any reason to use two different page_frag consider only
> > skb_page_frag_refill() is used?
> 
> One is used under process context, where preemption and GFP_KERNEL are
> allowed.

Yes but it is always used with napi disabled.

> One is used from softirq context and GFP_ATOMIC.

This one is used only under napi.

> You cant share a common
> page_frag.

So there isn't a conflict with respect to locking.

Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?

> Acked-by: Eric Dumazet <edumazet@google.com>
> 
> 

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[David Miller]

There is still feedback and/or minor adjustments being asked for wrt.
this series.   These changes have been sitting for more than a week
which is a bit rediculous.

Please resubmit these changes once everything is resolved to
everyone's satisfaction, thanks.

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[David Miller]

There is still feedback and/or minor adjustments being asked for wrt.
this series.   These changes have been sitting for more than a week
which is a bit rediculous.

Please resubmit these changes once everything is resolved to
everyone's satisfaction, thanks.

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Jason Wang]

On 12/17/2013 08:16 AM, Michael Dalton wrote:
> Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
> allocators") changed the mergeable receive buffer size from PAGE_SIZE to
> MTU-size, introducing a single-stream regression for benchmarks with large
> average packet size. There is no single optimal buffer size for all
> workloads.  For workloads with packet size <= MTU bytes, MTU + virtio-net
> header-sized buffers are preferred as larger buffers reduce the TCP window
> due to SKB truesize. However, single-stream workloads with large average
> packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
> are used.
>
> This commit auto-tunes the mergeable receiver buffer packet size by
> choosing the packet buffer size based on an EWMA of the recent packet
> sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
> virtio-net header len to PAGE_SIZE. This improves throughput for
> large packet workloads, as any workload with average packet size >=
> PAGE_SIZE will use PAGE_SIZE buffers.
>
> These optimizations interact positively with recent commit
> ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
> which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
> optimizations benefit buffers of any size.
>
> Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
> between two QEMU VMs on a single physical machine. Each VM has two VCPUs
> with all offloads & vhost enabled. All VMs and vhost threads run in a
> single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
> in the system will not be scheduled on the benchmark CPUs. Trunk includes
> SKB rx frag coalescing.
>
> net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
> net-next (MTU-size bufs):  13170.01Gb/s
> net-next + auto-tune: 14555.94Gb/s
>
> Signed-off-by: Michael Dalton <mwdalton@google.com>

The patch looks good to me and test this patch with mlx4, it help to
increase the rx performance from about 22Gb/s to about 26 Gb/s.

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael Dalton]

On Mon, Dec 23, 2013 at 4:51 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> OK so a high level benchmark shows it's worth it,
> but how well does the logic work?
> I think we should make the buffer size accessible in sysfs
> or debugfs, and look at it, otherwise we don't really know.
>
Exporting the size sounds good to me, it is definitely an
important metric and would give more visibility to the admin.

Do you have a preference for implementation strategy? I was
thinking just add a DEVICE_ATTR to create a read-only sysfs file,
'mergeable_rx_buffer_size', and return a space-separated list of the
current buffer size (computed from the average packet size) for each
receive queue. -EINVAL or a similar error could be returned if the
netdev was not configured for mergeable rx buffers.

> I don't get the real motivation for this.
>
> We have skbs A,B,C sharing a page, with chunk D being unused.
> This randomly charges chunk D to an skb that ended up last
> in the page.
> Correct?
> Why does this make sense?

The intent of this code is to adjust the SKB true size for
the packet. We should completely use each packet buffer except
for the last buffer. For all buffers except the last buffer, it
should be the case that 'len' (bytes received) = buffer size. For
the last buffer, this code adjusts the truesize by comparing the
approximated buffer size with the bytes received into the buffer,
and adding the difference to the SKB truesize if the buffer size
is greater than the number of bytes received.

We approximate the buffer size by using the last packet buffer size
from that same page, which as you have correctly noted may be a buffer
that belongs to a different packet on the same virtio-net device. This
buffer size should be very close to the actual buffer size because our
EWMA estimator uses a high weight (so the packet buffer size changes very
slowly) and there are only a handful packets on a page (even order-3).

> Why head_skb only? Why not full buffer size that comes from host?
> This is simply len.

Sorry, I believe this code fragment should be clearer. Basically, we
have a corner case in that for packets with size <= GOOD_COPY_LEN, there
are no frags because page_to_skb() already unref'd the page and the entire
packet contents are copied to skb->data. In this case, the SKB truesize
is already accurate and should not be updated (and it would be unsafe to
access page->private as page is already unref'd).

I'll look at the above code again and cleanup (please let me know if you
have a preference) and/or add a comment to clarify.

Best,

Mike

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael S. Tsirkin]

On Thu, Dec 26, 2013 at 12:06:23PM -0800, Michael Dalton wrote:
> On Mon, Dec 23, 2013 at 4:51 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > OK so a high level benchmark shows it's worth it,
> > but how well does the logic work?
> > I think we should make the buffer size accessible in sysfs
> > or debugfs, and look at it, otherwise we don't really know.
> >
> Exporting the size sounds good to me, it is definitely an
> important metric and would give more visibility to the admin.
> 
> Do you have a preference for implementation strategy? I was
> thinking just add a DEVICE_ATTR to create a read-only sysfs file,
> 'mergeable_rx_buffer_size', and return a space-separated list of the
> current buffer size (computed from the average packet size) for each
> receive queue. -EINVAL or a similar error could be returned if the
> netdev was not configured for mergeable rx buffers.

Not too important really. Maybe an attribute per queue? Might be
easier to parse and does not need tricky formatting.

> > I don't get the real motivation for this.
> >
> > We have skbs A,B,C sharing a page, with chunk D being unused.
> > This randomly charges chunk D to an skb that ended up last
> > in the page.
> > Correct?
> > Why does this make sense?
> 
> The intent of this code is to adjust the SKB true size for
> the packet. We should completely use each packet buffer except
> for the last buffer. For all buffers except the last buffer, it
> should be the case that 'len' (bytes received) = buffer size. For
> the last buffer, this code adjusts the truesize by comparing the
> approximated buffer size with the bytes received into the buffer,
> and adding the difference to the SKB truesize if the buffer size
> is greater than the number of bytes received.
> 
> We approximate the buffer size by using the last packet buffer size
> from that same page, which as you have correctly noted may be a buffer
> that belongs to a different packet on the same virtio-net device. This
> buffer size should be very close to the actual buffer size because our
> EWMA estimator uses a high weight (so the packet buffer size changes very
> slowly) and there are only a handful packets on a page (even order-3).

I'll need to ponder it, don't understand it exactly.

> > Why head_skb only? Why not full buffer size that comes from host?
> > This is simply len.
> 
> Sorry, I believe this code fragment should be clearer. Basically, we
> have a corner case in that for packets with size <= GOOD_COPY_LEN, there
> are no frags because page_to_skb() already unref'd the page and the entire
> packet contents are copied to skb->data. In this case, the SKB truesize
> is already accurate and should not be updated (and it would be unsafe to
> access page->private as page is already unref'd).
> 
> I'll look at the above code again and cleanup (please let me know if you
> have a preference) and/or add a comment to clarify.
> 
> Best,
> 
> Mike

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Michael Dalton]

On Mon, Dec 23, 2013 at 11:37 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> So there isn't a conflict with respect to locking.
>
> Is it problematic to use same page_frag with both GFP_ATOMIC and with
> GFP_KERNEL? If yes why?

I believe it is safe to use the same page_frag and I will send out a
followup patchset using just the per-receive page_frags. For future
consideration, Eric noted that disabling NAPI before GFP_KERNEL
allocs can potentially inhibit virtio-net network processing for some
time (e.g., during a blocking memory allocation or preemption).

Best,

Mike

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Michael S. Tsirkin]

On Thu, Dec 26, 2013 at 01:28:58PM -0800, Michael Dalton wrote:
> On Mon, Dec 23, 2013 at 11:37 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > So there isn't a conflict with respect to locking.
> >
> > Is it problematic to use same page_frag with both GFP_ATOMIC and with
> > GFP_KERNEL? If yes why?
> 
> I believe it is safe to use the same page_frag and I will send out a
> followup patchset using just the per-receive page_frags.

Seems easier to use it straight away I think.

> For future
> consideration, Eric noted that disabling NAPI before GFP_KERNEL
> allocs can potentially inhibit virtio-net network processing for some
> time (e.g., during a blocking memory allocation or preemption).
> 
> Best,
> 
> Mike

Interesting. But if we can't allocate a buffer how can we
do network processing?

If we can reproduce the problem, we can maybe move
allocation out of napi disabled section, but then
we'll need to add more locking.

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Eric Dumazet]

On Thu, 2013-12-26 at 13:28 -0800, Michael Dalton wrote:
> On Mon, Dec 23, 2013 at 11:37 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > So there isn't a conflict with respect to locking.
> >
> > Is it problematic to use same page_frag with both GFP_ATOMIC and with
> > GFP_KERNEL? If yes why?
> 
> I believe it is safe to use the same page_frag and I will send out a
> followup patchset using just the per-receive page_frags. For future
> consideration, Eric noted that disabling NAPI before GFP_KERNEL
> allocs can potentially inhibit virtio-net network processing for some
> time (e.g., during a blocking memory allocation or preemption).

Yep, using napi_disable() in the refill process looks quite inefficient
to me, it not buggy.

napi_disable() is a big hammer, while whole idea of having a process to
block on GFP_KERNEL allocations is to allow some asynchronous behavior.

I have hard time to convince myself virtio_net is safe anyway with this
work queue thing.

virtnet_open() seems racy for example :

        for (i = 0; i < vi->max_queue_pairs; i++) {
                if (i < vi->curr_queue_pairs)
                        /* Make sure we have some buffers: if oom use wq. */
                        if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
                                schedule_delayed_work(&vi->refill, 0);
                virtnet_napi_enable(&vi->rq[i]);


What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?

refill_work() will happily conflict with another cpu, two cpus could 
call try_fill_recv() at the same time, or worse napi_enable() would crash.

I do not have time to make a full check, but I guess there are
other races like this one.

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a98867a40..b8e2adb5d0c2 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -726,16 +726,18 @@ again:
 static int virtnet_open(struct net_device *dev)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
+	bool refill = false;
 	int i;
 
 	for (i = 0; i < vi->max_queue_pairs; i++) {
 		if (i < vi->curr_queue_pairs)
 			/* Make sure we have some buffers: if oom use wq. */
 			if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
-				schedule_delayed_work(&vi->refill, 0);
+				refill = true;
 		virtnet_napi_enable(&vi->rq[i]);
 	}
-
+	if (refill)
+		schedule_delayed_work(&vi->refill, 0);
 	return 0;
 }
 

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Eric Dumazet]

On Thu, 2013-12-26 at 13:28 -0800, Michael Dalton wrote:
> On Mon, Dec 23, 2013 at 11:37 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > So there isn't a conflict with respect to locking.
> >
> > Is it problematic to use same page_frag with both GFP_ATOMIC and with
> > GFP_KERNEL? If yes why?
> 
> I believe it is safe to use the same page_frag and I will send out a
> followup patchset using just the per-receive page_frags. For future
> consideration, Eric noted that disabling NAPI before GFP_KERNEL
> allocs can potentially inhibit virtio-net network processing for some
> time (e.g., during a blocking memory allocation or preemption).

Yep, using napi_disable() in the refill process looks quite inefficient
to me, it not buggy.

napi_disable() is a big hammer, while whole idea of having a process to
block on GFP_KERNEL allocations is to allow some asynchronous behavior.

I have hard time to convince myself virtio_net is safe anyway with this
work queue thing.

virtnet_open() seems racy for example :

        for (i = 0; i < vi->max_queue_pairs; i++) {
                if (i < vi->curr_queue_pairs)
                        /* Make sure we have some buffers: if oom use wq. */
                        if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
                                schedule_delayed_work(&vi->refill, 0);
                virtnet_napi_enable(&vi->rq[i]);


What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?

refill_work() will happily conflict with another cpu, two cpus could 
call try_fill_recv() at the same time, or worse napi_enable() would crash.

I do not have time to make a full check, but I guess there are
other races like this one.

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a98867a40..b8e2adb5d0c2 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -726,16 +726,18 @@ again:
 static int virtnet_open(struct net_device *dev)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
+	bool refill = false;
 	int i;
 
 	for (i = 0; i < vi->max_queue_pairs; i++) {
 		if (i < vi->curr_queue_pairs)
 			/* Make sure we have some buffers: if oom use wq. */
 			if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
-				schedule_delayed_work(&vi->refill, 0);
+				refill = true;
 		virtnet_napi_enable(&vi->rq[i]);
 	}
-
+	if (refill)
+		schedule_delayed_work(&vi->refill, 0);
 	return 0;
 }

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Eric Dumazet]

On Thu, 2013-12-26 at 23:37 +0200, Michael S. Tsirkin wrote:

> Interesting. But if we can't allocate a buffer how can we
> do network processing?

How typical NIC drivers handle this case ?

Answer : nothing special should happen, we drop incoming traffic,
and make sure the driver recovers properly. (like not NULL deref or
crazy things like that)

Why virtio_net should be different ?

> 
> If we can reproduce the problem, we can maybe move
> allocation out of napi disabled section, but then
> we'll need to add more locking.

More exactly, use appropriate locking ;)

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Eric Dumazet]

On Thu, 2013-12-26 at 23:37 +0200, Michael S. Tsirkin wrote:

> Interesting. But if we can't allocate a buffer how can we
> do network processing?

How typical NIC drivers handle this case ?

Answer : nothing special should happen, we drop incoming traffic,
and make sure the driver recovers properly. (like not NULL deref or
crazy things like that)

Why virtio_net should be different ?

> 
> If we can reproduce the problem, we can maybe move
> allocation out of napi disabled section, but then
> we'll need to add more locking.

More exactly, use appropriate locking ;)

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Jason Wang]

On 12/27/2013 04:06 AM, Michael Dalton wrote:
> On Mon, Dec 23, 2013 at 4:51 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
>> OK so a high level benchmark shows it's worth it,
>> but how well does the logic work?
>> I think we should make the buffer size accessible in sysfs
>> or debugfs, and look at it, otherwise we don't really know.
>>
> Exporting the size sounds good to me, it is definitely an
> important metric and would give more visibility to the admin.
>
> Do you have a preference for implementation strategy? I was
> thinking just add a DEVICE_ATTR to create a read-only sysfs file,
> 'mergeable_rx_buffer_size', and return a space-separated list of the
> current buffer size (computed from the average packet size) for each
> receive queue. -EINVAL or a similar error could be returned if the
> netdev was not configured for mergeable rx buffers.
>
>> I don't get the real motivation for this.
>>
>> We have skbs A,B,C sharing a page, with chunk D being unused.
>> This randomly charges chunk D to an skb that ended up last
>> in the page.
>> Correct?
>> Why does this make sense?
> The intent of this code is to adjust the SKB true size for
> the packet. We should completely use each packet buffer except
> for the last buffer. For all buffers except the last buffer, it
> should be the case that 'len' (bytes received) = buffer size. For
> the last buffer, this code adjusts the truesize by comparing the
> approximated buffer size with the bytes received into the buffer,
> and adding the difference to the SKB truesize if the buffer size
> is greater than the number of bytes received.

But look like it does not take the 'hole' into account which may cause
under estimation of trusize.
>
> We approximate the buffer size by using the last packet buffer size
> from that same page, which as you have correctly noted may be a buffer
> that belongs to a different packet on the same virtio-net device. This
> buffer size should be very close to the actual buffer size because our
> EWMA estimator uses a high weight (so the packet buffer size changes very
> slowly) and there are only a handful packets on a page (even order-3).

We can make this more accurate by using extra data structure to track
the real buf size and using it as token.
>> Why head_skb only? Why not full buffer size that comes from host?
>> This is simply len.
> Sorry, I believe this code fragment should be clearer. Basically, we
> have a corner case in that for packets with size <= GOOD_COPY_LEN, there
> are no frags because page_to_skb() already unref'd the page and the entire
> packet contents are copied to skb->data. In this case, the SKB truesize
> is already accurate and should not be updated (and it would be unsafe to
> access page->private as page is already unref'd).
>
> I'll look at the above code again and cleanup (please let me know if you
> have a preference) and/or add a comment to clarify.
>
> Best,
>
> Mike

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Jason Wang]

On 12/27/2013 05:56 AM, Eric Dumazet wrote:
> On Thu, 2013-12-26 at 13:28 -0800, Michael Dalton wrote:
>> On Mon, Dec 23, 2013 at 11:37 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
>>> So there isn't a conflict with respect to locking.
>>>
>>> Is it problematic to use same page_frag with both GFP_ATOMIC and with
>>> GFP_KERNEL? If yes why?
>> I believe it is safe to use the same page_frag and I will send out a
>> followup patchset using just the per-receive page_frags. For future
>> consideration, Eric noted that disabling NAPI before GFP_KERNEL
>> allocs can potentially inhibit virtio-net network processing for some
>> time (e.g., during a blocking memory allocation or preemption).
> Yep, using napi_disable() in the refill process looks quite inefficient
> to me, it not buggy.
>
> napi_disable() is a big hammer, while whole idea of having a process to
> block on GFP_KERNEL allocations is to allow some asynchronous behavior.
>
> I have hard time to convince myself virtio_net is safe anyway with this
> work queue thing.
>
> virtnet_open() seems racy for example :
>
>         for (i = 0; i < vi->max_queue_pairs; i++) {
>                 if (i < vi->curr_queue_pairs)
>                         /* Make sure we have some buffers: if oom use wq. */
>                         if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
>                                 schedule_delayed_work(&vi->refill, 0);
>                 virtnet_napi_enable(&vi->rq[i]);
>
>
> What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?

Then napi_disable() in refill_work() will busy wait until napi is
enabled by virtnet_napi_enable() which looks safe. Looks like the real
issue is in virtnet_restore() who calls try_fill_recv() in neither napi
context nor napi disabled context.
>
> refill_work() will happily conflict with another cpu, two cpus could 
> call try_fill_recv() at the same time, or worse napi_enable() would crash.
>
> I do not have time to make a full check, but I guess there are
> other races like this one.
>
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index c51a98867a40..b8e2adb5d0c2 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -726,16 +726,18 @@ again:
>  static int virtnet_open(struct net_device *dev)
>  {
>  	struct virtnet_info *vi = netdev_priv(dev);
> +	bool refill = false;
>  	int i;
>  
>  	for (i = 0; i < vi->max_queue_pairs; i++) {
>  		if (i < vi->curr_queue_pairs)
>  			/* Make sure we have some buffers: if oom use wq. */
>  			if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
> -				schedule_delayed_work(&vi->refill, 0);
> +				refill = true;
>  		virtnet_napi_enable(&vi->rq[i]);
>  	}
> -
> +	if (refill)
> +		schedule_delayed_work(&vi->refill, 0);
>  	return 0;
>  }
>  
>
>
>
>
>

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Eric Dumazet]

On Fri, 2013-12-27 at 12:55 +0800, Jason Wang wrote:
> On 12/27/2013 05:56 AM, Eric Dumazet wrote:
> > On Thu, 2013-12-26 at 13:28 -0800, Michael Dalton wrote:
> >> On Mon, Dec 23, 2013 at 11:37 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> >>> So there isn't a conflict with respect to locking.
> >>>
> >>> Is it problematic to use same page_frag with both GFP_ATOMIC and with
> >>> GFP_KERNEL? If yes why?
> >> I believe it is safe to use the same page_frag and I will send out a
> >> followup patchset using just the per-receive page_frags. For future
> >> consideration, Eric noted that disabling NAPI before GFP_KERNEL
> >> allocs can potentially inhibit virtio-net network processing for some
> >> time (e.g., during a blocking memory allocation or preemption).
> > Yep, using napi_disable() in the refill process looks quite inefficient
> > to me, it not buggy.
> >
> > napi_disable() is a big hammer, while whole idea of having a process to
> > block on GFP_KERNEL allocations is to allow some asynchronous behavior.
> >
> > I have hard time to convince myself virtio_net is safe anyway with this
> > work queue thing.
> >
> > virtnet_open() seems racy for example :
> >
> >         for (i = 0; i < vi->max_queue_pairs; i++) {
> >                 if (i < vi->curr_queue_pairs)
> >                         /* Make sure we have some buffers: if oom use wq. */
> >                         if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
> >                                 schedule_delayed_work(&vi->refill, 0);
> >                 virtnet_napi_enable(&vi->rq[i]);
> >
> >
> > What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?
> 
> Then napi_disable() in refill_work() will busy wait until napi is
> enabled by virtnet_napi_enable() which looks safe. Looks like the real
> issue is in virtnet_restore() who calls try_fill_recv() in neither napi
> context nor napi disabled context.

I think you don't really get the race.

The issue is the following :

CPU0                                                        CPU1

schedule_delayed_work()
                                          napi_disable(&rq->napi);
                                          try_fill_recv(rq, GFP_KERNEL);

virtnet_napi_enable(&vi->rq[i]);
...
try_fill_recv(rq, GFP_ATOMIC);

                                         napi_enable();// crash on :
                                           BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); 

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Jason Wang]

On 12/27/2013 01:46 PM, Eric Dumazet wrote:
> On Fri, 2013-12-27 at 12:55 +0800, Jason Wang wrote:
>> On 12/27/2013 05:56 AM, Eric Dumazet wrote:
>>> On Thu, 2013-12-26 at 13:28 -0800, Michael Dalton wrote:
>>>> On Mon, Dec 23, 2013 at 11:37 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
>>>>> So there isn't a conflict with respect to locking.
>>>>>
>>>>> Is it problematic to use same page_frag with both GFP_ATOMIC and with
>>>>> GFP_KERNEL? If yes why?
>>>> I believe it is safe to use the same page_frag and I will send out a
>>>> followup patchset using just the per-receive page_frags. For future
>>>> consideration, Eric noted that disabling NAPI before GFP_KERNEL
>>>> allocs can potentially inhibit virtio-net network processing for some
>>>> time (e.g., during a blocking memory allocation or preemption).
>>> Yep, using napi_disable() in the refill process looks quite inefficient
>>> to me, it not buggy.
>>>
>>> napi_disable() is a big hammer, while whole idea of having a process to
>>> block on GFP_KERNEL allocations is to allow some asynchronous behavior.
>>>
>>> I have hard time to convince myself virtio_net is safe anyway with this
>>> work queue thing.
>>>
>>> virtnet_open() seems racy for example :
>>>
>>>         for (i = 0; i < vi->max_queue_pairs; i++) {
>>>                 if (i < vi->curr_queue_pairs)
>>>                         /* Make sure we have some buffers: if oom use wq. */
>>>                         if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
>>>                                 schedule_delayed_work(&vi->refill, 0);
>>>                 virtnet_napi_enable(&vi->rq[i]);
>>>
>>>
>>> What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?
>> Then napi_disable() in refill_work() will busy wait until napi is
>> enabled by virtnet_napi_enable() which looks safe. Looks like the real
>> issue is in virtnet_restore() who calls try_fill_recv() in neither napi
>> context nor napi disabled context.
> I think you don't really get the race.
>
> The issue is the following :
>
> CPU0                                                        CPU1
>
> schedule_delayed_work()
>                                           napi_disable(&rq->napi);
>                                           try_fill_recv(rq, GFP_KERNEL);

If I didn't miss anything. In this case, for a specific rq,
napi_disable() won't return immediately since NAPI_STATE_SCHED bit was
still set. It will busy wait until NAPI_STATE_SCHED bit was clear by
virtnet_napi_enable(), and then reset the bit. So try_fill_recv() should
be called after its napi was enabled by virtnet_napi_enable() in CPU0.

So the following order were guaranteed:
- try_fill_recv(rq, GFP_ATOMIC) in CPU0
- virtnet_napi_enable(&vi->rq[i]) in CPU0
- napi_disable(&rq->napi) returned in CPU1
- try_fill_recv(rq) in CPU1
...
>
> virtnet_napi_enable(&vi->rq[i]);
> ...
> try_fill_recv(rq, GFP_ATOMIC);
>
>                                          napi_enable();// crash on :
>                                            BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); 
>
>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael Dalton]

I'm working on a followup patchset to address current feedback. I think
it will be cleaner to do a debugfs implementation for per-receive queue
packet buffer size exporting, so I'm trying that out.

On Thu, Dec 26, 2013 at 7:04 PM, Jason Wang <jasowang@redhat.com> wrote:
> We can make this more accurate by using extra data structure to track
> the real buf size and using it as token.

I agree -- we can do precise buffer total len tracking. Something like
struct mergeable_packet_buffer_ctx {
   void *buf;
   unsigned int total_len;
};

Each receive queue could have a pointer to an array of N buffer contexts,
where N is queue size (kzalloc'd in init_vqs or similar). That would
allow us to allocate all of our buffer context data at startup.

Would this be preferred to the current approach or is there another
approach you would prefer? All other things being equal, having precise
length tracking is advantageous, so I'm inclined to try this out and
see how it goes.

I think this is a big design point - for example, if we have an extra
buffer context structure, then per-receive queue frag allocators are not
required for auto-tuning and we can reduce the number of patches in
this patchset.

I'm happy to implement either way.  Thanks!

Best,

Mike

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Jason Wang]

On 12/28/2013 05:41 AM, Michael Dalton wrote:
> I'm working on a followup patchset to address current feedback. I think
> it will be cleaner to do a debugfs implementation for per-receive queue
> packet buffer size exporting, so I'm trying that out.
>
> On Thu, Dec 26, 2013 at 7:04 PM, Jason Wang <jasowang@redhat.com> wrote:
>> We can make this more accurate by using extra data structure to track
>> the real buf size and using it as token.
> I agree -- we can do precise buffer total len tracking. Something like
> struct mergeable_packet_buffer_ctx {
>    void *buf;
>    unsigned int total_len;
> };
>
> Each receive queue could have a pointer to an array of N buffer contexts,
> where N is queue size (kzalloc'd in init_vqs or similar). That would
> allow us to allocate all of our buffer context data at startup.
>
> Would this be preferred to the current approach or is there another
> approach you would prefer? All other things being equal, having precise
> length tracking is advantageous, so I'm inclined to try this out and
> see how it goes.

I think this is better since it was accurate and easier to be
implemented and understand.

Thanks
>
> I think this is a big design point - for example, if we have an extra
> buffer context structure, then per-receive queue frag allocators are not
> required for auto-tuning and we can reduce the number of patches in
> this patchset.
>
> I'm happy to implement either way.  Thanks!
>
> Best,
>
> Mike

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Jason Wang]

On 12/28/2013 05:41 AM, Michael Dalton wrote:
> I'm working on a followup patchset to address current feedback. I think
> it will be cleaner to do a debugfs implementation for per-receive queue
> packet buffer size exporting, so I'm trying that out.
>
> On Thu, Dec 26, 2013 at 7:04 PM, Jason Wang <jasowang@redhat.com> wrote:
>> We can make this more accurate by using extra data structure to track
>> the real buf size and using it as token.
> I agree -- we can do precise buffer total len tracking. Something like
> struct mergeable_packet_buffer_ctx {
>    void *buf;
>    unsigned int total_len;
> };
>
> Each receive queue could have a pointer to an array of N buffer contexts,
> where N is queue size (kzalloc'd in init_vqs or similar). That would
> allow us to allocate all of our buffer context data at startup.
>
> Would this be preferred to the current approach or is there another
> approach you would prefer? All other things being equal, having precise
> length tracking is advantageous, so I'm inclined to try this out and
> see how it goes.
>
> I think this is a big design point - for example, if we have an extra
> buffer context structure, then per-receive queue frag allocators are not
> required for auto-tuning and we can reduce the number of patches in
> this patchset.

Not required but better keep it. Consider we may have multiple
virtio-net cards, using per-receive queue frag may increase the
possibility of coalescing.
>
> I'm happy to implement either way.  Thanks!
>
> Best,
>
> Mike

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Amos Kong]

[-- Attachment #1: Type: text/plain, Size: 2562 bytes --]

On Mon, Dec 23, 2013 at 03:33:31PM +0200, Michael S. Tsirkin wrote:
> On Mon, Dec 16, 2013 at 04:16:29PM -0800, Michael Dalton wrote:
> > Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
> > allocators") changed the mergeable receive buffer size from PAGE_SIZE to
> > MTU-size, introducing a single-stream regression for benchmarks with large
> > average packet size. There is no single optimal buffer size for all
> > workloads.  For workloads with packet size <= MTU bytes, MTU + virtio-net
> > header-sized buffers are preferred as larger buffers reduce the TCP window
> > due to SKB truesize. However, single-stream workloads with large average
> > packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
> > are used.
> > 
> > This commit auto-tunes the mergeable receiver buffer packet size by
> > choosing the packet buffer size based on an EWMA of the recent packet
> > sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
> > virtio-net header len to PAGE_SIZE. This improves throughput for
> > large packet workloads, as any workload with average packet size >=
> > PAGE_SIZE will use PAGE_SIZE buffers.
> > 
> > These optimizations interact positively with recent commit
> > ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
> > which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
> > optimizations benefit buffers of any size.
> > 
> > Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
> > between two QEMU VMs on a single physical machine. Each VM has two VCPUs
> > with all offloads & vhost enabled. All VMs and vhost threads run in a
> > single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
> > in the system will not be scheduled on the benchmark CPUs. Trunk includes
> > SKB rx frag coalescing.
> > 
> > net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
> > net-next (MTU-size bufs):  13170.01Gb/s
> > net-next + auto-tune: 14555.94Gb/s

Hi Michael Dalton,

In Autotest framework, we compare performance results by T-test [1], and generate
an abundant/meaningful HTML files. It's helpful to check if performance regression
exists significantly.

If you have interest, we also use this in your netperf testing.

Attached analysis results of netperf tests.
[1] https://github.com/autotest/virt-test/wiki/PerformanceTesting


Thanks, Amos
 
> Also I guess this 1% difference is in the noise, right?
> Could you share data about host CPU utilization during
> these runs please?
 

[-- Attachment #2: netperf.html --]
[-- Type: text/html, Size: 103972 bytes --]

[-- Attachment #3: netperf.avg.html --]
[-- Type: text/html, Size: 228587 bytes --]

[-- Attachment #4: Type: text/plain, Size: 183 bytes --]

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

Currently because of how mm behaves (3.10.y) the code even before the
patch is a problem. I believe what may fix it is if instead of just
removing the conditional on __GFP_WAIT, the initial order > 0
allocation should be made GFP_ATOMIC, then fallback to the original
gfp mask for the order-0 allocations.

On systems that have highly fragmented main memory with pressure,
skb_page_frag_refill() causes problems. mm enters significant
compaction cycles on all cpu's which in itself is bad (add
considerable spinlock contention in isolate_migratepages_range() for
several seconds in kernel at 100% cpu), however even without this
happening basically we have large memory reclaimation when only
allocations from order-3 were necessary. For example, I might see half
the existing page cache on a system (2GB out of 8GB) reclaimed in a
burst, which effectively means the application has to wait even longer
after this compact/reclaim cycle for those pages to be read back from
disk. This is a significant reduction in useful memory from before
skb_page_frag_refill() existed, as one of our systems could run in
steady state will little free memory and 100% fragmentation. Now I see
10-30x more memory free (read: not utilized). Order > 0 allocations
were happening rarely before, now it happens consistently from this
function.

My suggestion above would avoid mm going through
__alloc_pages_direct_compact() and triggering the bad events above. It
will take me several days to try this experiment.

-Debabrata

On Tue, Dec 24, 2013 at 5:46 PM, David Miller <davem@davemloft.net> wrote:
>
> There is still feedback and/or minor adjustments being asked for wrt.
> this series.   These changes have been sitting for more than a week
> which is a bit rediculous.
>
> Please resubmit these changes once everything is resolved to
> everyone's satisfaction, thanks.
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

Currently because of how mm behaves (3.10.y) the code even before the
patch is a problem. I believe what may fix it is if instead of just
removing the conditional on __GFP_WAIT, the initial order > 0
allocation should be made GFP_ATOMIC, then fallback to the original
gfp mask for the order-0 allocations.

On systems that have highly fragmented main memory with pressure,
skb_page_frag_refill() causes problems. mm enters significant
compaction cycles on all cpu's which in itself is bad (add
considerable spinlock contention in isolate_migratepages_range() for
several seconds in kernel at 100% cpu), however even without this
happening basically we have large memory reclaimation when only
allocations from order-3 were necessary. For example, I might see half
the existing page cache on a system (2GB out of 8GB) reclaimed in a
burst, which effectively means the application has to wait even longer
after this compact/reclaim cycle for those pages to be read back from
disk. This is a significant reduction in useful memory from before
skb_page_frag_refill() existed, as one of our systems could run in
steady state will little free memory and 100% fragmentation. Now I see
10-30x more memory free (read: not utilized). Order > 0 allocations
were happening rarely before, now it happens consistently from this
function.

My suggestion above would avoid mm going through
__alloc_pages_direct_compact() and triggering the bad events above. It
will take me several days to try this experiment.

-Debabrata

On Tue, Dec 24, 2013 at 5:46 PM, David Miller <davem@davemloft.net> wrote:
>
> There is still feedback and/or minor adjustments being asked for wrt.
> this series.   These changes have been sitting for more than a week
> which is a bit rediculous.
>
> Please resubmit these changes once everything is resolved to
> everyone's satisfaction, thanks.
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Eric Dumazet]

On Thu, 2014-01-02 at 19:42 -0500, Debabrata Banerjee wrote:
> Currently because of how mm behaves (3.10.y) the code even before the
> patch is a problem. I believe what may fix it is if instead of just
> removing the conditional on __GFP_WAIT, the initial order > 0
> allocation should be made GFP_ATOMIC, then fallback to the original
> gfp mask for the order-0 allocations.
> 
> On systems that have highly fragmented main memory with pressure,
> skb_page_frag_refill() causes problems. mm enters significant
> compaction cycles on all cpu's which in itself is bad (add
> considerable spinlock contention in isolate_migratepages_range() for
> several seconds in kernel at 100% cpu), however even without this
> happening basically we have large memory reclaimation when only
> allocations from order-3 were necessary. For example, I might see half
> the existing page cache on a system (2GB out of 8GB) reclaimed in a
> burst, which effectively means the application has to wait even longer
> after this compact/reclaim cycle for those pages to be read back from
> disk. This is a significant reduction in useful memory from before
> skb_page_frag_refill() existed, as one of our systems could run in
> steady state will little free memory and 100% fragmentation. Now I see
> 10-30x more memory free (read: not utilized). Order > 0 allocations
> were happening rarely before, now it happens consistently from this
> function.
> 
> My suggestion above would avoid mm going through
> __alloc_pages_direct_compact() and triggering the bad events above. It
> will take me several days to try this experiment.

My suggestion is to use a recent kernel, and/or eventually backport the
mm fixes if any.

order-3 allocations should not reclaim 2GB out of 8GB.

There is a reason PAGE_ALLOC_COSTLY_ORDER exists and is 3

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Eric Dumazet]

On Thu, 2014-01-02 at 16:56 -0800, Eric Dumazet wrote:

> 
> My suggestion is to use a recent kernel, and/or eventually backport the
> mm fixes if any.
> 
> order-3 allocations should not reclaim 2GB out of 8GB.
> 
> There is a reason PAGE_ALLOC_COSTLY_ORDER exists and is 3

Hmm... it looks like I missed __GFP_NORETRY



diff --git a/net/core/sock.c b/net/core/sock.c
index 5393b4b719d7..5f42a4d70cb2 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
 		gfp_t gfp = prio;
 
 		if (order)
-			gfp |= __GFP_COMP | __GFP_NOWARN;
+			gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
 		pfrag->page = alloc_pages(gfp, order);
 		if (likely(pfrag->page)) {
 			pfrag->offset = 0;

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

On Thu, Jan 2, 2014 at 8:26 PM, Eric Dumazet <eric.dumazet@gmail.com> wrote:
> On Thu, 2014-01-02 at 16:56 -0800, Eric Dumazet wrote:
>
>>
>> My suggestion is to use a recent kernel, and/or eventually backport the
>> mm fixes if any.
>>
>> order-3 allocations should not reclaim 2GB out of 8GB.
>>
>> There is a reason PAGE_ALLOC_COSTLY_ORDER exists and is 3

Sorry 2GB cache out of 8GB phys, ~1GB gets reclaimed. Regardless the
reclaimation of cache is minor compared to the compaction event that
precedes it, I haven't spotted something addressing that yet -
isolate_migratepages_range ()/compact_checklock_irqsave(). If even
more of memory was unmoveable, the compaction routines would be hit
even harder as reclaimation wouldn't do anything - mm would have to
get very very smart about unmoveable pages being freed and just fail
allocations/oom kill if nothing has changed without running through
compaction/reclaim fruitlessly.

I guess this is a bit of a tangent since what I'm saying proves the
patch from Michael doesn't make this behavior worse.

>
> Hmm... it looks like I missed __GFP_NORETRY
>
>
>
> diff --git a/net/core/sock.c b/net/core/sock.c
> index 5393b4b719d7..5f42a4d70cb2 100644
> --- a/net/core/sock.c
> +++ b/net/core/sock.c
> @@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
>                 gfp_t gfp = prio;
>
>                 if (order)
> -                       gfp |= __GFP_COMP | __GFP_NOWARN;
> +                       gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
>                 pfrag->page = alloc_pages(gfp, order);
>                 if (likely(pfrag->page)) {
>                         pfrag->offset = 0;
>
>
>

Yes this seems like it will make the situation better, but one send()
may still cause a direct_compact and direct_reclaim() cycle to happen,
followed immediately by another direct_compact() if direct_reclaim()
didn't free an order-3. Now have all cpu's doing a send(), you can
still get heavy spinlock contention in the routines described above.
The major change I see here is that allocations > order-0 used to be
rare, now it's on every send().

I can try your patch to see how much things improve.

-Debabrata

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

On Thu, Jan 2, 2014 at 8:26 PM, Eric Dumazet <eric.dumazet@gmail.com> wrote:
> On Thu, 2014-01-02 at 16:56 -0800, Eric Dumazet wrote:
>
>>
>> My suggestion is to use a recent kernel, and/or eventually backport the
>> mm fixes if any.
>>
>> order-3 allocations should not reclaim 2GB out of 8GB.
>>
>> There is a reason PAGE_ALLOC_COSTLY_ORDER exists and is 3

Sorry 2GB cache out of 8GB phys, ~1GB gets reclaimed. Regardless the
reclaimation of cache is minor compared to the compaction event that
precedes it, I haven't spotted something addressing that yet -
isolate_migratepages_range ()/compact_checklock_irqsave(). If even
more of memory was unmoveable, the compaction routines would be hit
even harder as reclaimation wouldn't do anything - mm would have to
get very very smart about unmoveable pages being freed and just fail
allocations/oom kill if nothing has changed without running through
compaction/reclaim fruitlessly.

I guess this is a bit of a tangent since what I'm saying proves the
patch from Michael doesn't make this behavior worse.

>
> Hmm... it looks like I missed __GFP_NORETRY
>
>
>
> diff --git a/net/core/sock.c b/net/core/sock.c
> index 5393b4b719d7..5f42a4d70cb2 100644
> --- a/net/core/sock.c
> +++ b/net/core/sock.c
> @@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
>                 gfp_t gfp = prio;
>
>                 if (order)
> -                       gfp |= __GFP_COMP | __GFP_NOWARN;
> +                       gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
>                 pfrag->page = alloc_pages(gfp, order);
>                 if (likely(pfrag->page)) {
>                         pfrag->offset = 0;
>
>
>

Yes this seems like it will make the situation better, but one send()
may still cause a direct_compact and direct_reclaim() cycle to happen,
followed immediately by another direct_compact() if direct_reclaim()
didn't free an order-3. Now have all cpu's doing a send(), you can
still get heavy spinlock contention in the routines described above.
The major change I see here is that allocations > order-0 used to be
rare, now it's on every send().

I can try your patch to see how much things improve.

-Debabrata

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

>> On Thu, 2014-01-02 at 16:56 -0800, Eric Dumazet wrote:
>>
>> Hmm... it looks like I missed __GFP_NORETRY
>>
>>
>>
>> diff --git a/net/core/sock.c b/net/core/sock.c
>> index 5393b4b719d7..5f42a4d70cb2 100644
>> --- a/net/core/sock.c
>> +++ b/net/core/sock.c
>> @@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
>>                 gfp_t gfp = prio;
>>
>>                 if (order)
>> -                       gfp |= __GFP_COMP | __GFP_NOWARN;
>> +                       gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
>>                 pfrag->page = alloc_pages(gfp, order);
>>                 if (likely(pfrag->page)) {
>>                         pfrag->offset = 0;
>>
>>
>>

There is another patch needed (looks like good stable fixes):
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 06e72d3..d42d48c 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -378,7 +378,7 @@ refill:
                        gfp_t gfp = gfp_mask;

                        if (order)
-                               gfp |= __GFP_COMP | __GFP_NOWARN;
+                               gfp |= __GFP_COMP | __GFP_NOWARN |
__GFP_NORETRY;
                        nc->frag.page = alloc_pages(gfp, order);
                        if (likely(nc->frag.page))
                                break;

This reduces the really pathological compact/reclaim behavior but
doesn't fix it. Actually it still really quite bad because the whole
thing loops until it gets to order-0 so it's effectively trying the
allocation 4 times anyway. I typically see non-zero order allocations
very rarely without these two pieces of code. I hotpatched a running
system to get results from this quickly. Even setting the max order to
order-1 I still see bad behavior. If anything this behavior should be
conditional until this is ironed out.

Performance data: http://pastebin.ubuntu.com/6687527/

-Debabrata

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

>> On Thu, 2014-01-02 at 16:56 -0800, Eric Dumazet wrote:
>>
>> Hmm... it looks like I missed __GFP_NORETRY
>>
>>
>>
>> diff --git a/net/core/sock.c b/net/core/sock.c
>> index 5393b4b719d7..5f42a4d70cb2 100644
>> --- a/net/core/sock.c
>> +++ b/net/core/sock.c
>> @@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
>>                 gfp_t gfp = prio;
>>
>>                 if (order)
>> -                       gfp |= __GFP_COMP | __GFP_NOWARN;
>> +                       gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
>>                 pfrag->page = alloc_pages(gfp, order);
>>                 if (likely(pfrag->page)) {
>>                         pfrag->offset = 0;
>>
>>
>>

There is another patch needed (looks like good stable fixes):
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 06e72d3..d42d48c 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -378,7 +378,7 @@ refill:
                        gfp_t gfp = gfp_mask;

                        if (order)
-                               gfp |= __GFP_COMP | __GFP_NOWARN;
+                               gfp |= __GFP_COMP | __GFP_NOWARN |
__GFP_NORETRY;
                        nc->frag.page = alloc_pages(gfp, order);
                        if (likely(nc->frag.page))
                                break;

This reduces the really pathological compact/reclaim behavior but
doesn't fix it. Actually it still really quite bad because the whole
thing loops until it gets to order-0 so it's effectively trying the
allocation 4 times anyway. I typically see non-zero order allocations
very rarely without these two pieces of code. I hotpatched a running
system to get results from this quickly. Even setting the max order to
order-1 I still see bad behavior. If anything this behavior should be
conditional until this is ironed out.

Performance data: http://pastebin.ubuntu.com/6687527/

-Debabrata

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Eric Dumazet]

On Fri, 2014-01-03 at 17:47 -0500, Debabrata Banerjee wrote:
> >> On Thu, 2014-01-02 at 16:56 -0800, Eric Dumazet wrote:
> >>
> >> Hmm... it looks like I missed __GFP_NORETRY
> >>
> >>
> >>
> >> diff --git a/net/core/sock.c b/net/core/sock.c
> >> index 5393b4b719d7..5f42a4d70cb2 100644
> >> --- a/net/core/sock.c
> >> +++ b/net/core/sock.c
> >> @@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
> >>                 gfp_t gfp = prio;
> >>
> >>                 if (order)
> >> -                       gfp |= __GFP_COMP | __GFP_NOWARN;
> >> +                       gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
> >>                 pfrag->page = alloc_pages(gfp, order);
> >>                 if (likely(pfrag->page)) {
> >>                         pfrag->offset = 0;
> >>
> >>
> >>
> 
> There is another patch needed (looks like good stable fixes):
> diff --git a/net/core/skbuff.c b/net/core/skbuff.c
> index 06e72d3..d42d48c 100644
> --- a/net/core/skbuff.c
> +++ b/net/core/skbuff.c
> @@ -378,7 +378,7 @@ refill:
>                         gfp_t gfp = gfp_mask;
> 
>                         if (order)
> -                               gfp |= __GFP_COMP | __GFP_NOWARN;
> +                               gfp |= __GFP_COMP | __GFP_NOWARN |
> __GFP_NORETRY;
>                         nc->frag.page = alloc_pages(gfp, order);
>                         if (likely(nc->frag.page))
>                                 break;
> 

This is in GFP_ATOMIC cases, I dont think it can ever start compaction.

> This reduces the really pathological compact/reclaim behavior but
> doesn't fix it. Actually it still really quite bad because the whole
> thing loops until it gets to order-0 so it's effectively trying the
> allocation 4 times anyway. I typically see non-zero order allocations
> very rarely without these two pieces of code. I hotpatched a running
> system to get results from this quickly. Even setting the max order to
> order-1 I still see bad behavior. If anything this behavior should be
> conditional until this is ironed out.
> 
> Performance data: http://pastebin.ubuntu.com/6687527/

It seems that you shoot the messenger : If memory is fragmented, then
one order-1 allocation is going to start compaction.

It can be a simple fork().

If your workload never fork(), then yes, you never needed compaction.

It doesn't really matter to say that which memory allocation triggered
compaction, which is a normal step in mm layer.

If you believe its badly done, you should ask to mm guys to fix/improve
it, not netdev...

We are not trying to optimize the kernel behavior for hosts in deep
memory pressure.

Using order-3 pages in TCP stack improves performance for 99% of the
hosts, there might be something wrong on your side ?

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

On Fri, Jan 3, 2014 at 5:54 PM, Eric Dumazet <eric.dumazet@gmail.com> wrote:
>
> This is in GFP_ATOMIC cases, I dont think it can ever start compaction.

I think that's right I probably finally got it back to normal behavior
with order-0 allocations.

>
> It seems that you shoot the messenger : If memory is fragmented, then
> one order-1 allocation is going to start compaction.
>
> It can be a simple fork().
>
> If your workload never fork(), then yes, you never needed compaction.
>

Sure but the rate of network packets in and out and subsequent
allocations would be more equivalent to a fork bomb than normal
forking. I understand mm should work more sanely in this scenario but
at the same time we see a bad regression with this code, I see we're
not alone.

>
> We are not trying to optimize the kernel behavior for hosts in deep
> memory pressure.

We're leaving about half for the kernel so I wouldn't call it "deep".
Any server application that is using page cache and mlocked memory
will run into similar issues.

> It doesn't really matter to say that which memory allocation triggered
> compaction, which is a normal step in mm layer.
>
> If you believe its badly done, you should ask to mm guys to fix/improve
> it, not netdev...
>
> Using order-3 pages in TCP stack improves performance for 99% of the
> hosts, there might be something wrong on your side ?
>

Having lots of memory mlocked is bad right now yes, but not
necessarily an uncommon scenario. We're handing mm an almost
intractable problem. I see compaction of mlocked pages has been
discussed a few times over there, but no patch has actually made it
in.

-Debabrata

Re: [PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill

[Debabrata Banerjee]

On Fri, Jan 3, 2014 at 5:54 PM, Eric Dumazet <eric.dumazet@gmail.com> wrote:
>
> This is in GFP_ATOMIC cases, I dont think it can ever start compaction.

I think that's right I probably finally got it back to normal behavior
with order-0 allocations.

>
> It seems that you shoot the messenger : If memory is fragmented, then
> one order-1 allocation is going to start compaction.
>
> It can be a simple fork().
>
> If your workload never fork(), then yes, you never needed compaction.
>

Sure but the rate of network packets in and out and subsequent
allocations would be more equivalent to a fork bomb than normal
forking. I understand mm should work more sanely in this scenario but
at the same time we see a bad regression with this code, I see we're
not alone.

>
> We are not trying to optimize the kernel behavior for hosts in deep
> memory pressure.

We're leaving about half for the kernel so I wouldn't call it "deep".
Any server application that is using page cache and mlocked memory
will run into similar issues.

> It doesn't really matter to say that which memory allocation triggered
> compaction, which is a normal step in mm layer.
>
> If you believe its badly done, you should ask to mm guys to fix/improve
> it, not netdev...
>
> Using order-3 pages in TCP stack improves performance for 99% of the
> hosts, there might be something wrong on your side ?
>

Having lots of memory mlocked is bad right now yes, but not
necessarily an uncommon scenario. We're handing mm an almost
intractable problem. I see compaction of mlocked pages has been
discussed a few times over there, but no patch has actually made it
in.

-Debabrata

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Michael S. Tsirkin]

On Thu, Dec 26, 2013 at 02:00:58PM -0800, Eric Dumazet wrote:
> On Thu, 2013-12-26 at 23:37 +0200, Michael S. Tsirkin wrote:
> 
> > Interesting. But if we can't allocate a buffer how can we
> > do network processing?
> 
> How typical NIC drivers handle this case ?
> 
> Answer : nothing special should happen, we drop incoming traffic,
> and make sure the driver recovers properly. (like not NULL deref or
> crazy things like that)
> 
> Why virtio_net should be different ?

Basically yes, we could start dropping packets immediately
once GFP_ATOMIC allocations fail and repost the buffer to host,
and hope memory is available by the time we get the next interrupt.
But we wanted host to have visibility into the fact that
we are out of memory and packets are dropped, so we did not want to
repost.
If we don't repost how do we know memory is finally available?
We went for a timer based workqueue thing.
What do you suggest?



> > 
> > If we can reproduce the problem, we can maybe move
> > allocation out of napi disabled section, but then
> > we'll need to add more locking.
> 
> More exactly, use appropriate locking ;)
> 

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael S. Tsirkin]

On Fri, Dec 27, 2013 at 01:41:28PM -0800, Michael Dalton wrote:
> I'm working on a followup patchset to address current feedback. I think
> it will be cleaner to do a debugfs implementation for per-receive queue
> packet buffer size exporting, so I'm trying that out.
> 
> On Thu, Dec 26, 2013 at 7:04 PM, Jason Wang <jasowang@redhat.com> wrote:
> > We can make this more accurate by using extra data structure to track
> > the real buf size and using it as token.
> 
> I agree -- we can do precise buffer total len tracking. Something like
> struct mergeable_packet_buffer_ctx {
>    void *buf;
>    unsigned int total_len;

Maybe make total_len long so size is a power of 2.

> };

Hmm this doubles VQ cache footprint.
In the past when I tried increasong cache footprint
this hurt performance measureable. It's just a suggestion though,
YMMV, if numbers are good we don't need to argue about this.

> 
> Each receive queue could have a pointer to an array of N buffer contexts,
> where N is queue size (kzalloc'd in init_vqs or similar). That would
> allow us to allocate all of our buffer context data at startup.
> 
> Would this be preferred to the current approach or is there another
> approach you would prefer? All other things being equal, having precise
> length tracking is advantageous, so I'm inclined to try this out and
> see how it goes.
> 
> I think this is a big design point - for example, if we have an extra
> buffer context structure, then per-receive queue frag allocators are not
> required for auto-tuning and we can reduce the number of patches in
> this patchset.

I'd be careful with adding even more stuff in
mergeable_packet_buffer_ctx for above reason.

> I'm happy to implement either way.  Thanks!
> 
> Best,
> 
> Mike

Re: [PATCH net-next 3/3] net: auto-tune mergeable rx buffer size for improved performance

[Michael S. Tsirkin]

On Mon, Dec 30, 2013 at 06:14:46PM +0800, Amos Kong wrote:
> On Mon, Dec 23, 2013 at 03:33:31PM +0200, Michael S. Tsirkin wrote:
> > On Mon, Dec 16, 2013 at 04:16:29PM -0800, Michael Dalton wrote:
> > > Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
> > > allocators") changed the mergeable receive buffer size from PAGE_SIZE to
> > > MTU-size, introducing a single-stream regression for benchmarks with large
> > > average packet size. There is no single optimal buffer size for all
> > > workloads.  For workloads with packet size <= MTU bytes, MTU + virtio-net
> > > header-sized buffers are preferred as larger buffers reduce the TCP window
> > > due to SKB truesize. However, single-stream workloads with large average
> > > packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
> > > are used.
> > > 
> > > This commit auto-tunes the mergeable receiver buffer packet size by
> > > choosing the packet buffer size based on an EWMA of the recent packet
> > > sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
> > > virtio-net header len to PAGE_SIZE. This improves throughput for
> > > large packet workloads, as any workload with average packet size >=
> > > PAGE_SIZE will use PAGE_SIZE buffers.
> > > 
> > > These optimizations interact positively with recent commit
> > > ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
> > > which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
> > > optimizations benefit buffers of any size.
> > > 
> > > Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
> > > between two QEMU VMs on a single physical machine. Each VM has two VCPUs
> > > with all offloads & vhost enabled. All VMs and vhost threads run in a
> > > single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
> > > in the system will not be scheduled on the benchmark CPUs. Trunk includes
> > > SKB rx frag coalescing.
> > > 
> > > net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
> > > net-next (MTU-size bufs):  13170.01Gb/s
> > > net-next + auto-tune: 14555.94Gb/s
> 
> Hi Michael Dalton,
> 
> In Autotest framework, we compare performance results by T-test [1], and generate
> an abundant/meaningful HTML files. It's helpful to check if performance regression
> exists significantly.

Nice. Specifically, this checks Throughput divided by CPU utilization
which is important I think.
Sometimes host scheduler gives us much more CPU and throughput
goes up, but if that's what happens, it won't scale with other work running on same host.

> If you have interest, we also use this in your netperf testing.
> 
> Attached analysis results of netperf tests.
> [1] https://github.com/autotest/virt-test/wiki/PerformanceTesting
> 
> 
> Thanks, Amos
>  
> > Also I guess this 1% difference is in the noise, right?
> > Could you share data about host CPU utilization during
> > these runs please?
>  

> ####1. Description of setup#1
>  qemu-kvm-0.12.1.2-2.209.el6.x86_64
>  host kernel: 3.4.0.zerocopy+
>  guest kernel:2.6.32-220.el6.x86_64
> 
> ####2. Description of setup#2
>  qemu-kvm-0.12.1.2-2.209.el6.x86_64
>  host kernel: 3.4.0.zerocopy+
>  guest kernel:2.6.32-220.el6.x86_64
> 
>  The tests are 60 seconds sessions of "Netperf". 'throughput' was taken from netperf's report.
>  other measurements were taken on the host.
>  How to read the results:
>  - The Throughput is measured in Mbit/sec.
>  - io_exit: io exits of KVM.
>  - irq_inj: irq injections of KVM.
> 
>  - Every Avg line represents the average value based on *5* repetitions of the same test,
>    and the following SD line represents the Standard Deviation between the *5* repetitions.
>  - The Standard deviation is displayed as a percentage of the average.
>  - The significance of the differences between the two averages is calculated using unpaired T-test that
>    takes into account the SD of the averages.
>  - The paired t-test is computed for the averages of same category.
> 
> 
> ┌──────────┬──────────────────────────────┬──────────┬────────┬────────┬────────┬────────┬─────────┬───────────┬────────┬────────┬────────┬────────┬────────┬─────────┬─────────┐
> │==        │                              │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> │Regression│     Category:TCP_STREAM      │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> │Testing:  │                              │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> │netperf ==│                              │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┬────────┬────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    #     │    Tile    │  size  │sessions│throughput│  %CPU  │thr/%CPU│#tx-pkts│#rx-pkts│#tx-byts │ #rx-byts  │   #    │#tx-intr│#rx-intr│#io_exit│#irq_inj│ #tpkt/# │ #rpkt/# │
> │          │            │        │        │          │        │        │        │        │         │           │re-trans│        │        │        │        │  exit   │   irq   │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 3654.05  │ 22.30  │ 163.86 │2813996 │18928829│185871160│28658172593│   0    │   42   │2470996 │ 278105 │2530858 │  10.12  │  7.48   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   11%    │  14%   │   4%   │  38%   │  11%   │   38%   │    11%    │  0.0   │  39%   │  52%   │  12%   │  51%   │   32%   │  324%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 4033.57  │ 23.08  │ 174.76 │2681221 │20895339│177070935│31633911136│   0    │   43   │2766747 │ 221023 │2825177 │  12.13  │  7.40   │
> ├──────────┼────────────┤  4000  │   1    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   22%    │   7%   │  27%   │  27%   │  22%   │   27%   │    22%    │  0.0   │  32%   │  43%   │  51%   │  42%   │  478%   │   78%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ +10.386% │+3.498% │+6.652% │-4.718% │+10.389%│ -4.735% │ +10.384%  │  +0.0  │+2.381% │+11.969%│-20.525%│+11.629%│+19.862% │ -1.070% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.585   │ 0.360  │ 0.406  │ 0.174  │ 0.585  │  0.175  │   0.585   │ 0.653  │ 0.065  │ 0.287  │ 0.690  │ 0.286  │  0.648  │  0.506  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 3953.32  │ 23.07  │ 171.36 │2623463 │20480990│173270716│31008001226│   0    │   54   │2406189 │ 206773 │2472555 │  12.69  │  8.28   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   16%    │   6%   │  16%   │  22%   │  16%   │   22%   │    16%    │  0.0   │  50%   │  43%   │  68%   │  41%   │  186%   │  109%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 4818.95  │ 22.79  │ 211.45 │3171179 │24963857│209475231│37795242761│   0    │   48   │2754395 │ 371403 │2837031 │  8.54   │  8.80   │
> ├──────────┼────────────┤  4000  │   2    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   32%    │   7%   │  26%   │  57%   │  32%   │   57%   │    32%    │  0.0   │  57%   │  78%   │  34%   │  76%   │   20%   │  104%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ +21.896% │-1.214% │+23.395%│+20.878%│+21.888%│+20.895% │ +21.889%  │  +0.0  │-11.111%│+14.471%│+79.619%│+14.741%│-32.703% │ +6.280% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.724   │ 0.225  │ 0.797  │ 0.463  │ 0.724  │  0.464  │   0.724   │ 0.653  │ 0.256  │ 0.249  │ 0.914  │ 0.259  │  0.846  │  0.202  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4342.74  │ 26.09  │ 166.45 │3086160 │22499585│203857770│34064280140│   0    │  369   │2287182 │ 173759 │2392793 │  17.76  │  9.40   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   10%    │  14%   │   9%   │  42%   │  10%   │   42%   │    10%    │  0.0   │  36%   │  34%   │  13%   │  34%   │   51%   │   31%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 3963.69  │ 24.11  │ 164.40 │1870501 │20537591│123620068│31093773242│   0    │  435   │2506725 │ 143191 │2581189 │  13.06  │  7.96   │
> ├──────────┼────────────┤  4000  │   4    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │    4%    │   4%   │   3%   │  23%   │   4%   │   23%   │    4%     │  0.0   │  29%   │  18%   │  33%   │  18%   │   37%   │   18%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ -8.728%  │-7.589% │-1.232% │-39.391%│-8.720% │-39.360% │  -8.720%  │  +0.0  │+17.886%│+9.599% │-17.592%│+7.873% │-26.464% │-15.319% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.894   │ 0.716  │ 0.329  │ 0.918  │ 0.894  │  0.918  │   0.894   │ 0.653  │ 0.559  │ 0.399  │ 0.775  │ 0.339  │  0.610  │  0.799  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4442.45  │ 22.70  │ 195.70 │2269020 │23012158│149887965│34840362128│   0    │   34   │2187933 │ 273170 │2248565 │  8.31   │  10.23  │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   12%    │   6%   │  18%   │  36%   │  12%   │   36%   │    12%    │  0.0   │  36%   │  39%   │   4%   │  38%   │   33%   │   50%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 3784.41  │ 22.34  │ 169.40 │2839507 │19603918│187698543│29680297470│   0    │   43   │2188310 │ 282257 │2247171 │  10.06  │  8.72   │
> ├──────────┼────────────┤ 20000  │   1    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   18%    │  16%   │  17%   │  40%   │  18%   │   40%   │    18%    │  0.0   │  41%   │  60%   │  10%   │  59%   │   31%   │  266%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ -14.813% │-1.586% │-13.439%│+25.142%│-14.811%│+25.226% │ -14.811%  │  +0.0  │+26.471%│+0.017% │+3.326% │-0.062% │+21.059% │-14.761% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.874   │ 0.160  │ 0.767  │ 0.609  │ 0.874  │  0.611  │   0.874   │ 0.653  │ 0.600  │ 0.000  │ 0.467  │ 0.002  │  0.560  │  0.476  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4394.70  │ 22.29  │ 197.16 │2587799 │22768355│170940782│34470462005│   0    │   62   │3265444 │ 243982 │3339217 │  10.61  │  6.82   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   21%    │  15%   │  39%   │  19%   │  21%   │   19%   │    21%    │  0.0   │  81%   │  28%   │  37%   │  27%   │   79%   │   23%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 3908.76  │ 23.63  │ 165.42 │2989759 │20249863│197517079│30657452074│   0    │   48   │2925685 │ 252785 │2992574 │  11.83  │  6.77   │
> ├──────────┼────────────┤ 20000  │   2    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   20%    │   7%   │  17%   │  22%   │  20%   │   22%   │    20%    │  0.0   │  29%   │  47%   │  52%   │  46%   │  558%   │   87%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ -11.057% │+6.012% │-16.099%│+15.533%│-11.061%│+15.547% │ -11.062%  │  +0.0  │-22.581%│-10.405%│+3.608% │-10.381%│+11.499% │ -0.733% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.599   │ 0.550  │ 0.695  │ 0.693  │ 0.599  │  0.694  │   0.599   │ 0.653  │ 0.426  │ 0.342  │ 0.095  │ 0.348  │  0.589  │  0.475  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4894.89  │ 25.13  │ 194.78 │3082568 │25360893│203681023│38396295385│   0    │  364   │3068186 │ 239536 │3172145 │  12.87  │  7.99   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   34%    │   5%   │  30%   │  85%   │  34%   │   85%   │    34%    │  0.0   │  42%   │  30%   │  96%   │  31%   │   32%   │   5%    │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 5142.45  │ 26.12  │ 196.88 │3600457 │26643682│237950950│40338431022│   0    │  340   │2871917 │ 248898 │2992869 │  14.47  │  8.90   │
> ├──────────┼────────────┤ 20000  │   4    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   32%    │  11%   │  33%   │  63%   │  32%   │   64%   │    32%    │  0.0   │  58%   │  26%   │  91%   │  27%   │   59%   │   14%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ +5.058%  │+3.940% │+1.078% │+16.801%│+5.058% │+16.825% │  +5.058%  │  +0.0  │-6.593% │-6.397% │+3.908% │-5.652% │+12.432% │+11.389% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.183   │ 0.510  │ 0.090  │ 0.253  │ 0.183  │  0.253  │   0.183   │ 0.653  │ 0.165  │ 0.280  │ 0.050  │ 0.240  │  0.601  │  0.853  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4012.65  │ 21.78  │ 184.24 │2307643 │20785874│152551903│31469785004│   0    │   35   │1865340 │ 291537 │1925266 │  7.92   │  10.80  │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   20%    │  16%   │  18%   │  49%   │  20%   │   49%   │    20%    │  0.0   │  49%   │  65%   │   8%   │  63%   │   42%   │  218%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 4029.64  │ 22.23  │ 181.27 │2502225 │20873891│165368585│31603029704│   0    │   38   │1910923 │ 267096 │1970568 │  9.37   │  10.59  │
> ├──────────┼────────────┤ 40000  │   1    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   21%    │  15%   │  17%   │  52%   │  21%   │   52%   │    21%    │  0.0   │  52%   │  62%   │  10%   │  60%   │   42%   │  211%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ +0.423%  │+2.066% │-1.612% │+8.432% │+0.423% │ +8.402% │  +0.423%  │  +0.0  │+8.571% │+2.444% │-8.383% │+2.353% │+18.308% │ -1.944% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.025   │ 0.158  │ 0.131  │ 0.193  │ 0.025  │  0.192  │   0.025   │ 0.653  │ 0.196  │ 0.047  │ 0.821  │ 0.046  │  0.419  │  0.055  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4426.90  │ 23.58  │ 187.74 │2816686 │22934457│186110754│34721995043│   0    │   55   │4385070 │ 256976 │4457988 │  10.96  │  5.14   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   28%    │   5%   │  30%   │  43%   │  28%   │   43%   │    28%    │  0.0   │  58%   │  43%   │  47%   │  42%   │  207%   │   14%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 3948.19  │ 22.83  │ 172.94 │2065324 │20453563│136432079│30966657915│   0    │   58   │2172531 │ 224001 │2241149 │  9.22   │  9.13   │
> ├──────────┼────────────┤ 40000  │   2    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   11%    │   6%   │   7%   │  24%   │  11%   │   24%   │    11%    │  0.0   │  74%   │  34%   │  54%   │  32%   │   53%   │   27%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ -10.814% │-3.181% │-7.883% │-26.675%│-10.817%│-26.693% │ -10.815%  │  +0.0  │+5.455% │-50.456%│-12.832%│-49.727%│-15.876% │+77.626% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.568   │ 0.606  │ 0.446  │ 0.764  │ 0.568  │  0.765  │   0.568   │ 0.653  │ 0.090  │ -0.961 │ 0.323  │ -0.960 │  0.525  │ +0.995  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4205.99  │ 25.20  │ 166.90 │2596046 │21792738│171527637│32994109547│   0    │  308   │1914297 │ 205798 │2022008 │  12.61  │  10.78  │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   10%    │  11%   │   6%   │  58%   │  10%   │   58%   │    10%    │  0.0   │  21%   │  30%   │  33%   │  26%   │   26%   │   37%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 4054.46  │ 24.54  │ 165.22 │2241165 │21006264│148096081│31803374353│   0    │  430   │2268245 │ 171670 │2363749 │  13.06  │  8.89   │
> ├──────────┼────────────┤ 40000  │   4    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │    4%    │   6%   │   7%   │  42%   │   4%   │   42%   │    4%     │  0.0   │   9%   │  22%   │  24%   │  20%   │   21%   │   21%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ -3.603%  │-2.619% │-1.007% │-13.670%│-3.609% │-13.661% │  -3.609%  │  +0.0  │+39.610%│+18.490%│-16.583%│+16.901%│ +3.569% │-17.532% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.531   │ 0.344  │ 0.191  │ 0.334  │ 0.531  │  0.334  │   0.531   │ 0.653  │ +0.993 │ 0.674  │ 0.636  │ 0.691  │  0.321  │  0.738  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4249.31  │ 23.92  │ 177.65 │2662115 │22011879│175903720│33325960247│   0    │   40   │2453648 │ 281530 │2514216 │  9.46   │  8.75   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   13%    │   5%   │  16%   │  31%   │  13%   │   31%   │    13%    │  0.0   │  31%   │  34%   │   7%   │  34%   │   25%   │   59%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 3432.07  │ 17.06  │ 201.18 │1927368 │17779003│127284944│26916454466│   0    │   29   │1900741 │ 215296 │1958147 │  8.95   │  9.08   │
> ├──────────┼────────────┤ 64000  │   1    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   25%    │  58%   │ 2667%  │  47%   │  25%   │   47%   │    25%    │  0.0   │  48%   │  98%   │  52%   │  95%   │  546%   │  315%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ -19.232% │-28.679%│+13.245%│-27.600%│-19.230%│-27.639% │ -19.233%  │  +0.0  │-27.500%│-22.534%│-23.526%│-22.117%│ -5.391% │ +3.771% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.885   │ 0.839  │ 0.652  │ 0.784  │ 0.885  │  0.785  │   0.885   │ 0.653  │ 0.760  │ 0.437  │ 0.774  │ 0.440  │  0.591  │  0.812  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4384.12  │ 23.75  │ 184.59 │2236188 │22711421│147716182│34385017633│   0    │   51   │2469602 │ 214045 │2536198 │  10.45  │  8.95   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   15%    │   2%   │  14%   │  23%   │  15%   │   23%   │    15%    │  0.0   │  41%   │  20%   │  60%   │  19%   │  118%   │   42%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 3917.46  │ 23.30  │ 168.13 │2229968 │20294602│147309151│30725795840│   0    │   37   │2789905 │ 253739 │2858678 │  8.79   │  7.10   │
> ├──────────┼────────────┤ 64000  │   2    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   14%    │   7%   │  11%   │  17%   │  14%   │   17%   │    14%    │  0.0   │   8%   │  31%   │  44%   │  30%   │   83%   │   35%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ -10.644% │-1.895% │-8.917% │-0.278% │-10.641%│ -0.276% │ -10.642%  │  +0.0  │-27.451%│+12.970%│+18.545%│+12.715%│-15.885% │-20.670% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.739   │ 0.439  │ 0.736  │ 0.017  │ 0.739  │  0.017  │   0.739   │ 0.653  │ 0.823  │ 0.506  │ 0.384  │ 0.512  │  0.464  │  0.626  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    1     │    Avg     │        │        │ 4022.06  │ 24.92  │ 161.40 │2051216 │20840308│135569610│31552148131│   0    │  414   │2287715 │ 168291 │2380990 │  12.19  │  8.75   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │    9%    │  10%   │   4%   │  66%   │   9%   │   66%   │    9%     │  0.0   │  43%   │  36%   │  55%   │  33%   │   52%   │   60%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    2     │    Avg     │        │        │ 5250.56  │ 26.30  │ 199.64 │3490675 │27201806│230596578│41183428753│   0    │  405   │2560397 │ 227649 │2690396 │  15.33  │  10.11  │
> ├──────────┼────────────┤ 64000  │   4    ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │          │    %SD     │        │        │   34%    │  11%   │  50%   │  23%   │  34%   │   23%   │    34%    │  0.0   │  49%   │  37%   │  39%   │  35%   │   76%   │   25%   │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   %Diff    │        │        │ +30.544% │+5.538% │+23.693%│+70.176%│+30.525%│+70.095% │ +30.525%  │  +0.0  │-2.174% │+11.919%│+35.271%│+12.995%│+25.759% │+15.543% │
> │          │between Avg │        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> ├──────────┼────────────┤        │        ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │Significance│        │        │  0.832   │ 0.553  │ 0.667  │ 0.924  │ 0.832  │  0.924  │   0.832   │ 0.653  │ 0.054  │ 0.360  │ 0.669  │ 0.412  │  0.609  │  0.132  │
> ├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
> │    -     │   Total    │        │        │  0.245   │ 0.576  │ 0.181  │ 0.146  │ 0.245  │  0.146  │   0.245   │ 0.661  │ 0.597  │ 0.425  │ 0.161  │ 0.421  │  0.102  │  0.011  │
> │          │Significance│        │        │          │        │        │        │        │         │           │        │        │        │        │        │         │         │
> └──────────┴────────────┴────────┴────────┴──────────┴────────┴────────┴────────┴────────┴─────────┴───────────┴────────┴────────┴────────┴────────┴────────┴─────────┴─────────┘
> 
> ┌────────────────────────────┬────────┬──────────┬────────┬────────┬────────┬────────┬───────────┬───────────┬────────┬──────────┬────────┬────────┬────────┬─────────┬─────────┐
> │      Category:TCP_RR       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┬────────────┬───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   #   │    Tile    │ size  │sessions│throughput│  %CPU  │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts  │ #rx-byts  │   #    │ #tx-intr │#rx-intr│#io_exit│#irq_inj│ #tpkt/# │ #rpkt/# │
> │       │            │       │        │          │        │        │        │        │           │           │re-trans│          │        │        │        │  exit   │   irq   │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │ 54076.24 │ 22.51  │2402.32 │9431729 │9866906 │13601026829│13629730150│   12   │   143    │ 986607 │ 515743 │1044580 │  18.29  │  9.45   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   15%    │  11%   │   7%   │  16%   │  13%   │    15%    │    15%    │  31%   │   16%    │  71%   │  27%   │  66%   │   38%   │   98%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │ 36935.31 │ 20.97  │1761.34 │6396858 │7372127 │9286901282 │9351261832 │   5    │    97    │ 247655 │ 148899 │ 287270 │  42.96  │  25.66  │
> ├───────┼────────────┤ 4000  │   50   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   20%    │   6%   │  17%   │  19%   │  14%   │    20%    │    20%    │  55%   │   19%    │  50%   │  27%   │  43%   │   7%    │   42%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -31.698% │-6.841% │-26.682%│-32.177%│-25.284%│ -31.719%  │ -31.391%  │-58.333%│ -32.168% │-74.898%│-71.129%│-72.499%│+134.882%│+171.534%│
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  -0.992  │ 0.753  │ -0.997 │ -0.992 │ -0.992 │  -0.992   │  -0.992   │ -0.994 │  -0.992  │ -0.952 │ -0.999 │ -0.957 │ +1.000  │ +0.955  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │ 62327.45 │ 23.75  │2624.31 │10752984│11055030│15668725680│15688647646│   10   │   164    │1356548 │ 211331 │1416435 │  50.88  │  7.80   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   13%    │  10%   │   7%   │  14%   │  11%   │    13%    │    13%    │  67%   │   14%    │  45%   │  82%   │  43%   │   66%   │  109%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │ 42794.83 │ 22.66  │1888.56 │7308678 │8905297 │10753562439│10858930007│   8    │   112    │ 257954 │ 135412 │ 308370 │  53.97  │  28.88  │
> ├───────┼────────────┤ 4000  │  100   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   11%    │   2%   │  12%   │  10%   │   5%   │    11%    │    11%    │  78%   │   11%    │  55%   │  46%   │  46%   │   43%   │   43%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -31.339% │-4.589% │-28.036%│-32.031%│-19.446%│ -31.369%  │ -30.785%  │-20.000%│ -31.707% │-80.985%│-35.924%│-78.229%│ +6.073% │+270.256%│
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  -0.998  │ 0.668  │ -0.999 │ -0.998 │ -0.994 │  -0.998   │  -0.998   │ 0.295  │  -0.998  │ -0.996 │ 0.614  │ -0.996 │  0.406  │ +0.990  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │ 14703.32 │  5.10  │2883.00 │2460591 │2605664 │3691923448 │3701796248 │   0    │    37    │ 139516 │ 16276  │ 154010 │ 151.18  │  16.92  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   224%   │  213%  │  46%   │  224%  │  224%  │   224%    │   224%    │  0.0   │   226%   │  223%  │  220%  │  220%  │   44%   │   44%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │ 9936.94  │  4.98  │1995.37 │1668907 │2013806 │2495739547 │2518802907 │   0    │   1961   │ 222197 │  4433  │ 236784 │ 376.47  │  8.50   │
> ├───────┼────────────┤ 4000  │  250   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   224%   │  212%  │  47%   │  223%  │  223%  │   224%    │   223%    │  0.0   │   224%   │  223%  │  211%  │  221%  │   46%   │   43%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -32.417% │-2.353% │-30.788%│-32.175%│-22.714%│ -32.400%  │ -31.957%  │  +0.0  │+5200.000%│+59.263%│-72.764%│+53.746%│+149.021%│-49.764% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.205   │ 0.013  │ 0.192  │ 0.203  │ 0.138  │   0.205   │   0.202   │ 0.653  │  0.645   │ 0.239  │ 0.505  │ 0.226  │  0.422  │  0.338  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   3.42   │  0.26  │ 13.15  │  836   │  1078  │  1103860  │  1490637  │   0    │    0     │   97   │  260   │  2968  │  3.22   │  0.36   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   15%    │   8%   │   7%   │  13%   │  12%   │    14%    │    11%    │  0.0   │   0.0    │   6%   │   7%   │  18%   │   10%   │   18%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   1.56   │  0.27  │  5.78  │  900   │  1175  │  1190908  │  1613970  │   0    │    0     │  107   │  259   │  2702  │  3.47   │  0.43   │
> ├───────┼────────────┤ 4000  │  500   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   137%   │   4%   │  142%  │  20%   │  21%   │    22%    │    21%    │  0.0   │   0.0    │  15%   │   9%   │   4%   │   16%   │   19%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -54.386% │+3.846% │-56.046%│+7.656% │+8.998% │  +7.886%  │  +8.274%  │  +0.0  │   +0.0   │+10.309%│-0.385% │-8.962% │ +7.764% │+19.444% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.905   │ 0.525  │ 0.905  │ 0.483  │ 0.545  │   0.457   │   0.508   │ 0.653  │  0.653   │ 0.756  │ 0.059  │ 0.701  │  0.585  │  0.778  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │ 6034.32  │ 10.75  │ 561.33 │5153363 │5403272 │7582137362 │7598857558 │   1    │   6921   │ 715462 │ 15370  │ 747743 │ 335.29  │  7.23   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   137%   │  134%  │  56%   │  137%  │  137%  │   137%    │   137%    │  195%  │   220%   │  137%  │  152%  │  137%  │   75%   │   52%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │ 4639.39  │  9.59  │ 483.77 │3959334 │4172310 │5829152509 │5843461595 │   1    │  12782   │ 509659 │  4209  │ 542761 │ 940.68  │  7.69   │
> ├───────┼────────────┤ 20000 │   50   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   137%   │  133%  │  55%   │  137%  │  137%  │   137%    │   137%    │  179%  │   137%   │  137%  │  130%  │  136%  │   56%   │   52%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -23.117% │-10.791%│-13.817%│-23.170%│-22.782%│ -23.120%  │ -23.101%  │+0.000% │ +84.684% │-28.765%│-72.615%│-27.413%│+180.557%│ +6.362% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.228   │ 0.104  │ 0.125  │ 0.228  │ 0.224  │   0.228   │   0.227   │ 0.130  │  0.412   │ 0.288  │ 0.671  │ 0.274  │  0.585  │  0.066  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.58   │  0.26  │  2.23  │  888   │  1175  │  1193268  │  1657788  │   0    │    0     │   97   │  246   │  2711  │  3.61   │  0.43   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   60%    │   6%   │  59%   │  18%   │  18%   │    18%    │    18%    │  0.0   │   0.0    │  19%   │   7%   │   3%   │   12%   │   16%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.63   │  0.26  │  2.42  │  941   │  1220  │  1272993  │  1722531  │   0    │    0     │   92   │  238   │  3029  │  3.95   │  0.40   │
> ├───────┼────────────┤ 20000 │  100   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   57%    │   7%   │  59%   │  12%   │  11%   │    12%    │    12%    │  0.0   │   0.0    │  11%   │   3%   │  17%   │   9%    │   13%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ +8.621%  │+0.000% │+8.520% │+5.968% │+3.830% │  +6.681%  │  +3.905%  │  +0.0  │   +0.0   │-5.155% │-3.252% │+11.730%│ +9.418% │ -6.977% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.184   │ 0.144  │ 0.241  │ 0.444  │ 0.296  │   0.473   │   0.295   │ 0.653  │  0.653   │ 0.346  │ 0.654  │ 0.779  │  0.819  │  0.507  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.85   │  0.26  │  3.27  │  1031  │  1350  │  1391207  │  1920218  │   0    │    0     │  108   │  249   │  3066  │  4.14   │  0.44   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │    8%    │   6%   │  13%   │   6%   │   6%   │    6%     │    7%     │  0.0   │   0.0    │   8%   │   4%   │  21%   │   4%    │   18%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.15   │  0.26  │  0.58  │  893   │  1173  │  1209894  │  1641494  │   0    │    0     │   97   │  233   │  2666  │  3.83   │  0.44   │
> ├───────┼────────────┤ 20000 │  250   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   229%   │   5%   │  237%  │  24%   │  24%   │    25%    │    25%    │  0.0   │   0.0    │  16%   │   7%   │   2%   │   17%   │   22%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -82.353% │+0.000% │-82.263%│-13.385%│-13.111%│ -13.033%  │ -14.515%  │  +0.0  │   +0.0   │-10.185%│-6.426% │-13.046%│ -7.488% │ +0.000% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  -0.998  │ 0.479  │ -0.997 │ 0.800  │ 0.786  │   0.763   │   0.812   │ 0.653  │  0.653   │ 0.779  │ 0.906  │ 0.794  │  0.705  │  0.193  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.69   │  0.26  │  2.65  │  1044  │  1377  │  1410784  │  1952436  │   0    │    0     │  109   │  257   │  3026  │  4.06   │  0.46   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   59%    │   6%   │  62%   │  11%   │  13%   │    12%    │    13%    │  0.0   │   0.0    │   9%   │   5%   │  18%   │   8%    │   8%    │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.46   │  0.27  │  1.70  │  899   │  1179  │  1216410  │  1649146  │   0    │    0     │   98   │  233   │  2717  │  3.86   │  0.43   │
> ├───────┼────────────┤ 20000 │  500   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   94%    │   6%   │  95%   │  14%   │  14%   │    15%    │    15%    │  0.0   │   0.0    │  13%   │   5%   │   4%   │   9%    │   14%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -33.333% │+3.846% │-35.849%│-13.889%│-14.379%│ -13.778%  │ -15.534%  │  +0.0  │   +0.0   │-10.092%│-9.339% │-10.212%│ -4.926% │ -6.522% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.589   │ 0.653  │ 0.654  │ 0.900  │ 0.889  │   0.881   │   0.908   │ 0.653  │  0.653   │ 0.848  │ -0.988 │ 0.755  │  0.644  │  0.541  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.24   │  0.26  │  0.92  │  961   │  1231  │  1273726  │  1756207  │   0    │    0     │  100   │  247   │  3022  │  3.89   │  0.41   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   92%    │   5%   │  93%   │  12%   │  15%   │    15%    │    15%    │  0.0   │   0.0    │  13%   │   5%   │  21%   │   15%   │   25%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.15   │  0.27  │  0.56  │  920   │  1178  │  1218570  │  1665604  │   0    │    0     │   87   │  224   │  2657  │  4.11   │  0.44   │
> ├───────┼────────────┤ 40000 │   50   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   137%   │   6%   │  145%  │  12%   │  11%   │    11%    │    12%    │  0.0   │   0.0    │   6%   │   4%   │   2%   │   12%   │   12%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -37.500% │+3.846% │-39.130%│-4.266% │-4.305% │  -4.330%  │  -5.159%  │  +0.0  │   +0.0   │-13.000%│-9.312% │-12.078%│ +5.656% │ +7.317% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.466   │ 0.677  │ 0.456  │ 0.410  │ 0.390  │   0.393   │   0.440   │ 0.653  │  0.653   │ 0.919  │ -0.991 │ 0.763  │  0.435  │  0.343  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.21   │  0.26  │  0.81  │  869   │  1150  │  1165925  │  1635938  │   0    │    0     │   95   │  240   │  2681  │  3.62   │  0.43   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   91%    │   7%   │  92%   │   8%   │   8%   │    9%     │    8%     │  0.0   │   0.0    │   6%   │   3%   │   2%   │   7%    │   7%    │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.13   │  0.26  │  0.50  │  932   │  1216  │  1269201  │  1724294  │   0    │    0     │   86   │  229   │  2771  │  4.07   │  0.44   │
> ├───────┼────────────┤ 40000 │  100   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   138%   │   7%   │  134%  │  14%   │  13%   │    15%    │    13%    │  0.0   │   0.0    │  11%   │   6%   │   4%   │   12%   │   14%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -38.095% │+0.000% │-38.272%│+7.250% │+5.739% │  +8.858%  │  +5.401%  │  +0.0  │   +0.0   │-9.474% │-4.583% │+3.357% │+12.431% │ +2.326% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.485   │ 0.269  │ 0.521  │ 0.627  │ 0.562  │   0.686   │   0.518   │ 0.653  │  0.653   │ 0.883  │ 0.834  │ 0.847  │  0.893  │  0.342  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.37   │  0.25  │  1.48  │  939   │  1204  │  1247126  │  1708161  │   0    │    0     │   88   │  243   │  2975  │  3.86   │  0.40   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   11%    │   6%   │   8%   │  12%   │  11%   │    12%    │    11%    │  0.0   │   0.0    │  11%   │   9%   │  22%   │   8%    │   15%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.14   │  0.26  │  0.54  │  919   │  1194  │  1232652  │  1689413  │   0    │    0     │   87   │  229   │  2766  │  4.01   │  0.43   │
> ├───────┼────────────┤ 40000 │  250   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   133%   │  11%   │  127%  │   7%   │   8%   │    7%     │    8%     │  0.0   │   0.0    │   7%   │   4%   │   3%   │   4%    │   12%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -62.162% │+4.000% │-63.514%│-2.130% │-0.831% │  -1.161%  │  -1.098%  │  +0.0  │   +0.0   │-1.136% │-5.761% │-7.025% │ +3.886% │ +7.500% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  -0.974  │ 0.709  │ -0.987 │ 0.259  │ 0.107  │   0.141   │   0.138   │ 0.653  │  0.653   │ 0.241  │ 0.774  │ 0.507  │  0.609  │  0.452  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.22   │  0.26  │  0.85  │  928   │  1207  │  1250884  │  1717862  │   0    │    0     │   90   │  233   │  2712  │  3.98   │  0.45   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   97%    │   5%   │  96%   │  21%   │  21%   │    22%    │    21%    │  0.0   │   0.0    │  17%   │   8%   │   2%   │   17%   │   19%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.31   │  0.26  │  1.19  │  969   │  1293  │  1314718  │  1839094  │   0    │    0     │   92   │  235   │  2756  │  4.12   │  0.47   │
> ├───────┼────────────┤ 40000 │  500   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   60%    │   7%   │  62%   │  18%   │  18%   │    19%    │    18%    │  0.0   │   0.0    │  13%   │   5%   │   5%   │   14%   │   20%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ +40.909% │+0.000% │+40.000%│+4.418% │+7.125% │  +5.103%  │  +7.057%  │  +0.0  │   +0.0   │+2.222% │+0.858% │+1.622% │ +3.518% │ +4.444% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.504   │ 0.000  │ 0.558  │ 0.270  │ 0.417  │   0.291   │   0.401   │ 0.653  │  0.653   │ 0.192  │ 0.136  │ 0.473  │  0.275  │  0.363  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.24   │  0.26  │  0.92  │  1000  │  1293  │  1357674  │  1840914  │   0    │    0     │   88   │  235   │  2741  │  4.26   │  0.47   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   14%    │   8%   │  12%   │  16%   │  14%   │    17%    │    14%    │  0.0   │   0.0    │   6%   │   5%   │   3%   │   12%   │   13%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.21   │  0.26  │  0.81  │  846   │  1111  │  1126596  │  1575802  │   0    │    0     │   82   │  219   │  2776  │  3.86   │  0.40   │
> ├───────┼────────────┤ 64000 │   50   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   22%    │   3%   │  25%   │  21%   │  21%   │    23%    │    21%    │  0.0   │   0.0    │  16%   │   5%   │   2%   │   16%   │   22%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -12.500% │+0.000% │-11.957%│-15.400%│-14.076%│ -17.020%  │ -14.401%  │  +0.0  │   +0.0   │-6.818% │-6.809% │+1.277% │ -9.390% │-14.894% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.782   │ 0.290  │ 0.789  │ 0.820  │ 0.797  │   0.822   │   0.804   │ 0.653  │  0.653   │ 0.617  │ 0.923  │ 0.537  │  0.721  │  0.810  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.20   │  0.26  │  0.77  │  841   │  1091  │  1110570  │  1545252  │   0    │    0     │   81   │  225   │  3017  │  3.74   │  0.36   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   12%    │   7%   │  16%   │  12%   │  10%   │    15%    │    11%    │  0.0   │   0.0    │   3%   │   5%   │  19%   │   11%   │   23%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.19   │  0.27  │  0.70  │  1035  │  1343  │  1399732  │  1913629  │   0    │    0     │   95   │  233   │  2760  │  4.44   │  0.49   │
> ├───────┼────────────┤ 64000 │  100   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   57%    │   7%   │  62%   │  19%   │  18%   │    21%    │    19%    │  0.0   │   0.0    │  10%   │   5%   │   2%   │   15%   │   18%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -5.000%  │+3.846% │-9.091% │+23.068%│+23.098%│ +26.037%  │ +23.839%  │  +0.0  │   +0.0   │+17.284%│+3.556% │-8.518% │+18.717% │+36.111% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.184   │ 0.660  │ 0.293  │ 0.912  │ 0.929  │   0.910   │   0.927   │ 0.653  │  0.653   │ +0.987 │ 0.696  │ 0.657  │  0.917  │  0.933  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.26   │  0.26  │  1.00  │  1061  │  1380  │  1424087  │  1970910  │   0    │    0     │   99   │  237   │  2777  │  4.48   │  0.50   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   15%    │   7%   │  21%   │  12%   │  13%   │    13%    │    14%    │  0.0   │   0.0    │  14%   │   6%   │   3%   │   7%    │   13%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.25   │  0.26  │  0.96  │  990   │  1337  │  1338042  │  1903858  │   0    │    0     │   89   │  227   │  3262  │  4.36   │  0.41   │
> ├───────┼────────────┤ 64000 │  250   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   13%    │   9%   │  18%   │  12%   │  12%   │    13%    │    12%    │  0.0   │   0.0    │  14%   │   4%   │  24%   │   15%   │   32%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -3.846%  │+0.000% │-4.000% │-6.692% │-3.116% │  -6.042%  │  -3.402%  │  +0.0  │   +0.0   │-10.101%│-4.219% │+17.465%│ -2.679% │-18.000% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.206   │ 0.114  │ 0.190  │ 0.607  │ 0.304  │   0.536   │   0.313   │ 0.653  │  0.653   │ 0.759  │ 0.776  │ 0.793  │  0.198  │  0.627  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   1   │    Avg     │       │        │   0.19   │  0.26  │  0.73  │  899   │  1188  │  1186040  │  1686630  │   0    │    0     │   93   │  236   │  2711  │  3.81   │  0.44   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   64%    │   8%   │  61%   │  30%   │  30%   │    31%    │    32%    │  0.0   │   0.0    │  16%   │  10%   │   6%   │   24%   │   29%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   2   │    Avg     │       │        │   0.15   │  0.24  │  0.62  │  845   │  1106  │  1129531  │  1560339  │   0    │    0     │   82   │  220   │  2947  │  3.84   │  0.38   │
> ├───────┼────────────┤ 64000 │  500   ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │       │    %SD     │       │        │   65%    │  11%   │  71%   │  28%   │  29%   │    30%    │    29%    │  0.0   │   0.0    │  17%   │   8%   │  17%   │   22%   │   36%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   %Diff    │       │        │ -21.053% │-7.692% │-15.068%│-6.007% │-6.902% │  -4.765%  │  -7.488%  │  +0.0  │   +0.0   │-11.828%│-6.780% │+8.705% │ +0.787% │-13.636% │
> │       │between Avg │       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │Significance│       │        │  0.434   │ 0.721  │ 0.175  │ 0.256  │ 0.285  │   0.193   │   0.299   │ 0.653  │  0.653   │ 0.744  │ 0.766  │ 0.664  │  0.039  │  0.446  │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
> │   -   │   Total    │       │        │  0.893   │ 0.921  │ 0.927  │ 0.923  │ 0.933  │   0.925   │   0.925   │ 0.773  │  0.776   │ 0.850  │ 0.775  │ 0.851  │  0.808  │  0.692  │
> │       │Significance│       │        │          │        │        │        │        │           │           │        │          │        │        │        │         │         │
> └───────┴────────────┴───────┴────────┴──────────┴────────┴────────┴────────┴────────┴───────────┴───────────┴────────┴──────────┴────────┴────────┴────────┴─────────┴─────────┘
> 
> ┌────────────────────────────┬────────┬──────────┬────────┬────────┬────────┬────────┬───────────┬────────┬─────────┬────────┬────────┬─────────┬────────┬─────────┬────────┐
> │    Category:TCP_MAERTS     │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┬────────────┬───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   #   │    Tile    │ size  │sessions│throughput│  %CPU  │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts  │#rx-byts│#re-trans│#tx-intr│#rx-intr│#io_exit │#irq_inj│ #tpkt/# │#rpkt/# │
> │       │            │       │        │          │        │        │        │        │           │        │         │        │        │         │        │  exit   │  irq   │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 4025.16  │ 22.84  │ 176.23 │20851157│ 702802 │31567456210│46385250│    0    │ 87079  │ 62340  │  1193   │ 183511 │17477.92 │  3.83  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │   2%   │   2%   │   2%   │   5%   │    2%     │   5%   │   0.0   │   1%   │   3%   │   6%    │   3%   │   6%    │   3%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3044.14  │ 20.50  │ 148.49 │15771221│ 598877 │23874038498│39526212│    1    │ 67657  │ 26289  │  1115   │ 115186 │14144.59 │  5.20  │
> ├───────┼────────────┤ 4000  │   1    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    3%    │ 0.83%  │   3%   │   3%   │   1%   │    3%     │   1%   │  134%   │   2%   │   8%   │   8%    │   3%   │   6%    │   3%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -24.372% │-10.245%│-15.741%│-24.363%│-14.787%│ -24.371%  │-14.787%│  +0.0   │-22.304%│-57.830%│ -6.538% │-37.232%│-19.072% │+35.770%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  -1.000  │ -1.000 │ -1.000 │ -1.000 │ -1.000 │  -1.000   │ -1.000 │  0.633  │ -1.000 │ -1.000 │  0.821  │ -1.000 │ -1.000  │ +1.000 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3958.83  │ 23.40  │ 169.18 │20508673│ 854697 │31048301902│56410579│    1    │ 85430  │ 59633  │   621   │ 181895 │33025.24 │  4.70  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │   2%   │   2%   │   2%   │   2%   │    2%     │   2%   │  167%   │   2%   │   2%   │   7%    │   3%   │   8%    │   2%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3562.72  │ 20.90  │ 170.47 │18458461│ 857508 │27942035427│56596116│    0    │ 78196  │ 31911  │  1891   │ 136169 │ 9761.22 │  6.30  │
> ├───────┼────────────┤ 4000  │   2    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │   37%    │   2%   │  39%   │  37%   │  21%   │    37%    │  21%   │   0.0   │  34%   │  35%   │  113%   │  34%   │   71%   │  10%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -10.006% │-10.684%│+0.763% │-9.997% │+0.329% │ -10.005%  │+0.329% │-100.000%│-8.468% │-46.488%│+204.509%│-25.139%│-70.443% │+34.043%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  0.481   │ -1.000 │ 0.052  │ 0.480  │ 0.027  │   0.481   │ 0.027  │  0.805  │ 0.439  │ -0.999 │  0.780  │ 0.940  │ -0.999  │ +1.000 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3901.40  │ 23.70  │ 164.62 │20211963│1267454 │30598575481│83653083│    0    │ 82354  │ 58493  │   629   │ 177376 │32133.49 │  7.15  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │   1%   │   2%   │   2%   │   3%   │    2%     │   3%   │   0.0   │   3%   │   2%   │   22%   │   3%   │   17%   │   6%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 2910.91  │ 21.58  │ 134.89 │15082039│1160029 │22831307937│76563081│    0    │ 63438  │ 29078  │   535   │ 116572 │28190.73 │  9.95  │
> ├───────┼────────────┤ 4000  │   4    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    3%    │   2%   │   2%   │   3%   │   3%   │    3%     │   3%   │   0.0   │   2%   │   4%   │   4%    │   3%   │   4%    │   5%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -25.388% │-8.945% │-18.060%│-25.381%│-8.476% │ -25.384%  │-8.475% │  +0.0   │-22.969%│-50.288%│-14.944% │-34.280%│-12.270% │+39.161%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  -1.000  │ -1.000 │ -1.000 │ -1.000 │ -0.997 │  -1.000   │ -0.997 │  0.653  │ -1.000 │ -1.000 │  0.838  │ -1.000 │  0.907  │ +1.000 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3990.42  │ 24.09  │ 165.65 │20671202│ 664534 │31295001508│43859558│    0    │ 85100  │ 62132  │  5200   │ 203512 │ 3975.23 │  3.27  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │  15%   │  13%   │   2%   │   6%   │    2%     │   6%   │   0.0   │   3%   │   4%   │  175%   │  28%   │  196%   │  23%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3064.89  │ 20.34  │ 150.68 │15881474│ 572534 │24036756364│37787582│    0    │ 67629  │ 26538  │  1091   │ 114112 │14556.80 │  5.02  │
> ├───────┼────────────┤ 20000 │   1    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │   2%   │   3%   │   2%   │ 0.29%  │    2%     │ 0.29%  │   0.0   │   2%   │   4%   │   4%    │   2%   │   4%    │   1%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -23.194% │-15.567%│-9.037% │-23.171%│-13.844%│ -23.193%  │-13.844%│  +0.0   │-20.530%│-57.288%│-79.019% │-43.929%│+266.188%│+53.517%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  -1.000  │ -0.953 │ 0.880  │ -1.000 │ -0.999 │  -1.000   │ -0.999 │  0.653  │ -1.000 │ -1.000 │  0.658  │ -0.992 │  0.067  │ +0.999 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3943.19  │ 22.83  │ 172.72 │20427150│ 842967 │30925722054│55636408│    0    │ 85108  │ 59335  │   842   │ 175785 │24260.27 │  4.80  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │   1%   │   3%   │   2%   │   4%   │    2%     │   4%   │   0.0   │   2%   │   2%   │   37%   │   2%   │   29%   │   4%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 2978.75  │ 20.37  │ 146.23 │15432261│ 804840 │23362657135│53120047│    0    │ 66416  │ 27420  │   772   │ 112360 │19989.98 │  7.16  │
> ├───────┼────────────┤ 20000 │   2    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    1%    │ 0.83%  │ 0.83%  │   1%   │   4%   │    1%     │   4%   │   0.0   │   1%   │   4%   │   14%   │   2%   │   12%   │   6%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -24.458% │-10.775%│-15.337%│-24.452%│-4.523% │ -24.456%  │-4.523% │  +0.0   │-21.963%│-53.788%│ -8.314% │-36.081%│-17.602% │+49.167%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  -1.000  │ -1.000 │ -1.000 │ -1.000 │ 0.915  │  -1.000   │ 0.915  │  0.653  │ -1.000 │ -1.000 │  0.350  │ -1.000 │  0.898  │ +1.000 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 4635.26  │ 23.14  │ 200.31 │24013988│1392787 │36354488763│91925099│    1    │ 97818  │ 69897  │  1217   │ 201387 │19732.12 │  6.92  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │   38%    │   1%   │  40%   │  38%   │  15%   │    38%    │  15%   │  303%   │  40%   │  38%   │   54%   │  38%   │   63%   │  17%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3502.37  │ 20.76  │ 168.71 │18146027│1274730 │27469437009│84133329│    0    │ 76870  │ 35159  │   702   │ 134911 │25849.04 │  9.45  │
> ├───────┼────────────┤ 20000 │   4    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │   39%    │ 0.50%  │  39%   │  39%   │  13%   │    39%    │  13%   │   0.0   │  39%   │  30%   │   55%   │  36%   │   11%   │  17%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -24.441% │-10.285%│-15.776%│-24.436%│-8.476% │ -24.440%  │-8.476% │-100.000%│-21.415%│-49.699%│-42.317% │-33.009%│+31.000% │+36.561%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  0.713   │ -1.000 │ 0.494  │ 0.712  │ 0.639  │   0.713   │ 0.639  │  0.558  │ 0.635  │ -0.973 │  0.833  │ 0.861  │  0.324  │ +0.982 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 4022.67  │ 22.09  │ 182.10 │20838443│ 685924 │31548235695│45271283│    0    │ 85465  │ 64027  │  1180   │ 179593 │17659.70 │  3.82  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │  0.91%   │ 0.59%  │   1%   │ 0.91%  │   4%   │   0.91%   │   4%   │   0.0   │   1%   │   3%   │   15%   │ 0.65%  │   14%   │   4%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3036.88  │ 20.27  │ 149.82 │15735787│ 570734 │23817415383│37668804│    0    │ 66697  │ 26397  │  1098   │ 112629 │14331.32 │  5.07  │
> ├───────┼────────────┤ 40000 │   1    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    1%    │   3%   │   3%   │   1%   │ 0.18%  │    1%     │ 0.18%  │   0.0   │   2%   │   5%   │   8%    │   2%   │   7%    │   2%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -24.506% │-8.239% │-17.727%│-24.487%│-16.793%│ -24.505%  │-16.793%│  +0.0   │-21.960%│-58.772%│ -6.949% │-37.287%│-18.847% │+32.723%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  -1.000  │ -1.000 │ -1.000 │ -1.000 │ -1.000 │  -1.000   │ -1.000 │  0.793  │ -1.000 │ -1.000 │  0.607  │ -1.000 │ -0.983  │ +1.000 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3959.74  │ 24.48  │ 161.75 │20513131│ 878602 │31055765278│57988350│    0    │ 85255  │ 59696  │  4569   │ 201114 │ 4489.63 │  4.37  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │  17%   │  14%   │   2%   │   4%   │    2%     │   4%   │   0.0   │   3%   │   2%   │  190%   │  31%   │  299%   │  21%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3612.20  │ 20.41  │ 176.98 │18715784│ 848316 │28331890132│55989502│    0    │ 79801  │ 32639  │  1210   │ 135034 │15467.59 │  6.28  │
> ├───────┼────────────┤ 40000 │   2    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │   38%    │   2%   │  39%   │  38%   │  19%   │    38%    │  19%   │   0.0   │  37%   │  36%   │   35%   │  36%   │   28%   │  13%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -8.777%  │-16.626%│+9.416% │-8.762% │-3.447% │  -8.771%  │-3.447% │  +0.0   │-6.397% │-45.325%│-73.517% │-32.857%│+244.518%│+43.707%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  0.416   │ 0.940  │ 0.291  │ 0.415  │ 0.302  │   0.415   │ 0.302  │  0.653  │ 0.307  │ -0.999 │  0.587  │ 0.901  │  0.774  │ +0.992 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3745.38  │ 22.95  │ 163.20 │19403080│1308077 │29375006522│86334201│    0    │ 78885  │ 56216  │   856   │ 161976 │22667.15 │  8.08  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │   1%   │   1%   │   2%   │   6%   │    2%     │   6%   │   0.0   │   1%   │   2%   │   78%   │   3%   │   50%   │   9%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3493.96  │ 20.61  │ 169.53 │18102335│1299683 │27404210257│85780238│    0    │ 76414  │ 34729  │   989   │ 133336 │18303.68 │  9.75  │
> ├───────┼────────────┤ 40000 │   4    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │   39%    │   1%   │  39%   │  39%   │  14%   │    39%    │  14%   │   0.0   │  39%   │  30%   │   66%   │  36%   │   46%   │  16%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -6.713%  │-10.196%│+3.879% │-6.704% │-0.642% │  -6.709%  │-0.642% │  +0.0   │-3.132% │-38.222%│+15.537% │-17.682%│-19.250% │+20.668%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  0.306   │ -1.000 │ 0.160  │ 0.306  │ 0.073  │   0.306   │ 0.073  │  0.653  │ 0.143  │ -0.998 │  0.243  │ 0.780  │  0.732  │ +0.975 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 4026.58  │ 22.20  │ 181.38 │20858458│ 686402 │31578612205│45302864│    0    │ 85706  │ 63340  │  1127   │ 178639 │18507.95 │  3.84  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    1%    │   2%   │   2%   │   1%   │   4%   │    1%     │   4%   │   0.0   │   2%   │   3%   │  0.81%  │   2%   │  0.96%  │   2%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3059.87  │ 20.17  │ 151.70 │15854408│ 571828 │23997011560│37740969│    0    │ 67554  │ 26765  │  1071   │ 113644 │14803.37 │  5.03  │
> ├───────┼────────────┤ 64000 │   1    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    3%    │ 0.33%  │   2%   │   3%   │ 0.43%  │    3%     │ 0.43%  │   0.0   │   3%   │   6%   │   3%    │   3%   │   2%    │   3%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -24.008% │-9.144% │-16.363%│-23.991%│-16.692%│ -24.009%  │-16.692%│  +0.0   │-21.179%│-57.744%│ -4.969% │-36.383%│-20.016% │+30.990%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  -1.000  │ -1.000 │ -1.000 │ -1.000 │ -1.000 │  -1.000   │ -1.000 │  0.653  │ -1.000 │ -1.000 │ -0.993  │ -1.000 │ -1.000  │ +1.000 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3947.77  │ 23.99  │ 164.56 │20450669│ 858583 │30961307511│56667066│    0    │ 84716  │ 59427  │  1769   │ 193705 │11560.58 │  4.43  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │  10%   │   9%   │   2%   │   2%   │    2%     │   2%   │   0.0   │   2%   │   2%   │  133%   │  22%   │  112%   │  19%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 3597.82  │ 20.57  │ 174.91 │18641202│ 855720 │28218060821│56478120│    0    │ 79043  │ 31734  │  1075   │ 133925 │17340.65 │  6.39  │
> ├───────┼────────────┤ 64000 │   2    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │   37%    │   2%   │  39%   │  37%   │  18%   │    37%    │  18%   │   0.0   │  35%   │  35%   │   65%   │  35%   │   20%   │  13%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -8.864%  │-14.256%│+6.289% │-8.848% │-0.333% │  -8.860%  │-0.333% │  +0.0   │-6.696% │-46.600%│-39.231% │-30.861%│+49.998% │+44.244%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  0.431   │ -0.988 │ 0.244  │ 0.430  │ 0.032  │   0.431   │ 0.032  │  0.653  │ 0.343  │ -0.999 │  0.455  │ 0.932  │  0.603  │ +0.995 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   1   │    Avg     │       │        │ 3773.21  │ 23.40  │ 161.25 │19547537│1317873 │29593764943│86980763│    0    │ 79685  │ 56543  │  3466   │ 174616 │ 5639.80 │  7.55  │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    2%    │   6%   │   7%   │   2%   │   5%   │    2%     │   5%   │   0.0   │   3%   │   2%   │  185%   │  15%   │  268%   │  14%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   2   │    Avg     │       │        │ 2924.85  │ 21.20  │ 137.96 │15153638│1154835 │22940791452│76220282│    0    │ 63344  │ 29439  │   547   │ 113758 │27703.18 │ 10.15  │
> ├───────┼────────────┤ 64000 │   4    ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │       │    %SD     │       │        │    4%    │   2%   │   3%   │   4%   │   2%   │    4%     │   2%   │   0.0   │   3%   │   4%   │   7%    │   3%   │   8%    │   3%   │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   %Diff    │       │        │ -22.484% │-9.402% │-14.443%│-22.478%│-12.371%│ -22.481%  │-12.371%│  +0.0   │-20.507%│-47.935%│-84.218% │-34.852%│+391.209%│+34.437%│
> │       │between Avg │       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> ├───────┼────────────┤       │        ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │Significance│       │        │  -1.000  │ -0.993 │ -0.998 │ -1.000 │ -0.999 │  -1.000   │ -0.999 │  0.653  │ -1.000 │ -1.000 │  0.662  │ -0.999 │  0.080  │ +0.999 │
> ├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
> │   -   │   Total    │       │        │  -1.000  │ -1.000 │ -0.985 │ -1.000 │ -0.999 │  -1.000   │ -0.999 │  0.414  │ -1.000 │ -1.000 │  0.909  │ -1.000 │  0.183  │ +1.000 │
> │       │Significance│       │        │          │        │        │        │        │           │        │         │        │        │         │        │         │        │
> └───────┴────────────┴───────┴────────┴──────────┴────────┴────────┴────────┴────────┴───────────┴────────┴─────────┴────────┴────────┴─────────┴────────┴─────────┴────────┘

> ┌───────┬────────────────────────┬──────────┬───────┬───────┬────────┬────────┬───────────┬───────────┬────────┬───────┬───────┬───────┬───────┬────────┬───────┐
> │== Raw │                        │          │       │       │        │        │           │           │        │       │       │       │       │        │       │
> │data of│  Category:TCP_STREAM   │          │       │       │        │        │           │           │        │       │       │       │       │        │       │
> │sample │                        │          │       │       │        │        │           │           │        │       │       │       │       │        │       │
> │1 ==   │                        │          │       │       │        │        │           │           │        │       │       │       │       │        │       │
> ├───────┼───────┬───────┬────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   #   │ Tile  │ size  │sessions│throughput│ %CPU  │ thr/  │#tx-pkts│#rx-pkts│ #tx-byts  │ #rx-byts  │   #    │   #   │   #   │   #   │   #   │#tpkt/# │#rpkt/#│
> │       │       │       │        │          │       │ %CPU  │        │        │           │           │re-trans│tx-intr│rx-intr│io_exit│irq_inj│  exit  │  irq  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2914.69  │ 16.58 │175.80 │ 946015 │15098314│ 62478018  │22858578760│   0    │  14   │190786 │218640 │248844 │  4.33  │ 60.67 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3846.89  │ 23.30 │165.10 │3384334 │19928213│ 223561788 │30171286582│   0    │  51   │3106036│289974 │3167495│ 11.67  │ 6.29  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3829.48  │ 23.28 │164.49 │3633348 │19837462│ 239959917 │30033885522│   0    │  55   │3126068│291581 │3186759│ 12.46  │ 6.22  │
> ├───────┼───────┤ 4000  │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3852.38  │ 24.17 │159.39 │3208800 │19956361│ 211964196 │30213902432│   0    │  48   │2923113│295547 │2983047│ 10.86  │ 6.69  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3826.82  │ 24.18 │158.26 │2897483 │19823797│ 191391882 │30013209670│   0    │  44   │3008981│294786 │3068148│  9.83  │ 6.46  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3654.05  │ 22.30 │163.86 │2813996 │18928829│ 185871160 │28658172593│   0    │  42   │2470996│278105 │2530858│ 10.12  │ 7.48  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3150.22  │ 23.54 │133.82 │2801597 │16321786│ 185024490 │24710811662│   0    │  99   │3361056│ 46502 │3418283│ 60.25  │ 4.77  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3883.19  │ 20.74 │187.23 │3158250 │20118341│ 208596880 │30459134356│   0    │  48   │663771 │327600 │741974 │  9.64  │ 27.11 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4561.23  │ 24.63 │185.19 │3084901 │23628392│ 203720814 │35773349852│   0    │  47   │2617111│288865 │2686249│ 10.68  │ 8.80  │
> ├───────┼───────┤ 4000  │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4594.03  │ 23.19 │198.10 │1751767 │23798542│ 115716366 │36030958376│   0    │  27   │2548977│309503 │2618148│  5.66  │ 9.09  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3577.93  │ 23.25 │153.89 │2320804 │18537890│ 153295032 │28065751886│   0    │  49   │2840032│ 61399 │2898121│ 37.80  │ 6.40  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3953.32  │ 23.07 │171.36 │2623463 │20480990│ 173270716 │31008001226│   0    │  54   │2406189│206773 │2472555│ 12.69  │ 8.28  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3612.79  │ 21.11 │171.14 │2130367 │18718623│ 140797650 │28339815704│   0    │  390  │1288863│166504 │1365057│ 12.79  │ 13.71 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4460.59  │ 30.36 │146.92 │4553323 │23113273│ 300752346 │34993430332│   0    │  555  │3224635│140695 │3382961│ 32.36  │ 6.83  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4789.71  │ 28.39 │168.71 │4434046 │24814124│ 292821684 │37568519884│   0    │  387  │2232387│200883 │2350894│ 22.07  │ 10.56 │
> ├───────┼───────┤ 4000  │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4459.86  │ 23.46 │190.10 │1923225 │23105224│ 127036770 │34981220044│   0    │  330  │1842177│174892 │1912146│ 11.00  │ 12.08 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4390.76  │ 27.12 │161.90 │2389842 │22746683│ 157880400 │34438414736│   0    │  184  │2847848│185822 │2952908│ 12.86  │ 7.70  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4342.74  │ 26.09 │166.45 │3086160 │22499585│ 203857770 │34064280140│   0    │  369  │2287182│173759 │2392793│ 17.76  │ 9.40  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3841.00  │ 24.14 │159.11 │3093078 │19897547│ 204321312 │30124855328│   0    │  47   │3000038│286674 │3060205│ 10.79  │ 6.50  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3852.77  │ 24.42 │157.77 │3113317 │19958336│ 205666302 │30216890906│   0    │  47   │3239480│276424 │3299678│ 11.26  │ 6.05  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4876.24  │ 21.92 │222.46 │1827268 │25258600│ 120730740 │38241409172│   0    │  28   │1700264│264050 │1762228│  6.92  │ 14.33 │
> ├───────┼───────┤ 20000 │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4813.63  │ 21.68 │222.03 │2043971 │24934583│ 135005358 │37750924418│   0    │  31   │1404942│262419 │1464728│  7.79  │ 17.02 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4828.61  │ 21.34 │226.27 │1267470 │25011728│ 83716116  │37867730818│   0    │  19   │1594942│276285 │1655990│  4.59  │ 15.10 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4442.45  │ 22.70 │195.70 │2269020 │23012158│ 149887965 │34840362128│   0    │  34   │2187933│273170 │2248565│  8.31  │ 10.23 │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4614.01  │ 23.50 │196.34 │2236705 │23901627│ 147759066 │36187019896│   0    │  34   │2619512│281045 │2687252│  7.96  │ 8.89  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3012.82  │ 22.15 │136.02 │2290965 │15612079│ 151308010 │23636008424│   0    │  151  │2677904│ 83015 │2738943│ 27.60  │ 5.70  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4625.91  │ 24.23 │190.92 │2504936 │23963357│ 165589248 │36280488718│   0    │  38   │2723487│284637 │2791688│  8.80  │ 8.58  │
> ├───────┼───────┤ 20000 │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4140.85  │ 25.07 │165.17 │2477142 │21450701│ 163668984 │32476328990│   0    │  37   │3598697│303891 │3665244│  8.15  │ 5.85  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 5579.91  │ 16.52 │337.77 │3429251 │28914014│ 226378602 │43772464000│   0    │  52   │4707622│267324 │4812961│ 12.83  │ 6.01  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4394.70  │ 22.29 │197.16 │2587799 │22768355│ 170940782 │34470462005│   0    │  62   │3265444│243982 │3339217│ 10.61  │ 6.82  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3963.13  │ 24.37 │162.62 │1576022 │20535988│ 104202552 │31091368626│   0    │  516  │2566835│ 82331 │2635098│ 19.14  │ 7.79  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 7814.37  │ 26.41 │295.89 │7584132 │40484304│ 500952100 │61293157172│   0    │  119  │4694704│644533 │4901091│ 11.77  │ 8.26  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4264.56  │ 24.03 │177.47 │1736402 │22095527│ 114811464 │33452533114│   0    │  338  │2558984│146947 │2628159│ 11.82  │ 8.41  │
> ├───────┼───────┤ 20000 │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4394.78  │ 26.52 │165.72 │3126071 │22768346│ 206505030 │34471201720│   0    │  457  │2745724│178618 │2852158│ 17.50  │ 7.98  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4037.63  │ 24.32 │166.02 │1390213 │20920303│ 91933970  │31673216296│   0    │  394  │2774686│145251 │2844219│  9.57  │ 7.36  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4894.89  │ 25.13 │194.78 │3082568 │25360893│ 203681023 │38396295385│   0    │  364  │3068186│239536 │3172145│ 12.87  │ 7.99  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4696.12  │ 21.45 │218.93 │1746717 │24325944│ 115428234 │36829461018│   0    │  27   │1412919│299272 │1473390│  5.84  │ 16.51 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4857.96  │ 21.85 │222.33 │1877611 │25163730│ 124052082 │38097825926│   0    │  29   │1642255│281181 │1702943│  6.68  │ 14.78 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3834.49  │ 24.77 │154.80 │3536398 │19863734│ 233871300 │30073660456│   0    │  54   │3141635│306823 │3202110│ 11.53  │ 6.20  │
> ├───────┼───────┤ 40000 │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2877.93  │ 16.28 │176.78 │ 965972 │14907628│ 63803040  │22570131676│   0    │  14   │180473 │254054 │238407 │  3.80  │ 62.53 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3796.74  │ 24.57 │154.53 │3411520 │19668337│ 225604860 │29777845948│   0    │  52   │2949421│316357 │3009483│ 10.78  │ 6.54  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4012.65  │ 21.78 │184.24 │2307643 │20785874│ 152551903 │31469785004│   0    │  35   │1865340│291537 │1925266│  7.92  │ 10.80 │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4163.62  │ 24.39 │170.71 │2486527 │21568715│ 164397114 │32654995240│   0    │  38   │3494130│304520 │3560295│  8.17  │ 6.06  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 6479.87  │ 22.83 │283.83 │4907786 │33573135│ 324164328 │50826111556│   0    │  75   │7684977│336261 │7792528│ 14.60  │ 4.31  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4168.32  │ 24.97 │166.93 │1769613 │21592962│ 116901894 │32691709288│   0    │  27   │3977691│295565 │4044685│  5.99  │ 5.34  │
> ├───────┼───────┤ 40000 │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3192.35  │ 23.53 │135.67 │2660256 │16541052│ 175727580 │25043009116│   0    │  102  │3218560│ 44895 │3276488│ 59.26  │ 5.05  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4130.34  │ 22.16 │186.39 │2259249 │21396424│ 149362854 │32394150018│   0    │  34   │3549994│303642 │3615945│  7.44  │ 5.92  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4426.90  │ 23.58 │187.74 │2816686 │22934457│ 186110754 │34721995043│   0    │  55   │4385070│256976 │4457988│ 10.96  │ 5.14  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3722.13  │ 22.56 │164.99 │1484605 │19286605│ 98125982  │29199830180│   0    │  306  │2041127│172603 │2110311│  8.60  │ 9.14  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4075.12  │ 26.43 │154.19 │4216004 │21114946│ 278471452 │31967964016│   0    │  241  │1367095│293206 │1527824│ 14.38  │ 13.82 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4185.80  │ 23.41 │178.80 │1417592 │21688358│ 93727364  │32836027678│   0    │  358  │2554985│149408 │2623999│  9.49  │ 8.27  │
> ├───────┼───────┤ 40000 │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4851.13  │ 29.52 │164.33 │4261781 │25135249│ 281515090 │38054707246│   0    │  250  │1267474│264201 │1437691│ 16.13  │ 17.48 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4195.77  │ 24.10 │174.10 │1600252 │21738536│ 105798300 │32912018618│   0    │  387  │2340807│149572 │2410218│ 10.70  │ 9.02  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4205.99  │ 25.20 │166.90 │2596046 │21792738│ 171527637 │32994109547│   0    │  308  │1914297│205798 │2022008│ 12.61  │ 10.78 │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3863.66  │ 23.99 │161.05 │3050872 │20014608│ 201627408 │30302088890│   0    │  46   │2978200│287857 │3038123│ 10.60  │ 6.59  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4882.87  │ 21.95 │222.45 │1859234 │25293303│ 122793816 │38294025762│   0    │  28   │1648954│258666 │1709509│  7.19  │ 14.80 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3861.97  │ 23.90 │161.59 │3245206 │20005782│ 214361940 │30288737126│   0    │  49   │3283337│284083 │3344212│ 11.42  │ 5.98  │
> ├───────┼───────┤ 64000 │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4849.67  │ 25.07 │193.44 │1702945 │25120820│ 112472070 │38032901884│   0    │  26   │1437275│266204 │1499082│  6.40  │ 16.76 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3788.37  │ 24.67 │153.56 │3452319 │19624883│ 228263370 │29712047576│   0    │  52   │2920474│310841 │2980155│ 11.11  │ 6.59  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4249.31  │ 23.92 │177.65 │2662115 │22011879│ 175903720 │33325960247│   0    │  40   │2453648│281530 │2514216│  9.46  │ 8.75  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3562.91  │ 23.23 │153.38 │2300483 │18457417│ 151960722 │27944416164│   0    │  58   │2876396│ 63094 │2934764│ 36.46  │ 6.29  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4613.51  │ 23.90 │193.03 │2448721 │23898855│ 161744190 │36182832826│   0    │  37   │2690384│285235 │2757277│  8.58  │ 8.67  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 5257.18  │ 24.56 │214.05 │2793411 │27233581│ 184502818 │41231606500│   0    │  43   │1635722│348277 │1711248│  8.02  │ 15.91 │
> ├───────┼───────┤ 64000 │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4584.42  │ 23.13 │198.20 │2219308 │23748720│ 146595708 │35955528220│   0    │  34   │2455753│285711 │2526131│  7.77  │ 9.40  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3902.59  │ 23.93 │163.08 │1419017 │20218532│ 93777474  │30610704458│   0    │  85   │2689758│ 87910 │2751572│ 16.14  │ 7.35  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4384.12  │ 23.75 │184.59 │2236188 │22711421│ 147716182 │34385017633│   0    │  51   │2469602│214045 │2536198│ 10.45  │ 8.95  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3675.97  │ 22.57 │162.87 │1609139 │19045980│ 106378386 │28835553466│   0    │  347  │1925815│174777 │1996941│  9.21  │ 9.54  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3793.56  │ 24.91 │152.29 │1729962 │19657863│ 114362928 │29761926386│   0    │  666  │3102645│ 73302 │3170057│ 23.60  │ 6.20  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4569.95  │ 29.07 │157.21 │4440539 │23678370│ 293327866 │35848984990│   0    │  171  │1045134│320589 │1232501│ 13.85  │ 19.21 │
> ├───────┼───────┤ 64000 │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4007.90  │ 24.24 │165.34 │1363391 │20766245│ 90158958  │31439979966│   0    │  463  │2860456│140066 │2931297│  9.73  │ 7.08  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4062.92  │ 23.81 │170.64 │1113052 │21053084│ 73619916  │31874295850│   0    │  423  │2504525│132723 │2574155│  8.39  │ 8.18  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4022.06  │ 24.92 │161.40 │2051216 │20840308│ 135569610 │31552148131│   0    │  414  │2287715│168291 │2380990│ 12.19  │ 8.75  │
> ├───────┴───────┴───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │    Category:TCP_RR    │        │          │       │       │        │        │           │           │        │       │       │       │       │        │       │
> ├───────┬───────┬───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   #   │ Tile  │ size  │sessions│throughput│ %CPU  │ thr/  │#tx-pkts│#rx-pkts│ #tx-byts  │ #rx-byts  │   #    │   #   │   #   │   #   │   #   │#tpkt/# │#rpkt/#│
> │       │       │       │        │          │       │ %CPU  │        │        │           │           │re-trans│tx-intr│rx-intr│io_exit│irq_inj│  exit  │  irq  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 62063.30 │ 23.29 │2664.80│10898349│11138705│15614706386│15630551426│   13   │  166  │971977 │518626 │1023711│ 21.01  │ 10.88 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 44633.86 │ 19.54 │2284.23│7675417 │8425505 │11218936530│11268428118│   10   │  117  │316398 │740527 │379948 │ 10.36  │ 22.18 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 58846.04 │ 24.54 │2397.96│10361292│10595043│14807141312│14822541758│   19   │  158  │1708938│387095 │1763622│ 26.77  │ 6.01  │
> ├───────┼───────┤ 4000  │   50   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 46379.40 │ 20.40 │2273.50│7934994 │8651555 │11654996612│11702275190│   10   │  121  │273345 │529932 │336016 │ 14.97  │ 25.75 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 58458.58 │ 24.79 │2358.15│10288597│10523722│14709353306│14724854260│   12   │  157  │1662381│402538 │1719603│ 25.56  │ 6.12  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 54076.24 │ 22.51 │2402.32│9431729 │9866906 │13601026829│13629730150│   12   │  143  │986607 │515743 │1044580│ 18.29  │ 9.45  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 47725.79 │ 20.04 │2386.28│8143711 │8926371 │11992102514│12043740430│   17   │  124  │280776 │521236 │345152 │ 15.62  │ 25.86 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 66147.84 │ 23.00 │2875.99│11417925│11646862│16629505298│16644593452│   15   │  175  │1664709│157810 │1725000│ 72.35  │ 6.75  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 67209.33 │ 25.35 │2651.26│11637388│11791745│16898728136│16908915442│   0    │  177  │1777024│113475 │1835680│ 102.55 │ 6.42  │
> ├───────┼───────┤ 4000  │  100   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 66457.59 │ 25.03 │2655.12│11493181│11652590│16708810058│16719319572│   8    │  176  │1600205│133977 │1657520│ 85.78  │ 7.03  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 64096.68 │ 25.32 │2531.46│11072718│11257585│16114482396│16126669338│   10   │  168  │1460028│130160 │1518824│ 85.07  │ 7.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 62327.45 │ 23.75 │2624.31│10752984│11055030│15668725680│15688647646│   10   │  164  │1356548│211331 │1416435│ 50.88  │ 7.80  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.25  │ 0.00  │  875   │  1157  │  1173070  │  1606714  │   0    │   0   │  100  │  268  │ 2685  │  3.26  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 73510.59 │ 24.49 │3001.66│12299712│13024092│18455336036│18503152896│   3    │  187  │697196 │ 80346 │759284 │ 153.08 │ 17.15 │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │ 0.00  │  868   │  1097  │  1128200  │  1518018  │   0    │   0   │  105  │  260  │ 2785  │  3.34  │ 0.39  │
> ├───────┼───────┤ 4000  │  250   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   2.87   │ 0.24  │ 11.96 │  722   │  940   │  948860   │  1285684  │   0    │   0   │  88   │  246  │ 2621  │  2.93  │ 0.36  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.15   │ 0.26  │ 12.12 │  782   │  1035  │  1031076  │  1417930  │   0    │   0   │  92   │  260  │ 2678  │  3.01  │ 0.39  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 14703.32 │ 5.10  │2883.00│2460591 │2605664 │3691923448 │3701796248 │   0    │  37   │139516 │ 16276 │154010 │ 151.18 │ 16.92 │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.08   │ 0.25  │ 12.32 │  763   │  995   │  1004302  │  1376328  │   0    │   0   │  89   │  241  │ 2774  │  3.17  │ 0.36  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   4.27   │ 0.30  │ 14.23 │  998   │  1276  │  1334364  │  1751516  │   0    │   0   │  100  │  264  │ 2866  │  3.78  │ 0.45  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.00   │ 0.25  │ 12.00 │  726   │  954   │  942116   │  1304720  │   0    │   0   │  97   │  242  │ 2688  │  3.00  │ 0.35  │
> ├───────┼───────┤ 4000  │  500   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.35   │ 0.26  │ 12.88 │  845   │  1073  │  1108666  │  1493478  │   0    │   0   │  105  │  282  │ 2620  │  3.00  │ 0.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.38   │ 0.25  │ 13.52 │  851   │  1094  │  1129854  │  1527144  │   0    │   1   │  98   │  273  │ 3895  │  3.12  │ 0.28  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   3.42   │ 0.26  │ 13.15 │  836   │  1078  │  1103860  │  1490637  │   0    │   0   │  97   │  260  │ 2968  │  3.22  │ 0.36  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │ 0.00  │  662   │  862   │  872580   │  1197944  │   0    │   0   │  80   │  234  │ 2570  │  2.83  │ 0.34  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 14954.85 │ 26.51 │564.12 │12561026│13433340│18775875992│18833448696│   4    │  421  │1875885│ 53370 │1937836│ 235.36 │ 6.93  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 15216.05 │ 26.45 │575.28 │13203430│13579990│19131713484│19156583680│   3    │ 34188 │1701172│ 22748 │1792951│ 580.42 │ 7.57  │
> ├───────┼───────┤ 20000 │   50   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.68   │ 0.26  │ 2.62  │  907   │  1127  │  1154494  │  1587436  │   0    │   0   │  92   │  259  │ 2648  │  3.50  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.27  │ 0.00  │  792   │  1044  │  1070264  │  1470038  │   0    │   0   │  81   │  241  │ 2713  │  3.29  │ 0.38  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 6034.32  │ 10.75 │561.33 │5153363 │5403272 │7582137362 │7598857558 │   1    │ 6921  │715462 │ 15370 │747743 │ 335.29 │ 7.23  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.60   │ 0.25  │ 2.40  │  730   │  978   │  980108   │  1369008  │   0    │   0   │  74   │  228  │ 2750  │  3.20  │ 0.36  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.82   │ 0.27  │ 3.04  │  974   │  1318  │  1316596  │  1863320  │   0    │   0   │  109  │  254  │ 2684  │  3.83  │ 0.49  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.58   │ 0.28  │ 2.07  │  727   │  939   │  959446   │  1323908  │   0    │   0   │  81   │  227  │ 2644  │  3.20  │ 0.36  │
> ├───────┼───────┤ 20000 │  100   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.88   │ 0.26  │ 3.38  │  1096  │  1425  │  1469400  │  2010448  │   0    │   0   │  107  │  263  │ 2854  │  4.17  │ 0.50  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.24  │ 0.00  │  915   │  1219  │  1240790  │  1722258  │   0    │   0   │  115  │  261  │ 2626  │  3.51  │ 0.46  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.58   │ 0.26  │ 2.23  │  888   │  1175  │  1193268  │  1657788  │   0    │   0   │  97   │  246  │ 2711  │  3.61  │ 0.43  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.78   │ 0.28  │ 2.79  │  988   │  1247  │  1310460  │  1773674  │   0    │   0   │  111  │  236  │ 2859  │  4.19  │ 0.44  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.93   │ 0.25  │ 3.72  │  1120  │  1455  │  1520256  │  2075640  │   0    │   0   │  122  │  254  │ 2780  │  4.41  │ 0.52  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.80   │ 0.26  │ 3.08  │  974   │  1277  │  1322540  │  1818880  │   0    │   0   │  101  │  241  │ 2828  │  4.04  │ 0.45  │
> ├───────┼───────┤ 20000 │  250   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.85   │ 0.25  │ 3.40  │  1032  │  1364  │  1378648  │  1926622  │   0    │   0   │  108  │  259  │ 2647  │  3.98  │ 0.52  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.90   │ 0.24  │ 3.75  │  1042  │  1408  │  1424132  │  2006276  │   0    │   0   │  101  │  258  │ 4216  │  4.04  │ 0.33  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.85   │ 0.26  │ 3.27  │  1031  │  1350  │  1391207  │  1920218  │   0    │   0   │  108  │  249  │ 3066  │  4.14  │ 0.44  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │ 0.00  │  1060  │  1372  │  1436688  │  1942296  │   0    │   0   │  120  │  259  │ 2776  │  4.09  │ 0.49  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   1.05   │ 0.24  │ 4.38  │  1234  │  1663  │  1676340  │  2359874  │   0    │   0   │  116  │  277  │ 3976  │  4.45  │ 0.42  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.88   │ 0.27  │ 3.26  │  1043  │  1402  │  1403574  │  1993688  │   0    │   0   │  108  │  246  │ 2836  │  4.24  │ 0.49  │
> ├───────┼───────┤ 20000 │  500   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.77   │ 0.25  │ 3.08  │  945   │  1232  │  1256882  │  1736182  │   0    │   0   │  95   │  252  │ 2649  │  3.75  │ 0.47  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.77   │ 0.26  │ 2.96  │  940   │  1217  │  1280440  │  1730142  │   0    │   0   │  109  │  252  │ 2893  │  3.73  │ 0.42  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.69   │ 0.26  │ 2.65  │  1044  │  1377  │  1410784  │  1952436  │   0    │   0   │  109  │  257  │ 3026  │  4.06  │ 0.46  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │ 0.00  │  802   │  1025  │  1055124  │  1452832  │   0    │   0   │  91   │  245  │ 2743  │  3.27  │ 0.37  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.35   │ 0.25  │ 1.40  │  911   │  1141  │  1227270  │  1620910  │   0    │   0   │  96   │  248  │ 2804  │  3.67  │ 0.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.25  │ 0.00  │  975   │  1264  │  1276454  │  1798304  │   0    │   0   │  123  │  250  │ 2727  │  3.90  │ 0.46  │
> ├───────┼───────┤ 40000 │   50   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.37   │ 0.25  │ 1.48  │  990   │  1213  │  1241336  │  1728086  │   0    │   0   │  94   │  264  │ 4158  │  3.75  │ 0.29  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.47   │ 0.27  │ 1.74  │  1128  │  1514  │  1568448  │  2180904  │   0    │   0   │  96   │  231  │ 2679  │  4.88  │ 0.57  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.24   │ 0.26  │ 0.92  │  961   │  1231  │  1273726  │  1756207  │   0    │   0   │  100  │  247  │ 3022  │  3.89  │ 0.41  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.29  │ 0.00  │  981   │  1290  │  1334946  │  1839232  │   0    │   0   │  103  │  241  │ 2753  │  4.07  │ 0.47  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.37   │ 0.26  │ 1.42  │  871   │  1173  │  1181286  │  1672254  │   0    │   0   │  90   │  241  │ 2668  │  3.61  │ 0.44  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.33   │ 0.26  │ 1.27  │  807   │  1082  │  1063886  │  1542784  │   0    │   0   │  90   │  236  │ 2705  │  3.42  │ 0.40  │
> ├───────┼───────┤ 40000 │  100   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.35   │ 0.25  │ 1.40  │  871   │  1151  │  1152622  │  1630252  │   0    │   0   │  100  │  250  │ 2660  │  3.48  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.24  │ 0.00  │  815   │  1055  │  1096886  │  1495172  │   0    │   0   │  92   │  232  │ 2621  │  3.51  │ 0.40  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.21   │ 0.26  │ 0.81  │  869   │  1150  │  1165925  │  1635938  │   0    │   0   │  95   │  240  │ 2681  │  3.62  │ 0.43  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.42   │ 0.27  │ 1.56  │  1080  │  1350  │  1414736  │  1919280  │   0    │   0   │  99   │  248  │ 2797  │  4.35  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.40   │ 0.25  │ 1.60  │  1034  │  1328  │  1388932  │  1882180  │   0    │   0   │  95   │  277  │ 4118  │  3.73  │ 0.32  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.32   │ 0.24  │ 1.33  │  797   │  1031  │  1061490  │  1459482  │   0    │   0   │  76   │  224  │ 2635  │  3.56  │ 0.39  │
> ├───────┼───────┤ 40000 │  250   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.35   │ 0.25  │ 1.40  │  911   │  1152  │  1207526  │  1634406  │   0    │   0   │  91   │  233  │ 2640  │  3.91  │ 0.44  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.35   │ 0.23  │ 1.52  │  874   │  1160  │  1162948  │  1645460  │   0    │   0   │  83   │  233  │ 2686  │  3.75  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.37   │ 0.25  │ 1.48  │  939   │  1204  │  1247126  │  1708161  │   0    │   0   │  88   │  243  │ 2975  │  3.86  │ 0.40  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.23   │ 0.28  │ 0.82  │  598   │  771   │  788244   │  1069924  │   0    │   0   │  69   │  208  │ 2669  │  2.88  │ 0.29  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.24  │ 0.00  │  983   │  1265  │  1284630  │  1798626  │   0    │   0   │  92   │  223  │ 2632  │  4.41  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │ 0.00  │  955   │  1284  │  1308294  │  1831870  │   0    │   0   │  87   │  256  │ 2737  │  3.73  │ 0.47  │
> ├───────┼───────┤ 40000 │  500   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.42   │ 0.26  │ 1.62  │  1022  │  1346  │  1404428  │  1924040  │   0    │   0   │  91   │  247  │ 2745  │  4.14  │ 0.49  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.43   │ 0.26  │ 1.65  │  1084  │  1372  │  1468824  │  1964852  │   0    │   0   │  112  │  235  │ 2780  │  4.61  │ 0.49  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.22   │ 0.26  │ 0.85  │  928   │  1207  │  1250884  │  1717862  │   0    │   0   │  90   │  233  │ 2712  │  3.98  │ 0.45  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.23   │ 0.25  │ 0.92  │  968   │  1250  │  1308312  │  1771312  │   0    │   0   │  84   │  222  │ 2810  │  4.36  │ 0.44  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.20   │ 0.23  │ 0.87  │  846   │  1100  │  1120164  │  1573434  │   0    │   0   │  85   │  230  │ 2643  │  3.68  │ 0.42  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.22   │ 0.28  │ 0.79  │  879   │  1161  │  1184646  │  1640878  │   0    │   0   │  92   │  235  │ 2752  │  3.74  │ 0.42  │
> ├───────┼───────┤ 64000 │   50   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.27   │ 0.25  │ 1.08  │  1083  │  1423  │  1482126  │  2042986  │   0    │   0   │  85   │  234  │ 2679  │  4.63  │ 0.53  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.28   │ 0.28  │ 1.00  │  1226  │  1531  │  1693124  │  2175962  │   0    │   0   │  97   │  256  │ 2823  │  4.79  │ 0.54  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.24   │ 0.26  │ 0.92  │  1000  │  1293  │  1357674  │  1840914  │   0    │   0   │  88   │  235  │ 2741  │  4.26  │ 0.47  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.18   │ 0.28  │ 0.64  │  733   │  968   │  934722   │  1368346  │   0    │   0   │  79   │  227  │ 4022  │  3.23  │ 0.24  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.18   │ 0.25  │ 0.72  │  769   │  1022  │  988370   │  1436872  │   0    │   0   │  82   │  222  │ 2804  │  3.46  │ 0.36  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.23   │ 0.24  │ 0.96  │  986   │  1256  │  1337620  │  1774860  │   0    │   0   │  85   │  243  │ 2759  │  4.06  │ 0.46  │
> ├───────┼───────┤ 64000 │  100   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.22   │ 0.27  │ 0.81  │  913   │  1147  │  1233458  │  1639544  │   0    │   0   │  79   │  218  │ 2866  │  4.19  │ 0.40  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.20   │ 0.24  │ 0.83  │  808   │  1066  │  1058680  │  1506640  │   0    │   0   │  82   │  218  │ 2638  │  3.71  │ 0.40  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.20   │ 0.26  │ 0.77  │  841   │  1091  │  1110570  │  1545252  │   0    │   0   │  81   │  225  │ 3017  │  3.74  │ 0.36  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.25   │ 0.28  │ 0.89  │  1014  │  1293  │  1373748  │  1841014  │   0    │   0   │  97   │  231  │ 2825  │  4.39  │ 0.46  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.30   │ 0.25  │ 1.20  │  1224  │  1572  │  1632904  │  2263914  │   0    │   0   │  113  │  256  │ 2772  │  4.78  │ 0.57  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.28   │ 0.24  │ 1.17  │  1125  │  1491  │  1502834  │  2122642  │   0    │   0   │  116  │  248  │ 2690  │  4.54  │ 0.55  │
> ├───────┼───────┤ 64000 │  250   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.20   │ 0.28  │ 0.71  │  878   │  1109  │  1153140  │  1570166  │   0    │   0   │  84   │  223  │ 2696  │  3.94  │ 0.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.27   │ 0.25  │ 1.08  │  1066  │  1439  │  1457812  │  2056818  │   0    │   0   │  89   │  230  │ 2902  │  4.63  │ 0.50  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.26   │ 0.26  │ 1.00  │  1061  │  1380  │  1424087  │  1970910  │   0    │   0   │  99   │  237  │ 2777  │  4.48  │ 0.50  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.27   │ 0.29  │ 0.93  │  1084  │  1439  │  1441392  │  2054634  │   0    │   0   │  107  │  272  │ 2978  │  3.99  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.32   │ 0.27  │ 1.19  │  1247  │  1670  │  1667838  │  2404040  │   0    │   0   │  97   │  240  │ 2668  │  5.20  │ 0.63  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.24  │ 0.00  │  624   │  795   │  806336   │  1091926  │   0    │   0   │  82   │  213  │ 2638  │  2.93  │ 0.30  │
> ├───────┼───────┤ 64000 │  500   ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.20   │ 0.27  │ 0.74  │  863   │  1114  │  1126978  │  1580894  │   0    │   0   │  106  │  242  │ 2665  │  3.57  │ 0.42  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.17   │ 0.24  │ 0.71  │  677   │  924   │  887658   │  1301660  │   0    │   0   │  74   │  217  │ 2606  │  3.12  │ 0.35  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.19   │ 0.26  │ 0.73  │  899   │  1188  │  1186040  │  1686630  │   0    │   0   │  93   │  236  │ 2711  │  3.81  │ 0.44  │
> ├───────┴───────┴───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │  Category:TCP_MAERTS  │        │          │       │       │        │        │           │           │        │       │       │       │       │        │       │
> ├───────┬───────┬───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   #   │ Tile  │ size  │sessions│throughput│ %CPU  │ thr/  │#tx-pkts│#rx-pkts│ #tx-byts  │ #rx-byts  │   #    │   #   │   #   │   #   │   #   │#tpkt/# │#rpkt/#│
> │       │       │       │        │          │       │ %CPU  │        │        │           │           │re-trans│tx-intr│rx-intr│io_exit│irq_inj│  exit  │  irq  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3988.85  │ 22.30 │178.87 │20663566│ 702609 │31283038628│ 46372460  │   0    │ 87169 │ 61135 │ 1216  │180256 │16993.06│ 3.90  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4135.65  │ 23.64 │174.94 │21423513│ 741730 │32434120362│ 48954446  │   0    │ 88264 │ 63359 │ 1166  │193022 │18373.51│ 3.84  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3960.26  │ 22.88 │173.09 │20514440│ 676178 │31058530968│ 44628032  │   1    │ 87201 │ 60656 │ 1141  │179680 │17979.35│ 3.76  │
> ├───────┼───────┤ 4000  │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4066.22  │ 22.55 │180.32 │21064036│ 735381 │31889415224│ 48535460  │   3    │ 87256 │ 65406 │ 1311  │184763 │16067.15│ 3.98  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3974.81  │ 22.83 │174.10 │20590232│ 658115 │31172175872│ 43435856  │   0    │ 85508 │ 61146 │ 1135  │179836 │18141.17│ 3.66  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4025.16  │ 22.84 │176.23 │20851157│ 702802 │31567456210│ 46385250  │   0    │ 87079 │ 62340 │ 1193  │183511 │17477.92│ 3.83  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4019.54  │ 23.36 │172.07 │20823401│ 877237 │31523871386│ 57898240  │   4    │ 87464 │ 60938 │  583  │182355 │35717.67│ 4.81  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3918.00  │ 23.03 │170.13 │20297940│ 848490 │30728910688│ 56000908  │   2    │ 85495 │ 58819 │  562  │178602 │36117.33│ 4.75  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4070.35  │ 23.91 │170.24 │21085277│ 871966 │31922564178│ 57550288  │   0    │ 86792 │ 61284 │  639  │190685 │32997.30│ 4.57  │
> ├───────┼───────┤ 4000  │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3906.16  │ 23.10 │169.10 │20235381│ 837776 │30635105466│ 55293766  │   0    │ 84345 │ 58857 │  666  │177246 │30383.45│ 4.73  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3880.11  │ 23.62 │164.27 │20101370│ 838017 │30431057796│ 55309696  │   2    │ 83058 │ 58270 │  657  │180588 │30595.69│ 4.64  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3958.83  │ 23.40 │169.18 │20508673│ 854697 │31048301902│ 56410579  │   1    │ 85430 │ 59633 │  621  │181895 │33025.24│ 4.70  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3870.63  │ 24.01 │161.21 │20051651│1193761 │30356283574│ 78789404  │   0    │ 80351 │ 57986 │  546  │178208 │36724.64│ 6.70  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3956.89  │ 23.74 │166.68 │20500511│1276891 │31033832414│ 84275894  │   0    │ 83805 │ 59307 │  572  │180002 │35840.05│ 7.09  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3761.69  │ 23.31 │161.38 │19488129│1305199 │29503136162│ 86144252  │   0    │ 79995 │ 56420 │  527  │167531 │36979.37│ 7.79  │
> ├───────┼───────┤ 4000  │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3961.93  │ 23.58 │168.02 │20524829│1288000 │31073196066│ 85009094  │   0    │ 84571 │ 59466 │  645  │176957 │31821.44│ 7.28  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3955.87  │ 23.88 │165.66 │20494696│1273419 │31026429192│ 84046772  │   0    │ 83048 │ 59290 │  859  │184184 │23858.78│ 6.91  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3901.40  │ 23.70 │164.62 │20211963│1267454 │30598575481│ 83653083  │   0    │ 82354 │ 58493 │  629  │177376 │32133.49│ 7.15  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3939.69  │ 22.47 │175.33 │20408390│ 636433 │30896854212│ 42004868  │   0    │ 83640 │ 60558 │ 1086  │173872 │18792.26│ 3.66  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4050.50  │ 23.03 │175.88 │20982717│ 692047 │31766669514│ 45675398  │   0    │ 86543 │ 61909 │ 1138  │183047 │18438.24│ 3.78  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4115.30  │ 22.36 │184.05 │21317685│ 717320 │32273842266│ 47343434  │   0    │ 88329 │ 66337 │ 1164  │184388 │18314.16│ 3.89  │
> ├───────┼───────┤ 20000 │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3920.16  │ 22.16 │176.90 │20307155│ 636981 │30744055198│ 42041072  │   0    │ 84683 │ 59989 │ 1135  │172394 │17891.77│ 3.69  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3926.43  │ 30.41 │129.12 │20340064│ 639890 │30793586352│ 42233018  │   0    │ 82309 │ 61869 │ 21480 │303862 │ 946.93 │ 2.11  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3990.42  │ 24.09 │165.65 │20671202│ 664534 │31295001508│ 43859558  │   0    │ 85100 │ 62132 │ 5200  │203512 │3975.23 │ 3.27  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3959.39  │ 22.58 │175.35 │20510479│ 805041 │31052727934│ 53133328  │   0    │ 86778 │ 59644 │  734  │178305 │27943.43│ 4.51  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4050.01  │ 22.78 │177.79 │20980461│ 877866 │31763343418│ 57939700  │   0    │ 86554 │ 60885 │  700  │178383 │29972.09│ 4.92  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4011.19  │ 23.07 │173.87 │20779629│ 871885 │31459062858│ 57544960  │   0    │ 85376 │ 60277 │  755  │177244 │27522.69│ 4.92  │
> ├───────┼───────┤ 20000 │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3848.50  │ 22.61 │170.21 │19937377│ 830338 │30183571434│ 54802870  │   0    │ 83977 │ 57901 │  627  │170302 │31798.05│ 4.88  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3846.84  │ 23.13 │166.31 │19927805│ 829706 │30169904626│ 54761182  │   3    │ 82855 │ 57968 │ 1396  │174693 │14274.93│ 4.75  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3943.19  │ 22.83 │172.72 │20427150│ 842967 │30925722054│ 55636408  │   0    │ 85108 │ 59335 │  842  │175785 │24260.27│ 4.80  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3818.62  │ 23.43 │162.98 │19781592│1224374 │29948462512│ 80809886  │   7    │ 79444 │ 58162 │ 2085  │170555 │9487.57 │ 7.18  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3762.95  │ 23.13 │162.69 │19494796│1309165 │29512793160│ 86406026  │   0    │ 78322 │ 56323 │  577  │163565 │33786.47│ 8.00  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3837.60  │ 23.06 │166.42 │19880774│1283988 │30097771036│ 84744350  │   0    │ 80364 │ 57805 │ 1510  │166688 │13166.08│ 7.70  │
> ├───────┼───────┤ 20000 │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3967.47  │ 23.38 │169.70 │20553525│1383095 │31116665722│ 91285400  │   2    │ 83863 │ 59523 │  558  │168070 │36834.27│ 8.23  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 7789.67  │ 22.70 │343.16 │40359254│1763314 │61096751388│ 116379836 │   0    │167101 │117674 │ 1359  │338058 │29697.76│ 5.22  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4635.26  │ 23.14 │200.31 │24013988│1392787 │36354488763│ 91925099  │   1    │ 97818 │ 69897 │ 1217  │201387 │19732.12│ 6.92  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4052.75  │ 22.18 │182.72 │20994287│ 702295 │31784193038│ 46351760  │   0    │ 86061 │ 65178 │ 1092  │181116 │19225.54│ 3.88  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4018.67  │ 22.03 │182.42 │20818007│ 687128 │31517188942│ 45350732  │   0    │ 85422 │ 64407 │ 1122  │178529 │18554.37│ 3.85  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4033.66  │ 21.89 │184.27 │20895124│ 701588 │31634158688│ 46305074  │   0    │ 86101 │ 64900 │ 1097  │178966 │19047.52│ 3.92  │
> ├───────┼───────┤ 40000 │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3961.68  │ 22.21 │178.37 │20522707│ 637825 │31070139166│ 42096764  │   0    │ 83961 │ 60869 │ 1504  │178778 │13645.42│ 3.57  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4046.57  │ 22.13 │182.85 │20962093│ 700785 │31735498642│ 46252088  │   0    │ 85783 │ 64782 │ 1087  │180576 │19284.35│ 3.88  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4022.67  │ 22.09 │182.10 │20838443│ 685924 │31548235695│ 45271283  │   0    │ 85465 │ 64027 │ 1180  │179593 │17659.70│ 3.82  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4026.57  │ 22.74 │177.07 │20859499│ 854694 │31580307014│ 56410372  │   0    │ 86596 │ 61059 │  714  │176946 │29214.98│ 4.83  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3842.87  │ 22.33 │172.09 │19907513│ 888571 │30139156866│ 58646278  │   0    │ 83138 │ 57684 │  692  │166072 │28768.08│ 5.35  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3948.38  │ 22.72 │173.78 │20455465│ 854932 │30967374346│ 56426086  │   0    │ 85990 │ 59397 │  579  │174277 │35328.96│ 4.91  │
> ├───────┼───────┤ 40000 │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3946.23  │ 31.87 │123.82 │20441951│ 938273 │30948726710│ 61926556  │   0    │ 82487 │ 59234 │ 20108 │311634 │1016.61 │ 3.01  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4034.63  │ 22.73 │177.50 │20901228│ 856543 │31643261456│ 56532460  │   0    │ 88068 │ 61108 │  756  │176644 │27647.13│ 4.85  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3959.74  │ 24.48 │161.75 │20513131│ 878602 │31055765278│ 57988350  │   0    │ 85255 │ 59696 │ 4569  │201114 │4489.63 │ 4.37  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3743.58  │ 22.93 │163.26 │19393237│1321581 │29360433338│ 87225434  │   0    │ 78689 │ 56048 │  573  │160706 │33845.09│ 8.22  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3758.76  │ 23.39 │160.70 │19472185│1211187 │29480200370│ 79939478  │   0    │ 79328 │ 56333 │  578  │166718 │33688.90│ 7.26  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3729.28  │ 22.77 │163.78 │19320651│1433537 │29249089510│ 94614572  │   0    │ 78158 │ 55876 │  573  │158030 │33718.41│ 9.07  │
> ├───────┼───────┤ 40000 │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3829.35  │ 23.06 │166.06 │19837596│1271434 │30032803256│ 83915738  │   0    │ 80307 │ 57932 │ 2044  │166569 │9705.28 │ 7.63  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3665.94  │ 22.59 │162.28 │18991734│1302647 │28752506140│ 85975784  │   0    │ 77943 │ 54891 │  512  │157857 │37093.23│ 8.25  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3745.38  │ 22.95 │163.20 │19403080│1308077 │29375006522│ 86334201  │   0    │ 78885 │ 56216 │  856  │161976 │22667.15│ 8.08  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3951.11  │ 22.41 │176.31 │20467334│ 641396 │30986757444│ 42332444  │   0    │ 83827 │ 60678 │ 1124  │173757 │18209.37│ 3.69  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4022.83  │ 21.74 │185.04 │20839153│ 696117 │31548997058│ 45944006  │   0    │ 87346 │ 63989 │ 1115  │178829 │18689.82│ 3.89  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4084.46  │ 22.88 │178.52 │21158723│ 707793 │32032701670│ 46714628  │   1    │ 86788 │ 62586 │ 1140  │183126 │18560.28│ 3.87  │
> ├───────┼───────┤ 64000 │   1    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4026.67  │ 22.02 │182.86 │20858734│ 700613 │31579477516│ 46240772  │   0    │ 85411 │ 64538 │ 1127  │178786 │18508.19│ 3.92  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4047.81  │ 21.96 │184.33 │20968346│ 686094 │31745127340│ 45282470  │   0    │ 85159 │ 64912 │ 1131  │178700 │18539.65│ 3.84  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4026.58  │ 22.20 │181.38 │20858458│ 686402 │31578612205│ 45302864  │   0    │ 85706 │ 63340 │ 1127  │178639 │18507.95│ 3.84  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3876.48  │ 28.01 │138.40 │20081104│ 862098 │30402419344│ 56899012  │   0    │ 82929 │ 58601 │ 5979  │269621 │3358.61 │ 3.20  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3946.32  │ 22.82 │172.93 │20443637│ 848843 │30950716154│ 56024182  │   0    │ 86600 │ 59472 │  558  │171770 │36637.34│ 4.94  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4037.20  │ 23.29 │173.34 │20913327│ 845757 │31661129038│ 55820536  │   0    │ 85455 │ 60899 │  960  │179602 │21784.72│ 4.71  │
> ├───────┼───────┤ 64000 │   2    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4042.57  │ 23.58 │171.44 │20942024│ 842460 │31705740536│ 55602898  │   0    │ 85844 │ 60672 │  713  │184298 │29371.70│ 4.57  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3836.28  │ 22.24 │172.49 │19873255│ 893759 │30086532486│ 58988704  │   0    │ 82755 │ 57491 │  639  │163234 │31100.56│ 5.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3947.77  │ 23.99 │164.56 │20450669│ 858583 │30961307511│ 56667066  │   0    │ 84716 │ 59427 │ 1769  │193705 │11560.58│ 4.43  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3673.98  │ 25.74 │142.73 │19034041│1300898 │28816134826│ 85860392  │   0    │ 77098 │ 55140 │ 14927 │219874 │1275.14 │ 5.92  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3884.87  │ 22.52 │172.51 │20125917│1420081 │30469199442│ 93726458  │   0    │ 83004 │ 58313 │  776  │163768 │25935.46│ 8.67  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3742.00  │ 22.75 │164.48 │19385791│1302985 │29349125678│ 85998116  │   0    │ 78659 │ 56041 │  522  │160876 │37137.53│ 8.10  │
> ├───────┼───────┤ 64000 │   4    ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3811.54  │ 22.81 │167.10 │19745934│1342980 │29894074932│ 88637774  │   0    │ 80674 │ 57031 │  536  │163672 │36839.43│ 8.21  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3753.64  │ 23.20 │161.79 │19446003│1222424 │29440289838│ 80681078  │   0    │ 78992 │ 56190 │  570  │164890 │34115.79│ 7.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3773.21  │ 23.40 │161.25 │19547537│1317873 │29593764943│ 86980763  │   0    │ 79685 │ 56543 │ 3466  │174616 │5639.80 │ 7.55  │
> └───────┴───────┴───────┴────────┴──────────┴───────┴───────┴────────┴────────┴───────────┴───────────┴────────┴───────┴───────┴───────┴───────┴────────┴───────┘
> ┌───────┬────────────────────────┬──────────┬───────┬────────┬────────┬────────┬───────────┬───────────┬────────┬───────┬───────┬───────┬───────┬────────┬───────┐
> │== Raw │                        │          │       │        │        │        │           │           │        │       │       │       │       │        │       │
> │data of│  Category:TCP_STREAM   │          │       │        │        │        │           │           │        │       │       │       │       │        │       │
> │sample │                        │          │       │        │        │        │           │           │        │       │       │       │       │        │       │
> │2 ==   │                        │          │       │        │        │        │           │           │        │       │       │       │       │        │       │
> ├───────┼───────┬───────┬────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   #   │ Tile  │ size  │sessions│throughput│ %CPU  │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts  │ #rx-byts  │   #    │   #   │   #   │   #   │   #   │#tpkt/# │#rpkt/#│
> │       │       │       │        │          │       │        │        │        │           │           │re-trans│tx-intr│rx-intr│io_exit│irq_inj│  exit  │  irq  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2704.11  │ 22.95 │ 117.83 │2907775 │14012723│ 191913618 │21207695978│   0    │  57   │4284600│ 20911 │4334295│ 139.05 │ 3.23  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3888.53  │ 24.51 │ 158.65 │3349580 │20143602│ 221261328 │30497373152│   0    │  51   │3179002│283982 │3239097│ 11.80  │ 6.22  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3838.67  │ 24.72 │ 155.29 │3346052 │19885195│ 221044149 │30106152476│   0    │  51   │3224560│282863 │3285371│ 11.83  │ 6.05  │
> ├───────┼───────┤ 4000  │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4851.42  │ 21.26 │ 228.19 │1825669 │25130037│ 120580914 │38046840086│   0    │  27   │1575381│273544 │1636227│  6.67  │ 15.36 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4885.12  │ 21.95 │ 222.56 │1977029 │25305141│ 130554666 │38311493990│   0    │  30   │1570194│243815 │1630899│  8.11  │ 15.52 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4033.57  │ 23.08 │ 174.76 │2681221 │20895339│ 177070935 │31633911136│   0    │  43   │2766747│221023 │2825177│ 12.13  │ 7.40  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3786.22  │ 20.80 │ 182.03 │2983686 │19615638│ 197086348 │29698041134│   0    │  45   │605132 │347951 │682299 │  8.58  │ 28.75 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3652.15  │ 21.43 │ 170.42 │2053121 │18919870│ 135618378 │28644647774│   0    │  31   │1543871│310310 │1625181│  6.62  │ 11.64 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 7441.98  │ 24.34 │ 305.75 │6316423 │38551769│ 417229998 │58367329674│   0    │  96   │6246031│593528 │6363931│ 10.64  │ 6.06  │
> ├───────┼───────┤ 4000  │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4567.63  │ 23.81 │ 191.84 │2528145 │23660920│ 167012418 │35822600626│   0    │  39   │2603634│300354 │2670947│  8.42  │ 8.86  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4646.79  │ 23.57 │ 197.15 │1974521 │24071090│ 130429014 │36443594598│   0    │  30   │2773310│304875 │2842797│  6.48  │ 8.47  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4818.95  │ 22.79 │ 211.45 │3171179 │24963857│ 209475231 │37795242761│   0    │  48   │2754395│371403 │2837031│  8.54  │ 8.80  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4051.46  │ 24.45 │ 165.70 │1510339 │20993455│ 99856390  │31783956688│   0    │  409  │2713109│143234 │2784162│ 10.54  │ 7.54  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3866.81  │ 24.93 │ 155.11 │1574387 │20036882│ 104105382 │30335728454│   0    │  649  │2947421│ 69907 │3015449│ 22.52  │ 6.64  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3728.31  │ 22.59 │ 165.04 │1960031 │19317154│ 129472546 │29245991608│   0    │  364  │1965742│155990 │2034963│ 12.57  │ 9.49  │
> ├───────┼───────┤ 4000  │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4061.86  │ 24.28 │ 167.29 │1736036 │21046177│ 114746536 │31863847236│   0    │  328  │2856487│146552 │2925562│ 11.85  │ 7.19  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4110.01  │ 24.28 │ 169.28 │2571713 │21294288│ 169919486 │32239342226│   0    │  428  │2050867│200276 │2145812│ 12.84  │ 9.92  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3963.69  │ 24.11 │ 164.40 │1870501 │20537591│ 123620068 │31093773242│   0    │  435  │2506725│143191 │2581189│ 13.06  │ 7.96  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4744.82  │ 21.62 │ 219.46 │1881366 │24578233│ 124306620 │37211355172│   0    │  28   │1467339│253972 │1527534│  7.41  │ 16.09 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3769.12  │ 24.58 │ 153.34 │3965906 │19525211│ 262103844 │29561150912│   0    │  60   │3072677│313512 │3130847│ 12.65  │ 6.24  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3781.89  │ 24.75 │ 152.80 │3544977 │19591576│ 234446274 │29661625644│   0    │  54   │3152734│302048 │3212757│ 11.74  │ 6.10  │
> ├───────┼───────┤ 20000 │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2858.49  │ 16.28 │ 175.58 │1355178 │14806747│ 89510196  │22417397168│   0    │  20   │191452 │247752 │247840 │  5.47  │ 59.74 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3767.71  │ 24.45 │ 154.10 │3450112 │19517823│ 228125784 │29549958456│   0    │  53   │3057351│294002 │3116880│ 11.73  │ 6.26  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3784.41  │ 22.34 │ 169.40 │2839507 │19603918│ 187698543 │29680297470│   0    │  43   │2188310│282257 │2247171│ 10.06  │ 8.72  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4186.58  │ 24.53 │ 170.67 │2286900 │21687371│ 151181964 │32834613584│   0    │  35   │3576688│287746 │3643601│  7.95  │ 5.95  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4248.25  │ 25.40 │ 167.25 │2899171 │22006813│ 191591154 │33318281676│   0    │  44   │4008122│307063 │4075157│  9.44  │ 5.40  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3637.26  │ 21.54 │ 168.86 │2524675 │18842523│ 166796066 │28527543226│   0    │  38   │1534341│315457 │1611139│  8.00  │ 11.70 │
> ├───────┼───────┤ 20000 │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2686.80  │ 22.35 │ 120.21 │3252341 │13923063│ 214677438 │21075480536│   0    │  66   │4184634│ 20728 │4237244│ 156.91 │ 3.29  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4784.92  │ 24.34 │ 196.59 │3985712 │24789549│ 263338776 │37531341348│   0    │  61   │1324642│332932 │1395732│ 11.97  │ 17.76 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3908.76  │ 23.63 │ 165.42 │2989759 │20249863│ 197517079 │30657452074│   0    │  48   │2925685│252785 │2992574│ 11.83  │ 6.77  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 8022.09  │ 25.90 │ 309.73 │7152232 │41566518│ 472811332 │62931646336│   0    │  123  │4113976│644581 │4331730│ 11.10  │ 9.60  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4712.27  │ 30.61 │ 153.95 │4575153 │24414463│ 302214850 │36963432360│   0    │  312  │2904389│156807 │3056868│ 29.18  │ 7.99  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3901.15  │ 24.50 │ 159.23 │1813769 │20214332│ 119904282 │30604350588│   0    │  637  │2674732│ 74532 │2741872│ 24.34  │ 7.37  │
> ├───────┼───────┤ 20000 │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4452.37  │ 23.25 │ 191.50 │1787954 │23065956│ 118163796 │34921688978│   0    │  410  │2121276│166916 │2191769│ 10.71  │ 10.52 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4624.39  │ 26.32 │ 175.70 │2673177 │23957143│ 176660494 │36271036848│   0    │  222  │2545215│201657 │2642107│ 13.26  │ 9.07  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 5142.45  │ 26.12 │ 196.88 │3600457 │26643682│ 237950950 │40338431022│   0    │  340  │2871917│248898 │2992869│ 14.47  │ 8.90  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2880.97  │ 16.61 │ 173.45 │1080285 │14923347│ 71344822  │22593863376│   0    │  17   │194394 │228813 │250948 │  4.72  │ 59.47 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3772.68  │ 24.77 │ 152.31 │4077197 │19543225│ 269431086 │29588419476│   0    │  62   │2953852│292887 │3011843│ 13.92  │ 6.49  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4849.43  │ 24.25 │ 199.97 │1897426 │25119934│ 125365824 │38031556984│   0    │  29   │1716778│257484 │1779754│  7.37  │ 14.11 │
> ├───────┼───────┤ 40000 │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4858.75  │ 21.38 │ 227.26 │1755987 │25168751│ 116029578 │38105440512│   0    │  27   │1602459│260765 │1664066│  6.73  │ 15.12 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3786.37  │ 24.13 │ 156.92 │3700233 │19614201│ 244671618 │29695868172│   0    │  56   │3087133│295534 │3146233│ 12.52  │ 6.23  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4029.64  │ 22.23 │ 181.27 │2502225 │20873891│ 165368585 │31603029704│   0    │  38   │1910923│267096 │1970568│  9.37  │ 10.59 │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3639.98  │ 21.40 │ 170.09 │2343193 │18856650│ 154776390 │28548931822│   0    │  35   │1450223│305223 │1530321│  7.68  │ 12.32 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3519.15  │ 21.09 │ 166.86 │2589904 │18230675│ 171086228 │27601210992│   0    │  40   │1406963│348234 │1483017│  7.44  │ 12.29 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3897.87  │ 24.19 │ 161.14 │1395597 │20193214│ 92202474  │30572469616│   0    │  134  │3009384│ 84229 │3071018│ 16.57  │ 6.58  │
> ├───────┼───────┤ 40000 │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4088.77  │ 23.70 │ 172.52 │1725490 │21182211│ 113977944 │32069835208│   0    │  46   │2248564│103394 │2307795│ 16.69  │ 9.18  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4595.19  │ 23.77 │ 193.32 │2272436 │23805069│ 150117360 │36040841940│   0    │  35   │2747524│278929 │2813597│  8.15  │ 8.46  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3948.19  │ 22.83 │ 172.94 │2065324 │20453563│ 136432079 │30966657915│   0    │  58   │2172531│224001 │2241149│  9.22  │ 9.13  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4042.30  │ 25.90 │ 156.07 │3399939 │20941546│ 224574062 │31705416286│   0    │  438  │1571217│235937 │1719404│ 14.41  │ 12.18 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4073.20  │ 24.79 │ 164.31 │1623021 │21103387│ 107293106 │31950461424│   0    │  424  │2910404│142654 │2979999│ 11.38  │ 7.08  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4008.18  │ 25.92 │ 154.64 │3101260 │20767754│ 204855532 │31442279760│   0    │  485  │2372529│186924 │2491231│ 16.59  │ 8.34  │
> ├───────┼───────┤ 40000 │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4324.08  │ 23.84 │ 181.38 │1703347 │22403968│ 112636030 │33919514738│   0    │  371  │2409793│156562 │2479710│ 10.88  │ 9.03  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3824.55  │ 22.26 │ 171.81 │1378259 │19814667│ 91121678  │29999199560│   0    │  433  │2077285│136274 │2148403│ 10.11  │ 9.22  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4054.46  │ 24.54 │ 165.22 │2241165 │21006264│ 148096081 │31803374353│   0    │  430  │2268245│171670 │2363749│ 13.06  │ 8.89  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2875.73  │ 16.55 │ 173.76 │1046616 │14896281│ 69128628  │22552951756│   0    │  16   │186818 │234643 │242766 │  4.46  │ 61.36 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2773.04  │ 22.77 │ 121.78 │2472146 │14367855│ 163162140 │21748293224│   0    │  39   │4464194│ 21370 │4514215│ 115.68 │ 3.18  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2920.04  │ 0.24  │12166.83│1010770 │15125856│ 66761964  │22900524916│   0    │  15   │200458 │249728 │260159 │  4.05  │ 58.14 │
> ├───────┼───────┤ 64000 │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4798.54  │ 21.48 │ 223.40 │2009615 │24856224│ 132749118 │37632239906│   0    │  31   │1562879│272276 │1623563│  7.38  │ 15.31 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3793.02  │ 24.26 │ 156.35 │3097696 │19648800│ 204622872 │29748262530│   0    │  47   │3089358│298467 │3150032│ 10.38  │ 6.24  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3432.07  │ 17.06 │ 201.18 │1927368 │17779003│ 127284944 │26916454466│   0    │  29   │1900741│215296 │1958147│  8.95  │ 9.08  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4173.98  │ 24.44 │ 170.78 │2327586 │21623077│ 153753300 │32737291364│   0    │  35   │3475950│277360 │3541858│  8.39  │ 6.11  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3133.37  │ 21.89 │ 143.14 │1591111 │16235089│ 105073710 │24578920852│   0    │  41   │2520960│ 65417 │2582043│ 24.32  │ 6.29  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3622.76  │ 21.53 │ 168.27 │2592610 │18768922│ 171253880 │28416104520│   0    │  40   │1555293│357017 │1637673│  7.26  │ 11.46 │
> ├───────┼───────┤ 64000 │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4557.01  │ 23.66 │ 192.60 │2351702 │23606228│ 155409804 │35739794914│   0    │  36   │2630580│297393 │2698853│  7.91  │ 8.75  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4100.16  │ 25.00 │ 164.01 │2286834 │21239696│ 151055064 │32156867554│   0    │  35   │3766742│271508 │3832963│  8.42  │ 5.54  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3917.46  │ 23.30 │ 168.13 │2229968 │20294602│ 147309151 │30725795840│   0    │  37   │2789905│253739 │2858678│  8.79  │ 7.10  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4385.07  │ 27.80 │ 157.74 │3430323 │22718377│ 226572970 │34395529388│   0    │  278  │1854662│208876 │1998101│ 16.42  │ 11.37 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4512.11  │ 26.14 │ 172.61 │3209923 │23376545│ 212052366 │35392003060│   0    │  292  │1536969│221988 │1675066│ 14.46  │ 13.96 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 8433.51  │ 21.98 │ 383.69 │2823699 │43691063│ 186569130 │66148048156│   0    │  237  │3906821│370898 │4035657│  7.61  │ 10.83 │
> ├───────┼───────┤ 64000 │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4462.51  │ 29.99 │ 148.80 │4877568 │23119148│ 322251516 │35002322858│   0    │  701  │3029784│126916 │3184855│ 38.43  │ 7.26  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 4459.60  │ 25.59 │ 174.27 │3111864 │23103899│ 205536912 │34979240304│   0    │  520  │2473750│209567 │2558301│ 14.85  │ 9.03  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 5250.56  │ 26.30 │ 199.64 │3490675 │27201806│ 230596578 │41183428753│   0    │  405  │2560397│227649 │2690396│ 15.33  │ 10.11 │
> ├───────┴───────┴───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │    Category:TCP_RR    │        │          │       │        │        │        │           │           │        │       │       │       │       │        │       │
> ├───────┬───────┬───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   #   │ Tile  │ size  │sessions│throughput│ %CPU  │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts  │ #rx-byts  │   #    │   #   │   #   │   #   │   #   │#tpkt/# │#rpkt/#│
> │       │       │       │        │          │       │        │        │        │           │           │re-trans│tx-intr│rx-intr│io_exit│irq_inj│  exit  │  irq  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 36356.50 │ 21.56 │1686.29 │6329707 │7305598 │9143557878 │9207958148 │   6    │  96   │368276 │140693 │408918 │ 44.99  │ 17.87 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 36851.57 │ 21.72 │1696.67 │6417742 │7522264 │9268180532 │9341066308 │   9    │  98   │394656 │142462 │433435 │ 45.05  │ 17.35 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 31402.25 │ 18.59 │1689.20 │5466008 │6562438 │7897527952 │7969889264 │   2    │  83   │159095 │124398 │190401 │ 43.94  │ 34.47 │
> ├───────┼───────┤ 4000  │   50   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 49212.54 │ 21.74 │2263.69 │8415229 │8994054 │12366661314│12404856280│   6    │  128  │178364 │218356 │233510 │ 38.54  │ 38.52 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 30853.70 │ 21.25 │1451.93 │5355604 │6476282 │7758578736 │7832539160 │   3    │  81   │137886 │118588 │170090 │ 45.16  │ 38.08 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 36935.31 │ 20.97 │1761.34 │6396858 │7372127 │9286901282 │9351261832 │   5    │  97   │247655 │148899 │287270 │ 42.96  │ 25.66 │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 41946.49 │ 23.08 │1817.44 │7201206 │8953859 │10542855364│10658525090│   5    │  110  │190063 │126137 │237546 │ 57.09  │ 37.69 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 37981.85 │ 22.51 │1687.33 │6517503 │8277587 │9546220070 │9662382286 │   3    │  99   │493987 │ 70974 │541265 │ 91.83  │ 15.29 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 50123.47 │ 22.06 │2272.14 │8432605 │9343192 │12586656774│12646735932│   19   │  130  │187073 │221880 │249952 │ 38.01  │ 37.38 │
> ├───────┼───────┤ 4000  │  100   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 39227.62 │ 22.26 │1762.25 │6755439 │8640974 │9860901638 │9985339124 │   7    │  104  │135138 │ 84459 │179422 │ 79.98  │ 48.16 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 44694.70 │ 23.39 │1910.85 │7636640 │9310875 │11231178352│11341667606│   8    │  117  │283512 │173613 │333668 │ 43.99  │ 27.90 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 42794.83 │ 22.66 │1888.56 │7308678 │8905297 │10753562439│10858930007│   8    │  112  │257954 │135412 │308370 │ 53.97  │ 28.88 │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   4.00   │ 0.28  │ 14.29  │  957   │  1227  │  1273466  │  1689722  │   0    │   0   │  105  │  261  │ 2797  │  3.67  │ 0.44  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │  0.00  │  899   │  1160  │  1191638  │  1606878  │   0    │   0   │  111  │  263  │ 2695  │  3.42  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.72   │ 0.26  │ 14.31  │  909   │  1170  │  1202626  │  1616872  │   0    │   0   │  129  │  253  │ 2691  │  3.59  │ 0.43  │
> ├───────┼───────┤ 4000  │  250   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 49676.99 │ 23.86 │2082.02 │8340981 │10064491│12473985274│12587736678│   0    │ 9808  │1110558│ 21149 │1173132│ 394.39 │ 8.58  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │  0.00  │  790   │  986   │  1044732  │  1364386  │   0    │   0   │  86   │  239  │ 2609  │  3.31  │ 0.38  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 9936.94  │ 4.98  │1995.37 │1668907 │2013806 │2495739547 │2518802907 │   0    │ 1961  │222197 │ 4433  │236784 │ 376.47 │ 8.50  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │  0.00  │  636   │  824   │  815040   │  1120318  │   0    │   0   │  83   │  224  │ 2591  │  2.84  │ 0.32  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.92   │ 0.26  │ 15.08  │  939   │  1240  │  1256934  │  1714956  │   0    │   0   │  114  │  248  │ 2730  │  3.79  │ 0.45  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   3.88   │ 0.26  │ 14.92  │  972   │  1236  │  1277104  │  1693474  │   0    │   0   │  127  │  274  │ 2845  │  3.55  │ 0.43  │
> ├───────┼───────┤ 4000  │  500   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │  0.00  │  1132  │  1496  │  1535616  │  2052668  │   0    │   0   │  108  │  273  │ 2735  │  4.15  │ 0.55  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.27  │  0.00  │  825   │  1081  │  1069850  │  1488438  │   0    │   0   │  105  │  278  │ 2612  │  2.97  │ 0.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   1.56   │ 0.27  │  5.78  │  900   │  1175  │  1190908  │  1613970  │   0    │   0   │  107  │  259  │ 2702  │  3.47  │ 0.43  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 11344.08 │ 22.87 │ 496.02 │9824640 │10055813│14262352560│14277604386│   4    │ 30659 │1226543│ 10261 │1305288│ 957.47 │ 7.70  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.65   │ 0.28  │  2.32  │  814   │  1053  │  1062260  │  1454878  │   0    │   0   │  91   │  227  │ 2686  │  3.59  │ 0.39  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.73   │ 0.23  │  3.17  │  896   │  1161  │  1186760  │  1623804  │   0    │   0   │  99   │  230  │ 2625  │  3.90  │ 0.44  │
> ├───────┼───────┤ 20000 │   50   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 11850.73 │ 24.29 │ 487.89 │9969415 │10802298│14879924790│14934892060│   2    │ 33254 │1321477│ 10110 │1400512│ 986.09 │ 7.71  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.77   │ 0.27  │  2.85  │  908   │  1226  │  1236176  │  1732848  │   0    │   0   │  87   │  219  │ 2694  │  4.15  │ 0.46  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 4639.39  │ 9.59  │ 483.77 │3959334 │4172310 │5829152509 │5843461595 │   1    │ 12782 │509659 │ 4209  │542761 │ 940.68 │ 7.69  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.77   │ 0.26  │  2.96  │  948   │  1229  │  1275136  │  1730702  │   0    │   0   │  86   │  237  │ 2773  │  4.00  │ 0.44  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │  0.00  │  868   │  1113  │  1170752  │  1558662  │   0    │   0   │  92   │  238  │ 2844  │  3.65  │ 0.39  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.67   │ 0.25  │  2.68  │  808   │  1056  │  1082280  │  1475388  │   0    │   0   │  83   │  230  │ 2856  │  3.51  │ 0.37  │
> ├───────┼───────┤ 20000 │  100   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.88   │ 0.24  │  3.67  │  1084  │  1395  │  1481608  │  1986746  │   0    │   0   │  110  │  247  │ 3965  │  4.39  │ 0.35  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.83   │ 0.28  │  2.96  │  999   │  1309  │  1355190  │  1861158  │   0    │   0   │  93   │  239  │ 2707  │  4.18  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.63   │ 0.26  │  2.42  │  941   │  1220  │  1272993  │  1722531  │   0    │   0   │  92   │  238  │ 3029  │  3.95  │ 0.40  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │  0.00  │  934   │  1250  │  1265012  │  1757392  │   0    │   0   │  100  │  237  │ 2644  │  3.94  │ 0.47  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │  0.00  │  1200  │  1563  │  1653520  │  2219130  │   0    │   0   │  111  │  253  │ 2738  │  4.74  │ 0.57  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.25  │  0.00  │  803   │  1014  │  1083878  │  1406896  │   0    │   0   │  91   │  229  │ 2616  │  3.51  │ 0.39  │
> ├───────┼───────┤ 20000 │  250   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.77   │ 0.25  │  3.08  │  915   │  1224  │  1233790  │  1711772  │   0    │   0   │  113  │  239  │ 2691  │  3.83  │ 0.45  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.27  │  0.00  │  616   │  814   │  813272   │  1112280  │   0    │   0   │  74   │  211  │ 2641  │  2.92  │ 0.31  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.15   │ 0.26  │  0.58  │  893   │  1173  │  1209894  │  1641494  │   0    │   0   │  97   │  233  │ 2666  │  3.83  │ 0.44  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │  0.00  │  845   │  1092  │  1126410  │  1516820  │   0    │   0   │  93   │  232  │ 2607  │  3.64  │ 0.42  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.62   │ 0.28  │  2.21  │  749   │  993   │  1003666  │  1390734  │   0    │   0   │  82   │  218  │ 2820  │  3.44  │ 0.35  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.27  │  0.00  │  841   │  1091  │  1133178  │  1516794  │   0    │   0   │  93   │  227  │ 2615  │  3.70  │ 0.42  │
> ├───────┼───────┤ 20000 │  500   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.85   │ 0.25  │  3.40  │  1037  │  1350  │  1429114  │  1891752  │   0    │   0   │  111  │  243  │ 2690  │  4.27  │ 0.50  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.85   │ 0.29  │  2.93  │  1027  │  1369  │  1389686  │  1929630  │   0    │   0   │  112  │  246  │ 2854  │  4.17  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.46   │ 0.27  │  1.70  │  899   │  1179  │  1216410  │  1649146  │   0    │   0   │  98   │  233  │ 2717  │  3.86  │ 0.43  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.40   │ 0.25  │  1.60  │  963   │  1288  │  1333486  │  1838874  │   0    │   0   │  92   │  219  │ 2708  │  4.40  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.29  │  0.00  │  888   │  1143  │  1200232  │  1621170  │   0    │   0   │  81   │  223  │ 2700  │  3.98  │ 0.42  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.35   │ 0.26  │  1.35  │  905   │  1129  │  1163978  │  1578790  │   0    │   0   │  82   │  213  │ 2650  │  4.25  │ 0.43  │
> ├───────┼───────┤ 40000 │   50   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.27  │  0.00  │  772   │  1005  │  1028608  │  1410784  │   0    │   0   │  92   │  233  │ 2628  │  3.31  │ 0.38  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │  0.00  │  1074  │  1327  │  1366548  │  1878404  │   0    │   0   │  90   │  235  │ 2602  │  4.57  │ 0.51  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.15   │ 0.27  │  0.56  │  920   │  1178  │  1218570  │  1665604  │   0    │   0   │  87   │  224  │ 2657  │  4.11  │ 0.44  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │  0.00  │  1069  │  1368  │  1463354  │  1961296  │   0    │   0   │  88   │  240  │ 2843  │  4.45  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.30   │ 0.28  │  1.07  │  725   │  978   │  972450   │  1369912  │   0    │   0   │  76   │  223  │ 2912  │  3.25  │ 0.34  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │  0.00  │  991   │  1305  │  1358838  │  1839990  │   0    │   0   │  96   │  244  │ 2798  │  4.06  │ 0.47  │
> ├───────┼───────┤ 40000 │  100   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.24  │  0.00  │  985   │  1286  │  1333938  │  1828972  │   0    │   0   │  94   │  231  │ 2640  │  4.26  │ 0.49  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.35   │ 0.26  │  1.35  │  890   │  1145  │  1217428  │  1621302  │   0    │   0   │  78   │  209  │ 2663  │  4.26  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.13   │ 0.26  │  0.50  │  932   │  1216  │  1269201  │  1724294  │   0    │   0   │  86   │  229  │ 2771  │  4.07  │ 0.44  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.25  │  0.00  │  880   │  1114  │  1186088  │  1578700  │   0    │   0   │  77   │  223  │ 2862  │  3.95  │ 0.39  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.33   │ 0.31  │  1.06  │  845   │  1111  │  1147746  │  1578258  │   0    │   0   │  90   │  227  │ 2869  │  3.72  │ 0.39  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.24  │  0.00  │  1008  │  1347  │  1384200  │  1906276  │   0    │   0   │  93   │  244  │ 2663  │  4.13  │ 0.51  │
> ├───────┼───────┤ 40000 │  250   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.35   │ 0.25  │  1.40  │  921   │  1183  │  1224106  │  1661964  │   0    │   0   │  91   │  221  │ 2736  │  4.17  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.27  │  0.00  │  941   │  1215  │  1221122  │  1721868  │   0    │   0   │  85   │  233  │ 2704  │  4.04  │ 0.45  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.14   │ 0.26  │  0.54  │  919   │  1194  │  1232652  │  1689413  │   0    │   0   │  87   │  229  │ 2766  │  4.01  │ 0.43  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.28   │ 0.26  │  1.08  │  722   │  969   │  949508   │  1365904  │   0    │   0   │  85   │  221  │ 2938  │  3.27  │ 0.33  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.29  │  0.00  │  1063  │  1443  │  1445038  │  2055348  │   0    │   0   │  111  │  249  │ 2586  │  4.27  │ 0.56  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.37   │ 0.25  │  1.48  │  898   │  1200  │  1221324  │  1711300  │   0    │   0   │  79   │  230  │ 2747  │  3.90  │ 0.44  │
> ├───────┼───────┤ 40000 │  500   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.48   │ 0.26  │  1.85  │  1185  │  1559  │  1623266  │  2227442  │   0    │   0   │  96   │  248  │ 2836  │  4.78  │ 0.55  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.40   │ 0.24  │  1.67  │  981   │  1297  │  1334458  │  1835478  │   0    │   0   │  91   │  230  │ 2677  │  4.27  │ 0.48  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.31   │ 0.26  │  1.19  │  969   │  1293  │  1314718  │  1839094  │   0    │   0   │  92   │  235  │ 2756  │  4.12  │ 0.47  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.22   │ 0.26  │  0.85  │  903   │  1151  │  1209982  │  1637342  │   0    │   0   │  90   │  223  │ 2840  │  4.05  │ 0.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.18   │ 0.27  │  0.67  │  779   │  1015  │  1020414  │  1439818  │   0    │   0   │  78   │  222  │ 2817  │  3.51  │ 0.36  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.17   │ 0.26  │  0.65  │  691   │  930   │  898342   │  1305624  │   0    │   0   │  77   │  213  │ 2811  │  3.24  │ 0.33  │
> ├───────┼───────┤ 64000 │   50   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.18   │ 0.27  │  0.67  │  732   │  968   │  960704   │  1372004  │   0    │   0   │  68   │  204  │ 2695  │  3.59  │ 0.36  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.28   │ 0.25  │  1.12  │  1126  │  1495  │  1543540  │  2124226  │   0    │   0   │  101  │  236  │ 2720  │  4.77  │ 0.55  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.21   │ 0.26  │  0.81  │  846   │  1111  │  1126596  │  1575802  │   0    │   0   │  82   │  219  │ 2776  │  3.86  │ 0.40  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.28   │ 0.24  │  1.17  │  1106  │  1486  │  1487636  │  2122632  │   0    │   0   │  103  │  248  │ 2818  │  4.46  │ 0.53  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.22   │ 0.29  │  0.76  │  899   │  1161  │  1223934  │  1641038  │   0    │   0   │  86   │  219  │ 2681  │  4.11  │ 0.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.26  │  0.00  │  1351  │  1708  │  1868726  │  2457422  │   0    │   0   │  108  │  243  │ 2749  │  5.56  │ 0.62  │
> ├───────┼───────┤ 64000 │  100   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.22   │ 0.28  │  0.79  │  893   │  1151  │  1176338  │  1638874  │   0    │   0   │  87   │  231  │ 2829  │  3.87  │ 0.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.23   │ 0.27  │  0.85  │  929   │  1210  │  1242026  │  1708182  │   0    │   0   │  94   │  226  │ 2726  │  4.11  │ 0.44  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.19   │ 0.27  │  0.70  │  1035  │  1343  │  1399732  │  1913629  │   0    │   0   │  95   │  233  │ 2760  │  4.44  │ 0.49  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.25   │ 0.23  │  1.09  │  981   │  1335  │  1322786  │  1918634  │   0    │   0   │  91   │  231  │ 4053  │  4.25  │ 0.33  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.20   │ 0.29  │  0.69  │  794   │  1071  │  1043988  │  1506348  │   0    │   0   │  79   │  241  │ 4190  │  3.29  │ 0.26  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.28   │ 0.25  │  1.12  │  1102  │  1488  │  1496940  │  2123758  │   0    │   0   │  109  │  228  │ 2693  │  4.83  │ 0.55  │
> ├───────┼───────┤ 64000 │  250   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.27   │ 0.26  │  1.04  │  1037  │  1398  │  1413514  │  1985504  │   0    │   0   │  80   │  214  │ 2687  │  4.85  │ 0.52  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.27   │ 0.28  │  0.96  │  1040  │  1393  │  1412984  │  1985046  │   0    │   0   │  87   │  224  │ 2688  │  4.64  │ 0.52  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.25   │ 0.26  │  0.96  │  990   │  1337  │  1338042  │  1903858  │   0    │   0   │  89   │  227  │ 3262  │  4.36  │ 0.41  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.22   │ 0.23  │  0.96  │  912   │  1164  │  1220760  │  1641364  │   0    │   0   │  84   │  220  │ 2775  │  4.15  │ 0.42  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.00   │ 0.28  │  0.00  │  1091  │  1448  │  1478750  │  2057400  │   0    │   0   │  92   │  234  │ 2623  │  4.66  │ 0.55  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.20   │ 0.21  │  0.95  │  794   │  1069  │  1056788  │  1505966  │   0    │   0   │  83   │  224  │ 3825  │  3.54  │ 0.28  │
> ├───────┼───────┤ 64000 │  500   ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.23   │ 0.24  │  0.96  │  964   │  1257  │  1309096  │  1774516  │   0    │   0   │  93   │  231  │ 2740  │  4.17  │ 0.46  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │   0.10   │ 0.26  │  0.38  │  464   │  596   │  582264   │  822450   │   0    │   0   │  59   │  191  │ 2775  │  2.43  │ 0.21  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │   0.15   │ 0.24  │  0.62  │  845   │  1106  │  1129531  │  1560339  │   0    │   0   │  82   │  220  │ 2947  │  3.84  │ 0.38  │
> ├───────┴───────┴───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │  Category:TCP_MAERTS  │        │          │       │        │        │        │           │           │        │       │       │       │       │        │       │
> ├───────┬───────┬───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   #   │ Tile  │ size  │sessions│throughput│ %CPU  │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts  │ #rx-byts  │   #    │   #   │   #   │   #   │   #   │#tpkt/# │#rpkt/#│
> │       │       │       │        │          │       │        │        │        │           │           │re-trans│tx-intr│rx-intr│io_exit│irq_inj│  exit  │  irq  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2924.86  │ 20.37 │ 143.59 │15153060│ 596993 │22939024640│ 39401828  │   1    │ 65442 │ 23813 │ 1027  │110030 │14754.68│ 5.43  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3000.11  │ 20.66 │ 145.21 │15545222│ 591445 │23528740948│ 39035672  │   3    │ 66561 │ 25393 │ 1059  │114212 │14679.15│ 5.18  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3136.45  │ 20.60 │ 152.25 │16248525│ 604735 │24598009138│ 39912824  │   3    │ 69224 │ 28870 │ 1268  │119171 │12814.29│ 5.07  │
> ├───────┼───────┤ 4000  │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3078.63  │ 20.27 │ 151.88 │15947778│ 606104 │24144602852│ 40003130  │   0    │ 69303 │ 25358 │ 1131  │114809 │14100.60│ 5.28  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3080.67  │ 20.61 │ 149.47 │15961524│ 595110 │24159814912│ 39277610  │   1    │ 67755 │ 28011 │ 1094  │117709 │14590.06│ 5.06  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3044.14  │ 20.50 │ 148.49 │15771221│ 598877 │23874038498│ 39526212  │   1    │ 67657 │ 26289 │ 1115  │115186 │14144.59│ 5.20  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2965.27  │ 20.74 │ 142.97 │15363170│ 790720 │23255408228│ 52188166  │   0    │ 65418 │ 26707 │ 1468  │113316 │10465.37│ 6.98  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2947.36  │ 20.97 │ 140.55 │15268548│ 794787 │23115454336│ 52456486  │   0    │ 66070 │ 26903 │  700  │114160 │21812.21│ 6.96  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2960.73  │ 21.41 │ 138.29 │15339370│ 751896 │23221092668│ 49625686  │   0    │ 65545 │ 25601 │  811  │115892 │18914.14│ 6.49  │
> ├───────┼───────┤ 4000  │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 5907.73  │ 20.45 │ 288.89 │30610205│1177641 │46333281674│ 77724922  │   2    │125770 │ 51857 │ 5667  │219300 │5401.48 │ 5.37  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3032.53  │ 20.92 │ 144.96 │15711016│ 772496 │23784940232│ 50985322  │   0    │ 68181 │ 28487 │  809  │118180 │19420.29│ 6.54  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3562.72  │ 20.90 │ 170.47 │18458461│ 857508 │27942035427│ 56596116  │   0    │ 78196 │ 31911 │ 1891  │136169 │9761.22 │ 6.30  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2852.03  │ 21.36 │ 133.52 │14777845│1191964 │22369351906│ 78670742  │   0    │ 62091 │ 28441 │  556  │113287 │26578.86│ 10.52 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3040.04  │ 21.94 │ 138.56 │15751006│1139198 │23844389428│ 75188192  │   0    │ 65927 │ 30407 │  555  │122201 │28380.19│ 9.32  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2887.70  │ 21.77 │ 132.65 │14961461│1109226 │22649146810│ 73210034  │   0    │ 62700 │ 29235 │  539  │116775 │27757.81│ 9.50  │
> ├───────┼───────┤ 4000  │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2895.58  │ 21.13 │ 137.04 │15002968│1186035 │22711234744│ 78279392  │   0    │ 63732 │ 29783 │  518  │115563 │28963.26│ 10.26 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2879.21  │ 21.68 │ 132.80 │14916915│1173726 │22582416798│ 77467046  │   0    │ 62743 │ 27524 │  507  │115038 │29421.92│ 10.20 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 2910.91  │ 21.58 │ 134.89 │15082039│1160029 │22831307937│ 76563081  │   0    │ 63438 │ 29078 │  535  │116572 │28190.73│ 9.95  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3049.48  │ 20.40 │ 149.48 │15801215│ 572190 │23915846174│ 37764806  │   0    │ 66915 │ 26969 │ 1112  │113792 │14209.73│ 5.03  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3105.77  │ 20.16 │ 154.06 │16092598│ 573938 │24356811972│ 37880198  │   0    │ 68960 │ 27455 │ 1068  │115385 │15067.98│ 4.97  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3024.33  │ 20.97 │ 144.22 │15672214│ 571122 │23719355572│ 37694330  │   0    │ 66545 │ 25304 │ 1147  │113705 │13663.66│ 5.02  │
> ├───────┼───────┤ 20000 │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3128.98  │ 20.36 │ 153.68 │16213305│ 574600 │24539185562│ 37923866  │   0    │ 68775 │ 27714 │ 1085  │116210 │14943.14│ 4.94  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3015.91  │ 19.82 │ 152.16 │15628040│ 570824 │23652582544│ 37674710  │   0    │ 66952 │ 25248 │ 1043  │111470 │14983.74│ 5.12  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3064.89  │ 20.34 │ 150.68 │15881474│ 572534 │24036756364│ 37787582  │   0    │ 67629 │ 26538 │ 1091  │114112 │14556.80│ 5.02  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3017.86  │ 20.57 │ 146.71 │15635768│ 770376 │23670471288│ 50845462  │   0    │ 66599 │ 28225 │  971  │114210 │16102.75│ 6.75  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2915.46  │ 20.22 │ 144.19 │15104449│ 847355 │22865454450│ 55925980  │   0    │ 65162 │ 25640 │  730  │108818 │20691.03│ 7.79  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2989.10  │ 20.45 │ 146.17 │15485617│ 789257 │23444217290│ 52091512  │   0    │ 66846 │ 27968 │  739  │113575 │20954.83│ 6.95  │
> ├───────┼───────┤ 20000 │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2959.78  │ 20.17 │ 146.74 │15334182│ 823187 │23213729212│ 54330940  │   0    │ 65965 │ 26989 │  706  │111125 │21719.80│ 7.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3011.53  │ 20.44 │ 147.34 │15601291│ 794027 │23619413438│ 52406344  │   0    │ 67510 │ 28280 │  716  │114074 │21789.51│ 6.96  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 2978.75  │ 20.37 │ 146.23 │15432261│ 804840 │23362657135│ 53120047  │   0    │ 66416 │ 27420 │  772  │112360 │19989.98│ 7.16  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 5919.16  │ 20.64 │ 286.78 │30669866│1573766 │46423091212│ 103869704 │   0    │129892 │ 53972 │ 1393  │221189 │22017.13│ 7.12  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2932.03  │ 20.71 │ 141.58 │15190875│1191597 │22995985998│ 78646496  │   0    │ 64535 │ 31583 │  519  │115086 │29269.51│ 10.35 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2883.39  │ 20.89 │ 138.03 │14937997│1211603 │22614678186│ 79966964  │   3    │ 62928 │ 29990 │  519  │112534 │28782.27│ 10.77 │
> ├───────┼───────┤ 20000 │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2902.41  │ 20.85 │ 139.20 │15037120│1181920 │22764686232│ 78007886  │   0    │ 63619 │ 30671 │  554  │113895 │27142.82│ 10.38 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2874.86  │ 20.73 │ 138.68 │14894278│1214765 │22548743420│ 80175596  │   0    │ 63378 │ 29579 │  529  │111854 │28155.53│ 10.86 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3502.37  │ 20.76 │ 168.71 │18146027│1274730 │27469437009│ 84133329  │   0    │ 76870 │ 35159 │  702  │134911 │25849.04│ 9.45  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3026.53  │ 21.15 │ 143.10 │15682830│ 570773 │23737051676│ 37671368  │   3    │ 64978 │ 25555 │ 1230  │112859 │12750.27│ 5.06  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3007.50  │ 19.92 │ 150.98 │15582909│ 570840 │23586592498│ 37675730  │   0    │ 66487 │ 26878 │ 1043  │112219 │14940.47│ 5.09  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3051.44  │ 20.19 │ 151.14 │15810348│ 571137 │23930723472│ 37695380  │   0    │ 67459 │ 27639 │ 1055  │114241 │14986.11│ 5.00  │
> ├───────┼───────┤ 40000 │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3104.11  │ 20.32 │ 152.76 │16084451│ 571867 │24344925270│ 37743524  │   1    │ 68005 │ 27596 │ 1153  │114806 │13950.09│ 4.98  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2994.81  │ 19.77 │ 151.48 │15518401│ 569056 │23487784002│ 37558022  │   0    │ 66556 │ 24318 │ 1012  │109020 │15334.39│ 5.22  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3036.88  │ 20.27 │ 149.82 │15735787│ 570734 │23817415383│ 37668804  │   0    │ 66697 │ 26397 │ 1098  │112629 │14331.32│ 5.07  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3002.65  │ 20.07 │ 149.61 │15555724│ 800442 │23549407192│ 52829722  │   0    │ 66232 │ 28291 │  777  │112836 │20020.24│ 7.09  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2993.13  │ 20.31 │ 147.37 │15506325│ 798478 │23473944914│ 52700182  │   0    │ 66886 │ 28241 │  775  │112960 │20008.16│ 7.07  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3071.56  │ 20.86 │ 147.25 │15913500│ 774753 │24089503224│ 51134320  │   0    │ 67559 │ 26368 │ 1347  │114552 │11814.03│ 6.76  │
> ├───────┼───────┤ 40000 │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 6046.40  │ 20.19 │ 299.47 │31335834│1138243 │47431866492│ 75124678  │   0    │132886 │ 53864 │ 1767  │223038 │17733.92│ 5.10  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2947.26  │ 20.63 │ 142.86 │15267540│ 729667 │23114728840│ 48158608  │   2    │ 65443 │ 26431 │ 1386  │111787 │11015.54│ 6.53  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3612.20  │ 20.41 │ 176.98 │18715784│ 848316 │28331890132│ 55989502  │   0    │ 79801 │ 32639 │ 1210  │135034 │15467.59│ 6.28  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2807.88  │ 20.79 │ 135.06 │14547494│1208116 │22024180476│ 79736750  │   0    │ 61306 │ 29138 │  572  │109567 │25432.68│ 11.03 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2917.26  │ 20.27 │ 143.92 │15114879│1255244 │22880404702│ 82847234  │   0    │ 63728 │ 30183 │ 1436  │112031 │10525.68│ 11.20 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2916.93  │ 20.77 │ 140.44 │15111738│1197709 │22877637916│ 79049918  │   0    │ 63861 │ 29963 │  508  │113011 │29747.52│ 10.60 │
> ├───────┼───────┤ 40000 │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 5951.84  │ 20.76 │ 286.70 │30837735│1621963 │46682529726│ 107050718 │   0    │129575 │ 53550 │ 1912  │219019 │16128.52│ 7.41  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2875.91  │ 20.45 │ 140.63 │14899829│1215386 │22556298466│ 80216570  │   0    │ 63602 │ 30811 │  517  │113055 │28819.79│ 10.75 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3493.96  │ 20.61 │ 169.53 │18102335│1299683 │27404210257│ 85780238  │   0    │ 76414 │ 34729 │  989  │133336 │18303.68│ 9.75  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2971.65  │ 20.18 │ 147.26 │15397768│ 569660 │23305187480│ 37597886  │   0    │ 65136 │ 25160 │ 1025  │109390 │15022.21│ 5.21  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3039.73  │ 20.06 │ 151.53 │15749554│ 571308 │23838836620│ 37706594  │   0    │ 67399 │ 27241 │ 1082  │113354 │14555.96│ 5.04  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3167.18  │ 20.24 │ 156.48 │16409368│ 575765 │24838314384│ 38000750  │   0    │ 69998 │ 28670 │ 1083  │118643 │15151.77│ 4.85  │
> ├───────┼───────┤ 64000 │   1    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3011.43  │ 20.18 │ 149.23 │15604231│ 569984 │23617080958│ 37619234  │   0    │ 66146 │ 25188 │ 1053  │110588 │14818.83│ 5.15  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3109.37  │ 20.19 │ 154.01 │16111119│ 572426 │24385638358│ 37780382  │   0    │ 69092 │ 27567 │ 1114  │116245 │14462.40│ 4.92  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3059.87  │ 20.17 │ 151.70 │15854408│ 571828 │23997011560│ 37740969  │   0    │ 67554 │ 26765 │ 1071  │113644 │14803.37│ 5.03  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3002.80  │ 20.96 │ 143.26 │15557039│ 775172 │23551006550│ 51161914  │   0    │ 66128 │ 26892 │  807  │113574 │19277.62│ 6.83  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3054.32  │ 20.94 │ 145.86 │15824738│ 771426 │23955007524│ 50914678  │   1    │ 67755 │ 26255 │  751  │114455 │21071.56│ 6.74  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 5946.64  │ 19.96 │ 297.93 │30816873│1130615 │46642948666│ 74621242  │   0    │128038 │ 51569 │ 2330  │216705 │13226.13│ 5.22  │
> ├───────┼───────┤ 64000 │   2    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2932.97  │ 20.34 │ 144.20 │15194209│ 803587 │23002830338│ 53037304  │   0    │ 65293 │ 26419 │  686  │109847 │22148.99│ 7.32  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3052.39  │ 20.66 │ 147.74 │15813155│ 797801 │23938511030│ 52655464  │   0    │ 68003 │ 27538 │  802  │115045 │19717.15│ 6.93  │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 3597.82  │ 20.57 │ 174.91 │18641202│ 855720 │28218060821│ 56478120  │   0    │ 79043 │ 31734 │ 1075  │133925 │17340.65│ 6.39  │
> ├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2838.78  │ 21.19 │ 133.97 │14708072│1132027 │22266304904│ 74714924  │   0    │ 61319 │ 28323 │  548  │110491 │26839.55│ 10.25 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2802.43  │ 20.80 │ 134.73 │14518769│1154549 │21980236458│ 76201340  │   0    │ 61500 │ 28722 │  540  │109217 │26886.61│ 10.57 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3050.89  │ 21.62 │ 141.11 │15807079│1131706 │23930034342│ 74693732  │   0    │ 64955 │ 29623 │  532  │117696 │29712.55│ 9.62  │
> ├───────┼───────┤ 64000 │   4    ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 2929.79  │ 21.40 │ 136.91 │15179680│1166291 │22978798720│ 76976300  │   0    │ 62978 │ 29165 │  608  │114391 │24966.58│ 10.20 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │       │       │       │        │ 3002.34  │ 21.01 │ 142.90 │15554591│1189606 │23548582838│ 78515114  │   0    │ 65970 │ 31366 │  511  │116996 │30439.51│ 10.17 │
> ├───────┼───────┤       │        ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
> │   -   │  Avg  │       │        │ 2924.85  │ 21.20 │ 137.96 │15153638│1154835 │22940791452│ 76220282  │   0    │ 63344 │ 29439 │  547  │113758 │27703.18│ 10.15 │
> └───────┴───────┴───────┴────────┴──────────┴───────┴────────┴────────┴────────┴───────────┴───────────┴────────┴───────┴───────┴───────┴───────┴────────┴───────┘

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Eric Dumazet]

On Wed, 2014-01-08 at 19:21 +0200, Michael S. Tsirkin wrote:

> Basically yes, we could start dropping packets immediately
> once GFP_ATOMIC allocations fail and repost the buffer to host,
> and hope memory is available by the time we get the next interrupt.

> But we wanted host to have visibility into the fact that
> we are out of memory and packets are dropped, so we did not want to
> repost.

bufferbloat alert :)

> If we don't repost how do we know memory is finally available?
> We went for a timer based workqueue thing.
> What do you suggest?


In normal networking land, when a host A sends frames to host B,
nothing prevents A to pause the traffic to B if B is dropping packets
under stress.

A physical NIC do not use a workqueue to refill its RX queue but uses
the following strategy :

0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
   allocations with all needed (sleep/retry/shout) logic...
1) IRQ is handled.
2) Can we allocate a new buffer (GFP_ATOMIC) ?
   If yes, we accept the frame,
      and post the new buffer for the 'next frame'
   If no, we drop the frame and recycle the memory for next round.

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Michael S. Tsirkin]

On Wed, Jan 08, 2014 at 10:09:47AM -0800, Eric Dumazet wrote:
> On Wed, 2014-01-08 at 19:21 +0200, Michael S. Tsirkin wrote:
> 
> > Basically yes, we could start dropping packets immediately
> > once GFP_ATOMIC allocations fail and repost the buffer to host,
> > and hope memory is available by the time we get the next interrupt.
> 
> > But we wanted host to have visibility into the fact that
> > we are out of memory and packets are dropped, so we did not want to
> > repost.
> 
> bufferbloat alert :)
>

I guess you are saying we never need to signal host/device
that we are out of memory, it's enough that packets are dropped?
It seemed like a useful thing for hypervisor to know about on general
principles, even though I don't think kvm uses this info at this point.

> > If we don't repost how do we know memory is finally available?
> > We went for a timer based workqueue thing.
> > What do you suggest?
> 
> 
> In normal networking land, when a host A sends frames to host B,
> nothing prevents A to pause the traffic to B if B is dropping packets
> under stress.
> 
> A physical NIC do not use a workqueue to refill its RX queue but uses
> the following strategy :
> 
> 0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
>    allocations with all needed (sleep/retry/shout) logic...
> 1) IRQ is handled.
> 2) Can we allocate a new buffer (GFP_ATOMIC) ?
>    If yes, we accept the frame,
>       and post the new buffer for the 'next frame'
>    If no, we drop the frame and recycle the memory for next round.
> 

Exactly, this is what I tried to describe in the part that
you have snipped out - but this means queue is always full.

Also, I wonder whether allocating before passing
frame to the stack might slow us down a tiny bit e.g. if an application
is polling this socket on another CPU.

Maybe a slightly better strategy is to do the above when queue depth
is running low. E.g. when queue is 3/4 empty, try
allocating before giving frames to net core,
and recycle buffers on error.

Not sure how much of a win this is.

-- 
MST

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[David Miller]

From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Wed, 08 Jan 2014 10:09:47 -0800

> A physical NIC do not use a workqueue to refill its RX queue but uses
> the following strategy :
> 
> 0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
>    allocations with all needed (sleep/retry/shout) logic...
> 1) IRQ is handled.
> 2) Can we allocate a new buffer (GFP_ATOMIC) ?
>    If yes, we accept the frame,
>       and post the new buffer for the 'next frame'
>    If no, we drop the frame and recycle the memory for next round.

+1

Re: [PATCH net-next 2/3] virtio-net: use per-receive queue page frag alloc for mergeable bufs

[Rick Jones]

On 01/08/2014 10:09 AM, Eric Dumazet wrote:
> In normal networking land, when a host A sends frames to host B,
> nothing prevents A to pause the traffic to B if B is dropping packets
> under stress.
>
> A physical NIC do not use a workqueue to refill its RX queue but uses
> the following strategy :
>
> 0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
>     allocations with all needed (sleep/retry/shout) logic...
> 1) IRQ is handled.
> 2) Can we allocate a new buffer (GFP_ATOMIC) ?
>     If yes, we accept the frame,
>        and post the new buffer for the 'next frame'
>     If no, we drop the frame and recycle the memory for next round.

          and increment a suitably specific statistic so someone trying 
to diagnose performance/other problems can know we dropped the frame.

rick jones