Discussion:
[PATCH net-next 1/3] net: allow > 0 order atomic page alloc in skb_page_frag_refill
(too old to reply)
Michael Dalton
2013-12-17 00:16:27 UTC
Permalink
skb_page_frag_refill currently permits only order-0 page allocs
unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
higher-order page allocations whether or not GFP_WAIT is used. If
memory cannot be allocated, the allocator will fall back to
successively smaller page allocs (down to order-0 page allocs).

This change brings skb_page_frag_refill in line with the existing
page allocation strategy employed by netdev_alloc_frag, which attempts
higher-order page allocations whether or not GFP_WAIT is set, falling
back to successively lower-order page allocations on failure. Part
of migration of virtio-net to per-receive queue page frag allocators.

Signed-off-by: Michael Dalton <***@google.com>
---
net/core/sock.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/net/core/sock.c b/net/core/sock.c
index ab20ed9..7383d23 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
put_page(pfrag->page);
}

- /* We restrict high order allocations to users that can afford to wait */
- order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
-
+ order = SKB_FRAG_PAGE_ORDER;
do {
gfp_t gfp = prio;
--
1.8.5.1
Michael Dalton
2013-12-17 00:16:28 UTC
Permalink
The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
mergeable rx buffer allocations. This commit migrates virtio-net to use
per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
mergeable rx buffer memory allocation, which now will use skb_refill_frag()
for both atomic and GFP-WAIT buffer allocations.

To address fragmentation concerns, if after buffer allocation there
is too little space left in the page frag to allocate a subsequent
buffer, the remaining space is added to the current allocated buffer
so that the remaining space can be used to store packet data.

Signed-off-by: Michael Dalton <***@google.com>
---
drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
1 file changed, 38 insertions(+), 31 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a988..d38d130 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -78,6 +78,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;

+ /* Page frag for GFP_ATOMIC packet buffer allocation. */
+ struct page_frag atomic_frag;
+
/* RX: fragments + linear part + virtio header */
struct scatterlist sg[MAX_SKB_FRAGS + 2];

@@ -127,9 +130,9 @@ struct virtnet_info {
struct mutex config_lock;

/* Page_frag for GFP_KERNEL packet buffer allocation when we run
- * low on memory.
+ * low on memory. May sleep.
*/
- struct page_frag alloc_frag;
+ struct page_frag sleep_frag;

/* Does the affinity hint is set for virtqueues? */
bool affinity_hint_set;
@@ -336,8 +339,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
int num_buf = hdr->mhdr.num_buffers;
struct page *page = virt_to_head_page(buf);
int offset = buf - page_address(page);
- struct sk_buff *head_skb = page_to_skb(rq, page, offset, len,
- MERGE_BUFFER_LEN);
+ int truesize = max_t(int, len, MERGE_BUFFER_LEN);
+ struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
struct sk_buff *curr_skb = head_skb;

if (unlikely(!curr_skb))
@@ -353,11 +356,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
dev->stats.rx_length_errors++;
goto err_buf;
}
- if (unlikely(len > MERGE_BUFFER_LEN)) {
- pr_debug("%s: rx error: merge buffer too long\n",
- dev->name);
- len = MERGE_BUFFER_LEN;
- }

page = virt_to_head_page(buf);
--rq->num;
@@ -376,19 +374,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
head_skb->truesize += nskb->truesize;
num_skb_frags = 0;
}
+ truesize = max_t(int, len, MERGE_BUFFER_LEN);
if (curr_skb != head_skb) {
head_skb->data_len += len;
head_skb->len += len;
- head_skb->truesize += MERGE_BUFFER_LEN;
+ head_skb->truesize += truesize;
}
offset = buf - page_address(page);
if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
put_page(page);
skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
- len, MERGE_BUFFER_LEN);
+ len, truesize);
} else {
skb_add_rx_frag(curr_skb, num_skb_frags, page,
- offset, len, MERGE_BUFFER_LEN);
+ offset, len, truesize);
}
}

@@ -579,24 +578,24 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
- char *buf = NULL;
- int err;
+ struct page_frag *alloc_frag;
+ char *buf;
+ int err, len, hole;

- if (gfp & __GFP_WAIT) {
- if (skb_page_frag_refill(MERGE_BUFFER_LEN, &vi->alloc_frag,
- gfp)) {
- buf = (char *)page_address(vi->alloc_frag.page) +
- vi->alloc_frag.offset;
- get_page(vi->alloc_frag.page);
- vi->alloc_frag.offset += MERGE_BUFFER_LEN;
- }
- } else {
- buf = netdev_alloc_frag(MERGE_BUFFER_LEN);
- }
- if (!buf)
+ alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
+ if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
return -ENOMEM;
+ buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
+ get_page(alloc_frag->page);
+ len = MERGE_BUFFER_LEN;
+ alloc_frag->offset += len;
+ hole = alloc_frag->size - alloc_frag->offset;
+ if (hole < MERGE_BUFFER_LEN) {
+ len += hole;
+ alloc_frag->offset += hole;
+ }

- sg_init_one(rq->sg, buf, MERGE_BUFFER_LEN);
+ sg_init_one(rq->sg, buf, len);
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
put_page(virt_to_head_page(buf));
@@ -1377,6 +1376,16 @@ static void free_receive_bufs(struct virtnet_info *vi)
}
}

+static void free_receive_page_frags(struct virtnet_info *vi)
+{
+ int i;
+ for (i = 0; i < vi->max_queue_pairs; i++)
+ if (vi->rq[i].atomic_frag.page)
+ put_page(vi->rq[i].atomic_frag.page);
+ if (vi->sleep_frag.page)
+ put_page(vi->sleep_frag.page);
+}
+
static void free_unused_bufs(struct virtnet_info *vi)
{
void *buf;
@@ -1706,8 +1715,7 @@ free_recv_bufs:
free_vqs:
cancel_delayed_work_sync(&vi->refill);
virtnet_del_vqs(vi);
- if (vi->alloc_frag.page)
- put_page(vi->alloc_frag.page);
+ free_receive_page_frags(vi);
free_stats:
free_percpu(vi->stats);
free:
@@ -1741,8 +1749,7 @@ static void virtnet_remove(struct virtio_device *vdev)
unregister_netdev(vi->dev);

remove_vq_common(vi);
- if (vi->alloc_frag.page)
- put_page(vi->alloc_frag.page);
+ free_receive_page_frags(vi);

flush_work(&vi->config_work);
--
1.8.5.1
Jason Wang
2013-12-23 08:12:21 UTC
Permalink
Post by Michael Dalton
The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
mergeable rx buffer allocations. This commit migrates virtio-net to use
per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
mergeable rx buffer memory allocation, which now will use skb_refill_frag()
for both atomic and GFP-WAIT buffer allocations.
To address fragmentation concerns, if after buffer allocation there
is too little space left in the page frag to allocate a subsequent
buffer, the remaining space is added to the current allocated buffer
so that the remaining space can be used to store packet data.
---
drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
1 file changed, 38 insertions(+), 31 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a988..d38d130 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -78,6 +78,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;
+ /* Page frag for GFP_ATOMIC packet buffer allocation. */
+ struct page_frag atomic_frag;
+
/* RX: fragments + linear part + virtio header */
struct scatterlist sg[MAX_SKB_FRAGS + 2];
@@ -127,9 +130,9 @@ struct virtnet_info {
struct mutex config_lock;
/* Page_frag for GFP_KERNEL packet buffer allocation when we run
- * low on memory.
+ * low on memory. May sleep.
*/
- struct page_frag alloc_frag;
+ struct page_frag sleep_frag;
Any reason to use two different page_frag consider only
skb_page_frag_refill() is used?
Post by Michael Dalton
/* Does the affinity hint is set for virtqueues? */
bool affinity_hint_set;
@@ -336,8 +339,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
int num_buf = hdr->mhdr.num_buffers;
struct page *page = virt_to_head_page(buf);
int offset = buf - page_address(page);
- struct sk_buff *head_skb = page_to_skb(rq, page, offset, len,
- MERGE_BUFFER_LEN);
+ int truesize = max_t(int, len, MERGE_BUFFER_LEN);
+ struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
struct sk_buff *curr_skb = head_skb;
if (unlikely(!curr_skb))
@@ -353,11 +356,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
dev->stats.rx_length_errors++;
goto err_buf;
}
- if (unlikely(len > MERGE_BUFFER_LEN)) {
- pr_debug("%s: rx error: merge buffer too long\n",
- dev->name);
- len = MERGE_BUFFER_LEN;
- }
page = virt_to_head_page(buf);
--rq->num;
@@ -376,19 +374,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
head_skb->truesize += nskb->truesize;
num_skb_frags = 0;
}
+ truesize = max_t(int, len, MERGE_BUFFER_LEN);
if (curr_skb != head_skb) {
head_skb->data_len += len;
head_skb->len += len;
- head_skb->truesize += MERGE_BUFFER_LEN;
+ head_skb->truesize += truesize;
}
offset = buf - page_address(page);
if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
put_page(page);
skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
- len, MERGE_BUFFER_LEN);
+ len, truesize);
} else {
skb_add_rx_frag(curr_skb, num_skb_frags, page,
- offset, len, MERGE_BUFFER_LEN);
+ offset, len, truesize);
}
}
@@ -579,24 +578,24 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
- char *buf = NULL;
- int err;
+ struct page_frag *alloc_frag;
+ char *buf;
+ int err, len, hole;
- if (gfp & __GFP_WAIT) {
- if (skb_page_frag_refill(MERGE_BUFFER_LEN, &vi->alloc_frag,
- gfp)) {
- buf = (char *)page_address(vi->alloc_frag.page) +
- vi->alloc_frag.offset;
- get_page(vi->alloc_frag.page);
- vi->alloc_frag.offset += MERGE_BUFFER_LEN;
- }
- } else {
- buf = netdev_alloc_frag(MERGE_BUFFER_LEN);
- }
- if (!buf)
+ alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
+ if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
return -ENOMEM;
+ buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
+ get_page(alloc_frag->page);
+ len = MERGE_BUFFER_LEN;
+ alloc_frag->offset += len;
+ hole = alloc_frag->size - alloc_frag->offset;
+ if (hole < MERGE_BUFFER_LEN) {
+ len += hole;
+ alloc_frag->offset += hole;
+ }
- sg_init_one(rq->sg, buf, MERGE_BUFFER_LEN);
+ sg_init_one(rq->sg, buf, len);
I wonder whether we can use get_a_page() and give_pages() to recycle the
pages like before which may help the performance. We can also do some
optimizations for this in vhost.
Post by Michael Dalton
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
put_page(virt_to_head_page(buf));
@@ -1377,6 +1376,16 @@ static void free_receive_bufs(struct virtnet_info *vi)
}
}
+static void free_receive_page_frags(struct virtnet_info *vi)
+{
+ int i;
+ for (i = 0; i < vi->max_queue_pairs; i++)
+ if (vi->rq[i].atomic_frag.page)
+ put_page(vi->rq[i].atomic_frag.page);
+ if (vi->sleep_frag.page)
+ put_page(vi->sleep_frag.page);
+}
+
static void free_unused_bufs(struct virtnet_info *vi)
{
void *buf;
cancel_delayed_work_sync(&vi->refill);
virtnet_del_vqs(vi);
- if (vi->alloc_frag.page)
- put_page(vi->alloc_frag.page);
+ free_receive_page_frags(vi);
free_percpu(vi->stats);
@@ -1741,8 +1749,7 @@ static void virtnet_remove(struct virtio_device *vdev)
unregister_netdev(vi->dev);
remove_vq_common(vi);
- if (vi->alloc_frag.page)
- put_page(vi->alloc_frag.page);
+ free_receive_page_frags(vi);
flush_work(&vi->config_work);
Eric Dumazet
2013-12-23 17:27:07 UTC
Permalink
Post by Jason Wang
Post by Michael Dalton
The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
mergeable rx buffer allocations. This commit migrates virtio-net to use
per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
mergeable rx buffer memory allocation, which now will use skb_refill_frag()
for both atomic and GFP-WAIT buffer allocations.
To address fragmentation concerns, if after buffer allocation there
is too little space left in the page frag to allocate a subsequent
buffer, the remaining space is added to the current allocated buffer
so that the remaining space can be used to store packet data.
---
drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
1 file changed, 38 insertions(+), 31 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a988..d38d130 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -78,6 +78,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;
+ /* Page frag for GFP_ATOMIC packet buffer allocation. */
+ struct page_frag atomic_frag;
+
/* RX: fragments + linear part + virtio header */
struct scatterlist sg[MAX_SKB_FRAGS + 2];
@@ -127,9 +130,9 @@ struct virtnet_info {
struct mutex config_lock;
/* Page_frag for GFP_KERNEL packet buffer allocation when we run
- * low on memory.
+ * low on memory. May sleep.
*/
- struct page_frag alloc_frag;
+ struct page_frag sleep_frag;
Any reason to use two different page_frag consider only
skb_page_frag_refill() is used?
One is used under process context, where preemption and GFP_KERNEL are
allowed.

One is used from softirq context and GFP_ATOMIC. You cant share a common
page_frag.

Acked-by: Eric Dumazet <***@google.com>
Michael S. Tsirkin
2013-12-23 19:37:04 UTC
Permalink
Post by Eric Dumazet
Post by Jason Wang
Post by Michael Dalton
The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
mergeable rx buffer allocations. This commit migrates virtio-net to use
per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
mergeable rx buffer memory allocation, which now will use skb_refill_frag()
for both atomic and GFP-WAIT buffer allocations.
To address fragmentation concerns, if after buffer allocation there
is too little space left in the page frag to allocate a subsequent
buffer, the remaining space is added to the current allocated buffer
so that the remaining space can be used to store packet data.
---
drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
1 file changed, 38 insertions(+), 31 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a988..d38d130 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -78,6 +78,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;
+ /* Page frag for GFP_ATOMIC packet buffer allocation. */
+ struct page_frag atomic_frag;
+
/* RX: fragments + linear part + virtio header */
struct scatterlist sg[MAX_SKB_FRAGS + 2];
@@ -127,9 +130,9 @@ struct virtnet_info {
struct mutex config_lock;
/* Page_frag for GFP_KERNEL packet buffer allocation when we run
- * low on memory.
+ * low on memory. May sleep.
*/
- struct page_frag alloc_frag;
+ struct page_frag sleep_frag;
Any reason to use two different page_frag consider only
skb_page_frag_refill() is used?
One is used under process context, where preemption and GFP_KERNEL are
allowed.
Yes but it is always used with napi disabled.
Post by Eric Dumazet
One is used from softirq context and GFP_ATOMIC.
This one is used only under napi.
Post by Eric Dumazet
You cant share a common
page_frag.
So there isn't a conflict with respect to locking.

Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?
Michael Dalton
2013-12-26 21:28:58 UTC
Permalink
Post by Michael S. Tsirkin
So there isn't a conflict with respect to locking.
Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?
I believe it is safe to use the same page_frag and I will send out a
followup patchset using just the per-receive page_frags. For future
consideration, Eric noted that disabling NAPI before GFP_KERNEL
allocs can potentially inhibit virtio-net network processing for some
time (e.g., during a blocking memory allocation or preemption).

Best,

Mike
Michael S. Tsirkin
2013-12-26 21:37:27 UTC
Permalink
Post by Michael Dalton
Post by Michael S. Tsirkin
So there isn't a conflict with respect to locking.
Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?
I believe it is safe to use the same page_frag and I will send out a
followup patchset using just the per-receive page_frags.
Seems easier to use it straight away I think.
Post by Michael Dalton
For future
consideration, Eric noted that disabling NAPI before GFP_KERNEL
allocs can potentially inhibit virtio-net network processing for some
time (e.g., during a blocking memory allocation or preemption).
Best,
Mike
Interesting. But if we can't allocate a buffer how can we
do network processing?

If we can reproduce the problem, we can maybe move
allocation out of napi disabled section, but then
we'll need to add more locking.
Eric Dumazet
2013-12-26 22:00:58 UTC
Permalink
Post by Michael S. Tsirkin
Interesting. But if we can't allocate a buffer how can we
do network processing?
How typical NIC drivers handle this case ?

Answer : nothing special should happen, we drop incoming traffic,
and make sure the driver recovers properly. (like not NULL deref or
crazy things like that)

Why virtio_net should be different ?
Post by Michael S. Tsirkin
If we can reproduce the problem, we can maybe move
allocation out of napi disabled section, but then
we'll need to add more locking.
More exactly, use appropriate locking ;)
Michael S. Tsirkin
2014-01-08 17:21:04 UTC
Permalink
Post by Eric Dumazet
Post by Michael S. Tsirkin
Interesting. But if we can't allocate a buffer how can we
do network processing?
How typical NIC drivers handle this case ?
Answer : nothing special should happen, we drop incoming traffic,
and make sure the driver recovers properly. (like not NULL deref or
crazy things like that)
Why virtio_net should be different ?
Basically yes, we could start dropping packets immediately
once GFP_ATOMIC allocations fail and repost the buffer to host,
and hope memory is available by the time we get the next interrupt.
But we wanted host to have visibility into the fact that
we are out of memory and packets are dropped, so we did not want to
repost.
If we don't repost how do we know memory is finally available?
We went for a timer based workqueue thing.
What do you suggest?
Post by Eric Dumazet
Post by Michael S. Tsirkin
If we can reproduce the problem, we can maybe move
allocation out of napi disabled section, but then
we'll need to add more locking.
More exactly, use appropriate locking ;)
Eric Dumazet
2014-01-08 18:09:47 UTC
Permalink
Post by Michael S. Tsirkin
Basically yes, we could start dropping packets immediately
once GFP_ATOMIC allocations fail and repost the buffer to host,
and hope memory is available by the time we get the next interrupt.
But we wanted host to have visibility into the fact that
we are out of memory and packets are dropped, so we did not want to
repost.
bufferbloat alert :)
Post by Michael S. Tsirkin
If we don't repost how do we know memory is finally available?
We went for a timer based workqueue thing.
What do you suggest?
In normal networking land, when a host A sends frames to host B,
nothing prevents A to pause the traffic to B if B is dropping packets
under stress.

A physical NIC do not use a workqueue to refill its RX queue but uses
the following strategy :

0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
allocations with all needed (sleep/retry/shout) logic...
1) IRQ is handled.
2) Can we allocate a new buffer (GFP_ATOMIC) ?
If yes, we accept the frame,
and post the new buffer for the 'next frame'
If no, we drop the frame and recycle the memory for next round.
Michael S. Tsirkin
2014-01-08 18:57:25 UTC
Permalink
Post by Eric Dumazet
Post by Michael S. Tsirkin
Basically yes, we could start dropping packets immediately
once GFP_ATOMIC allocations fail and repost the buffer to host,
and hope memory is available by the time we get the next interrupt.
But we wanted host to have visibility into the fact that
we are out of memory and packets are dropped, so we did not want to
repost.
bufferbloat alert :)
I guess you are saying we never need to signal host/device
that we are out of memory, it's enough that packets are dropped?
It seemed like a useful thing for hypervisor to know about on general
principles, even though I don't think kvm uses this info at this point.
Post by Eric Dumazet
Post by Michael S. Tsirkin
If we don't repost how do we know memory is finally available?
We went for a timer based workqueue thing.
What do you suggest?
In normal networking land, when a host A sends frames to host B,
nothing prevents A to pause the traffic to B if B is dropping packets
under stress.
A physical NIC do not use a workqueue to refill its RX queue but uses
0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
allocations with all needed (sleep/retry/shout) logic...
1) IRQ is handled.
2) Can we allocate a new buffer (GFP_ATOMIC) ?
If yes, we accept the frame,
and post the new buffer for the 'next frame'
If no, we drop the frame and recycle the memory for next round.
Exactly, this is what I tried to describe in the part that
you have snipped out - but this means queue is always full.

Also, I wonder whether allocating before passing
frame to the stack might slow us down a tiny bit e.g. if an application
is polling this socket on another CPU.

Maybe a slightly better strategy is to do the above when queue depth
is running low. E.g. when queue is 3/4 empty, try
allocating before giving frames to net core,
and recycle buffers on error.

Not sure how much of a win this is.
--
MST
David Miller
2014-01-08 19:54:20 UTC
Permalink
From: Eric Dumazet <***@gmail.com>
Date: Wed, 08 Jan 2014 10:09:47 -0800
Post by Eric Dumazet
A physical NIC do not use a workqueue to refill its RX queue but uses
0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
allocations with all needed (sleep/retry/shout) logic...
1) IRQ is handled.
2) Can we allocate a new buffer (GFP_ATOMIC) ?
If yes, we accept the frame,
and post the new buffer for the 'next frame'
If no, we drop the frame and recycle the memory for next round.
+1
Rick Jones
2014-01-08 21:16:44 UTC
Permalink
Post by Eric Dumazet
In normal networking land, when a host A sends frames to host B,
nothing prevents A to pause the traffic to B if B is dropping packets
under stress.
A physical NIC do not use a workqueue to refill its RX queue but uses
0) Pre filling of RX ring buffer with N frames. This can use GFP_KERNEL
allocations with all needed (sleep/retry/shout) logic...
1) IRQ is handled.
2) Can we allocate a new buffer (GFP_ATOMIC) ?
If yes, we accept the frame,
and post the new buffer for the 'next frame'
If no, we drop the frame and recycle the memory for next round.
and increment a suitably specific statistic so someone trying
to diagnose performance/other problems can know we dropped the frame.

rick jones

Eric Dumazet
2013-12-26 21:56:31 UTC
Permalink
Post by Michael Dalton
Post by Michael S. Tsirkin
So there isn't a conflict with respect to locking.
Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?
I believe it is safe to use the same page_frag and I will send out a
followup patchset using just the per-receive page_frags. For future
consideration, Eric noted that disabling NAPI before GFP_KERNEL
allocs can potentially inhibit virtio-net network processing for some
time (e.g., during a blocking memory allocation or preemption).
Yep, using napi_disable() in the refill process looks quite inefficient
to me, it not buggy.

napi_disable() is a big hammer, while whole idea of having a process to
block on GFP_KERNEL allocations is to allow some asynchronous behavior.

I have hard time to convince myself virtio_net is safe anyway with this
work queue thing.

virtnet_open() seems racy for example :

for (i = 0; i < vi->max_queue_pairs; i++) {
if (i < vi->curr_queue_pairs)
/* Make sure we have some buffers: if oom use wq. */
if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
schedule_delayed_work(&vi->refill, 0);
virtnet_napi_enable(&vi->rq[i]);


What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?

refill_work() will happily conflict with another cpu, two cpus could
call try_fill_recv() at the same time, or worse napi_enable() would crash.

I do not have time to make a full check, but I guess there are
other races like this one.

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a98867a40..b8e2adb5d0c2 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -726,16 +726,18 @@ again:
static int virtnet_open(struct net_device *dev)
{
struct virtnet_info *vi = netdev_priv(dev);
+ bool refill = false;
int i;

for (i = 0; i < vi->max_queue_pairs; i++) {
if (i < vi->curr_queue_pairs)
/* Make sure we have some buffers: if oom use wq. */
if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
- schedule_delayed_work(&vi->refill, 0);
+ refill = true;
virtnet_napi_enable(&vi->rq[i]);
}
-
+ if (refill)
+ schedule_delayed_work(&vi->refill, 0);
return 0;
}
Jason Wang
2013-12-27 04:55:43 UTC
Permalink
Post by Eric Dumazet
Post by Michael Dalton
Post by Michael S. Tsirkin
So there isn't a conflict with respect to locking.
Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?
I believe it is safe to use the same page_frag and I will send out a
followup patchset using just the per-receive page_frags. For future
consideration, Eric noted that disabling NAPI before GFP_KERNEL
allocs can potentially inhibit virtio-net network processing for some
time (e.g., during a blocking memory allocation or preemption).
Yep, using napi_disable() in the refill process looks quite inefficient
to me, it not buggy.
napi_disable() is a big hammer, while whole idea of having a process to
block on GFP_KERNEL allocations is to allow some asynchronous behavior.
I have hard time to convince myself virtio_net is safe anyway with this
work queue thing.
for (i = 0; i < vi->max_queue_pairs; i++) {
if (i < vi->curr_queue_pairs)
/* Make sure we have some buffers: if oom use wq. */
if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
schedule_delayed_work(&vi->refill, 0);
virtnet_napi_enable(&vi->rq[i]);
What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?
Then napi_disable() in refill_work() will busy wait until napi is
enabled by virtnet_napi_enable() which looks safe. Looks like the real
issue is in virtnet_restore() who calls try_fill_recv() in neither napi
context nor napi disabled context.
Post by Eric Dumazet
refill_work() will happily conflict with another cpu, two cpus could
call try_fill_recv() at the same time, or worse napi_enable() would crash.
I do not have time to make a full check, but I guess there are
other races like this one.
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a98867a40..b8e2adb5d0c2 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
static int virtnet_open(struct net_device *dev)
{
struct virtnet_info *vi = netdev_priv(dev);
+ bool refill = false;
int i;
for (i = 0; i < vi->max_queue_pairs; i++) {
if (i < vi->curr_queue_pairs)
/* Make sure we have some buffers: if oom use wq. */
if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
- schedule_delayed_work(&vi->refill, 0);
+ refill = true;
virtnet_napi_enable(&vi->rq[i]);
}
-
+ if (refill)
+ schedule_delayed_work(&vi->refill, 0);
return 0;
}
Eric Dumazet
2013-12-27 05:46:50 UTC
Permalink
Post by Jason Wang
Post by Eric Dumazet
Post by Michael Dalton
Post by Michael S. Tsirkin
So there isn't a conflict with respect to locking.
Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?
I believe it is safe to use the same page_frag and I will send out a
followup patchset using just the per-receive page_frags. For future
consideration, Eric noted that disabling NAPI before GFP_KERNEL
allocs can potentially inhibit virtio-net network processing for some
time (e.g., during a blocking memory allocation or preemption).
Yep, using napi_disable() in the refill process looks quite inefficient
to me, it not buggy.
napi_disable() is a big hammer, while whole idea of having a process to
block on GFP_KERNEL allocations is to allow some asynchronous behavior.
I have hard time to convince myself virtio_net is safe anyway with this
work queue thing.
for (i = 0; i < vi->max_queue_pairs; i++) {
if (i < vi->curr_queue_pairs)
/* Make sure we have some buffers: if oom use wq. */
if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
schedule_delayed_work(&vi->refill, 0);
virtnet_napi_enable(&vi->rq[i]);
What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?
Then napi_disable() in refill_work() will busy wait until napi is
enabled by virtnet_napi_enable() which looks safe. Looks like the real
issue is in virtnet_restore() who calls try_fill_recv() in neither napi
context nor napi disabled context.
I think you don't really get the race.

The issue is the following :

CPU0 CPU1

schedule_delayed_work()
napi_disable(&rq->napi);
try_fill_recv(rq, GFP_KERNEL);

virtnet_napi_enable(&vi->rq[i]);
...
try_fill_recv(rq, GFP_ATOMIC);

napi_enable();// crash on :
BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
Jason Wang
2013-12-27 06:12:41 UTC
Permalink
Post by Eric Dumazet
Post by Jason Wang
Post by Eric Dumazet
Post by Michael Dalton
Post by Michael S. Tsirkin
So there isn't a conflict with respect to locking.
Is it problematic to use same page_frag with both GFP_ATOMIC and with
GFP_KERNEL? If yes why?
I believe it is safe to use the same page_frag and I will send out a
followup patchset using just the per-receive page_frags. For future
consideration, Eric noted that disabling NAPI before GFP_KERNEL
allocs can potentially inhibit virtio-net network processing for some
time (e.g., during a blocking memory allocation or preemption).
Yep, using napi_disable() in the refill process looks quite inefficient
to me, it not buggy.
napi_disable() is a big hammer, while whole idea of having a process to
block on GFP_KERNEL allocations is to allow some asynchronous behavior.
I have hard time to convince myself virtio_net is safe anyway with this
work queue thing.
for (i = 0; i < vi->max_queue_pairs; i++) {
if (i < vi->curr_queue_pairs)
/* Make sure we have some buffers: if oom use wq. */
if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
schedule_delayed_work(&vi->refill, 0);
virtnet_napi_enable(&vi->rq[i]);
What if the workqueue is scheduled _before_ the call to virtnet_napi_enable(&vi->rq[i]) ?
Then napi_disable() in refill_work() will busy wait until napi is
enabled by virtnet_napi_enable() which looks safe. Looks like the real
issue is in virtnet_restore() who calls try_fill_recv() in neither napi
context nor napi disabled context.
I think you don't really get the race.
CPU0 CPU1
schedule_delayed_work()
napi_disable(&rq->napi);
try_fill_recv(rq, GFP_KERNEL);
If I didn't miss anything. In this case, for a specific rq,
napi_disable() won't return immediately since NAPI_STATE_SCHED bit was
still set. It will busy wait until NAPI_STATE_SCHED bit was clear by
virtnet_napi_enable(), and then reset the bit. So try_fill_recv() should
be called after its napi was enabled by virtnet_napi_enable() in CPU0.

So the following order were guaranteed:
- try_fill_recv(rq, GFP_ATOMIC) in CPU0
- virtnet_napi_enable(&vi->rq[i]) in CPU0
- napi_disable(&rq->napi) returned in CPU1
- try_fill_recv(rq) in CPU1
...
Post by Eric Dumazet
virtnet_napi_enable(&vi->rq[i]);
...
try_fill_recv(rq, GFP_ATOMIC);
BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
2013-12-23 13:31:06 UTC
Permalink
Post by Michael Dalton
The virtio-net driver currently uses netdev_alloc_frag() for GFP_ATOMIC
mergeable rx buffer allocations. This commit migrates virtio-net to use
per-receive queue page frags for GFP_ATOMIC allocation. This change unifies
mergeable rx buffer memory allocation, which now will use skb_refill_frag()
for both atomic and GFP-WAIT buffer allocations.
OK so just to clarify, this is intended as a cleanup
and preparation for 3/3, not as an optimization?
Some notes below.
Post by Michael Dalton
To address fragmentation concerns, if after buffer allocation there
is too little space left in the page frag to allocate a subsequent
buffer, the remaining space is added to the current allocated buffer
so that the remaining space can be used to store packet data.
---
drivers/net/virtio_net.c | 69 ++++++++++++++++++++++++++----------------------
1 file changed, 38 insertions(+), 31 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c51a988..d38d130 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -78,6 +78,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;
+ /* Page frag for GFP_ATOMIC packet buffer allocation. */
+ struct page_frag atomic_frag;
+
/* RX: fragments + linear part + virtio header */
struct scatterlist sg[MAX_SKB_FRAGS + 2];
@@ -127,9 +130,9 @@ struct virtnet_info {
struct mutex config_lock;
/* Page_frag for GFP_KERNEL packet buffer allocation when we run
- * low on memory.
+ * low on memory. May sleep.
*/
- struct page_frag alloc_frag;
+ struct page_frag sleep_frag;
/* Does the affinity hint is set for virtqueues? */
bool affinity_hint_set;
@@ -336,8 +339,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
int num_buf = hdr->mhdr.num_buffers;
struct page *page = virt_to_head_page(buf);
int offset = buf - page_address(page);
- struct sk_buff *head_skb = page_to_skb(rq, page, offset, len,
- MERGE_BUFFER_LEN);
+ int truesize = max_t(int, len, MERGE_BUFFER_LEN);
+ struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
struct sk_buff *curr_skb = head_skb;
if (unlikely(!curr_skb))
@@ -353,11 +356,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
dev->stats.rx_length_errors++;
goto err_buf;
}
- if (unlikely(len > MERGE_BUFFER_LEN)) {
- pr_debug("%s: rx error: merge buffer too long\n",
- dev->name);
- len = MERGE_BUFFER_LEN;
- }
page = virt_to_head_page(buf);
--rq->num;
@@ -376,19 +374,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
head_skb->truesize += nskb->truesize;
num_skb_frags = 0;
}
+ truesize = max_t(int, len, MERGE_BUFFER_LEN);
if (curr_skb != head_skb) {
head_skb->data_len += len;
head_skb->len += len;
- head_skb->truesize += MERGE_BUFFER_LEN;
+ head_skb->truesize += truesize;
}
offset = buf - page_address(page);
if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
put_page(page);
skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
- len, MERGE_BUFFER_LEN);
+ len, truesize);
} else {
skb_add_rx_frag(curr_skb, num_skb_frags, page,
- offset, len, MERGE_BUFFER_LEN);
+ offset, len, truesize);
}
}
@@ -579,24 +578,24 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
- char *buf = NULL;
- int err;
+ struct page_frag *alloc_frag;
+ char *buf;
+ int err, len, hole;
- if (gfp & __GFP_WAIT) {
- if (skb_page_frag_refill(MERGE_BUFFER_LEN, &vi->alloc_frag,
- gfp)) {
- buf = (char *)page_address(vi->alloc_frag.page) +
- vi->alloc_frag.offset;
- get_page(vi->alloc_frag.page);
- vi->alloc_frag.offset += MERGE_BUFFER_LEN;
- }
- } else {
- buf = netdev_alloc_frag(MERGE_BUFFER_LEN);
- }
- if (!buf)
+ alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
+ if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
return -ENOMEM;
I note that netdev_alloc_frag sets __GFP_COLD which kind of
makes sense.

I also note that netdev_alloc_frag uses some tricks to
reduce the amount of cache lines dirtied - need to look
at whether they actually apply here.
Post by Michael Dalton
+ buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
+ get_page(alloc_frag->page);
+ len = MERGE_BUFFER_LEN;
+ alloc_frag->offset += len;
+ hole = alloc_frag->size - alloc_frag->offset;
+ if (hole < MERGE_BUFFER_LEN) {
+ len += hole;
+ alloc_frag->offset += hole;
+ }
- sg_init_one(rq->sg, buf, MERGE_BUFFER_LEN);
+ sg_init_one(rq->sg, buf, len);
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
put_page(virt_to_head_page(buf));
@@ -1377,6 +1376,16 @@ static void free_receive_bufs(struct virtnet_info *vi)
}
}
+static void free_receive_page_frags(struct virtnet_info *vi)
+{
+ int i;
+ for (i = 0; i < vi->max_queue_pairs; i++)
+ if (vi->rq[i].atomic_frag.page)
+ put_page(vi->rq[i].atomic_frag.page);
+ if (vi->sleep_frag.page)
+ put_page(vi->sleep_frag.page);
+}
+
static void free_unused_bufs(struct virtnet_info *vi)
{
void *buf;
cancel_delayed_work_sync(&vi->refill);
virtnet_del_vqs(vi);
- if (vi->alloc_frag.page)
- put_page(vi->alloc_frag.page);
+ free_receive_page_frags(vi);
free_percpu(vi->stats);
@@ -1741,8 +1749,7 @@ static void virtnet_remove(struct virtio_device *vdev)
unregister_netdev(vi->dev);
remove_vq_common(vi);
- if (vi->alloc_frag.page)
- put_page(vi->alloc_frag.page);
+ free_receive_page_frags(vi);
flush_work(&vi->config_work);
--
1.8.5.1
Michael Dalton
2013-12-17 00:16:29 UTC
Permalink
Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads. For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.

This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.

These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.

Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.

net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs): 13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s

Signed-off-by: Michael Dalton <***@google.com>
---
drivers/net/virtio_net.c | 63 +++++++++++++++++++++++++++++++++++-------------
1 file changed, 46 insertions(+), 17 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index d38d130..904af37 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -26,6 +26,7 @@
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/cpu.h>
+#include <linux/average.h>

static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -36,11 +37,15 @@ module_param(gso, bool, 0444);

/* FIXME: MTU in config. */
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
-#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \
- sizeof(struct virtio_net_hdr_mrg_rxbuf), \
- L1_CACHE_BYTES))
#define GOOD_COPY_LEN 128

+/* Weight used for the RX packet size EWMA. The average packet size is used to
+ * determine the packet buffer size when refilling RX rings. As the entire RX
+ * ring may be refilled at once, the weight is chosen so that the EWMA will be
+ * insensitive to short-term, transient changes in packet size.
+ */
+#define RECEIVE_AVG_WEIGHT 64
+
#define VIRTNET_DRIVER_VERSION "1.0.0"

struct virtnet_stats {
@@ -78,6 +83,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;

+ /* Average packet length for mergeable receive buffers. */
+ struct ewma mrg_avg_pkt_len;
+
/* Page frag for GFP_ATOMIC packet buffer allocation. */
struct page_frag atomic_frag;

@@ -339,13 +347,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
int num_buf = hdr->mhdr.num_buffers;
struct page *page = virt_to_head_page(buf);
int offset = buf - page_address(page);
- int truesize = max_t(int, len, MERGE_BUFFER_LEN);
- struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
+ struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, len);
struct sk_buff *curr_skb = head_skb;

if (unlikely(!curr_skb))
goto err_skb;
-
while (--num_buf) {
int num_skb_frags;

@@ -374,23 +380,40 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
head_skb->truesize += nskb->truesize;
num_skb_frags = 0;
}
- truesize = max_t(int, len, MERGE_BUFFER_LEN);
if (curr_skb != head_skb) {
head_skb->data_len += len;
head_skb->len += len;
- head_skb->truesize += truesize;
+ head_skb->truesize += len;
}
offset = buf - page_address(page);
if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
put_page(page);
skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
- len, truesize);
+ len, len);
} else {
skb_add_rx_frag(curr_skb, num_skb_frags, page,
- offset, len, truesize);
+ offset, len, len);
}
}

+ /* All frags before the last frag are fully used -- for those frags,
+ * truesize = len. Use the size of the most recent buffer allocation
+ * from the last frag's page to estimate the truesize of the last frag.
+ * EWMA with a weight of 64 makes the size adjustments quite small in
+ * the frags allocated on one page (even a order-3 one), and truesize
+ * doesn't need to be 100% accurate.
+ */
+ if (skb_is_nonlinear(head_skb)) {
+ u32 est_buffer_len = page_private(page);
+ if (est_buffer_len > len) {
+ u32 truesize_delta = est_buffer_len - len;
+
+ curr_skb->truesize += truesize_delta;
+ if (curr_skb != head_skb)
+ head_skb->truesize += truesize_delta;
+ }
+ }
+ ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
return head_skb;

err_skb:
@@ -578,24 +601,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
+ const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
struct page_frag *alloc_frag;
char *buf;
- int err, len, hole;
+ int err, hole;
+ u32 buflen;

+ buflen = hdr_len + clamp_t(u32, ewma_read(&rq->mrg_avg_pkt_len),
+ GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
+ buflen = ALIGN(buflen, L1_CACHE_BYTES);
alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
- if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
+ if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, gfp)))
return -ENOMEM;
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
get_page(alloc_frag->page);
- len = MERGE_BUFFER_LEN;
- alloc_frag->offset += len;
+ alloc_frag->offset += buflen;
+ set_page_private(alloc_frag->page, buflen);
hole = alloc_frag->size - alloc_frag->offset;
- if (hole < MERGE_BUFFER_LEN) {
- len += hole;
+ if (hole < buflen) {
+ buflen += hole;
alloc_frag->offset += hole;
}

- sg_init_one(rq->sg, buf, len);
+ sg_init_one(rq->sg, buf, buflen);
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
put_page(virt_to_head_page(buf));
@@ -1516,6 +1544,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
napi_weight);

sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
+ ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
}
--
1.8.5.1
Michael S. Tsirkin
2013-12-23 12:51:23 UTC
Permalink
Post by Michael Dalton
Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads. For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.
This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.
These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.
Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.
net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs): 13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s
OK so a high level benchmark shows it's worth it,
but how well does the logic work?
I think we should make the buffer size accessible in sysfs
or debugfs, and look at it, otherwise we don't really know.
Post by Michael Dalton
---
drivers/net/virtio_net.c | 63 +++++++++++++++++++++++++++++++++++-------------
1 file changed, 46 insertions(+), 17 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index d38d130..904af37 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -26,6 +26,7 @@
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/cpu.h>
+#include <linux/average.h>
static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -36,11 +37,15 @@ module_param(gso, bool, 0444);
/* FIXME: MTU in config. */
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
-#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \
- sizeof(struct virtio_net_hdr_mrg_rxbuf), \
- L1_CACHE_BYTES))
#define GOOD_COPY_LEN 128
+/* Weight used for the RX packet size EWMA. The average packet size is used to
+ * determine the packet buffer size when refilling RX rings. As the entire RX
+ * ring may be refilled at once, the weight is chosen so that the EWMA will be
+ * insensitive to short-term, transient changes in packet size.
+ */
+#define RECEIVE_AVG_WEIGHT 64
+
#define VIRTNET_DRIVER_VERSION "1.0.0"
struct virtnet_stats {
@@ -78,6 +83,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;
+ /* Average packet length for mergeable receive buffers. */
+ struct ewma mrg_avg_pkt_len;
+
/* Page frag for GFP_ATOMIC packet buffer allocation. */
struct page_frag atomic_frag;
@@ -339,13 +347,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
int num_buf = hdr->mhdr.num_buffers;
struct page *page = virt_to_head_page(buf);
int offset = buf - page_address(page);
- int truesize = max_t(int, len, MERGE_BUFFER_LEN);
- struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
+ struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, len);
struct sk_buff *curr_skb = head_skb;
if (unlikely(!curr_skb))
goto err_skb;
-
Don't like this chunk :)
Post by Michael Dalton
while (--num_buf) {
int num_skb_frags;
@@ -374,23 +380,40 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
head_skb->truesize += nskb->truesize;
num_skb_frags = 0;
}
- truesize = max_t(int, len, MERGE_BUFFER_LEN);
if (curr_skb != head_skb) {
head_skb->data_len += len;
head_skb->len += len;
- head_skb->truesize += truesize;
+ head_skb->truesize += len;
}
offset = buf - page_address(page);
if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
put_page(page);
skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
- len, truesize);
+ len, len);
} else {
skb_add_rx_frag(curr_skb, num_skb_frags, page,
- offset, len, truesize);
+ offset, len, len);
}
}
+ /* All frags before the last frag are fully used -- for those frags,
+ * truesize = len. Use the size of the most recent buffer allocation
+ * from the last frag's page to estimate the truesize of the last frag.
I don't get the real motivation for this.

We have skbs A,B,C sharing a page, with chunk D being unused.
This randomly charges chunk D to an skb that ended up last
in the page.
Correct?
Why does this make sense?
Post by Michael Dalton
+ * EWMA with a weight of 64 makes the size adjustments quite small in
+ * the frags allocated on one page (even a order-3 one), and truesize
+ * doesn't need to be 100% accurate.
If the explanation for the above is that we don't care where D is
charged, let's not charge it to any skbs.
Post by Michael Dalton
+ */
+ if (skb_is_nonlinear(head_skb)) {
+ u32 est_buffer_len = page_private(page);
+ if (est_buffer_len > len) {
+ u32 truesize_delta = est_buffer_len - len;
+
+ curr_skb->truesize += truesize_delta;
+ if (curr_skb != head_skb)
+ head_skb->truesize += truesize_delta;
+ }
+ }
+ ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
Why head_skb only? Why not full buffer size that comes from host?
This is simply len.
Post by Michael Dalton
return head_skb;
@@ -578,24 +601,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
+ const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
struct page_frag *alloc_frag;
char *buf;
- int err, len, hole;
+ int err, hole;
+ u32 buflen;
+ buflen = hdr_len + clamp_t(u32, ewma_read(&rq->mrg_avg_pkt_len),
+ GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
+ buflen = ALIGN(buflen, L1_CACHE_BYTES);
alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
- if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
+ if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, gfp)))
return -ENOMEM;
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
get_page(alloc_frag->page);
- len = MERGE_BUFFER_LEN;
- alloc_frag->offset += len;
+ alloc_frag->offset += buflen;
+ set_page_private(alloc_frag->page, buflen);
hole = alloc_frag->size - alloc_frag->offset;
- if (hole < MERGE_BUFFER_LEN) {
- len += hole;
+ if (hole < buflen) {
+ buflen += hole;
alloc_frag->offset += hole;
}
- sg_init_one(rq->sg, buf, len);
+ sg_init_one(rq->sg, buf, buflen);
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
put_page(virt_to_head_page(buf));
@@ -1516,6 +1544,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
napi_weight);
sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
+ ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
}
--
1.8.5.1
Michael S. Tsirkin
2013-12-23 13:33:31 UTC
Permalink
Post by Michael Dalton
Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads. For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.
This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.
These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.
Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.
net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs): 13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s
Also I guess this 1% difference is in the noise, right?
Could you share data about host CPU utilization during
these runs please?
Post by Michael Dalton
---
drivers/net/virtio_net.c | 63 +++++++++++++++++++++++++++++++++++-------------
1 file changed, 46 insertions(+), 17 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index d38d130..904af37 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -26,6 +26,7 @@
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/cpu.h>
+#include <linux/average.h>
static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -36,11 +37,15 @@ module_param(gso, bool, 0444);
/* FIXME: MTU in config. */
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
-#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \
- sizeof(struct virtio_net_hdr_mrg_rxbuf), \
- L1_CACHE_BYTES))
#define GOOD_COPY_LEN 128
+/* Weight used for the RX packet size EWMA. The average packet size is used to
+ * determine the packet buffer size when refilling RX rings. As the entire RX
+ * ring may be refilled at once, the weight is chosen so that the EWMA will be
+ * insensitive to short-term, transient changes in packet size.
+ */
+#define RECEIVE_AVG_WEIGHT 64
+
#define VIRTNET_DRIVER_VERSION "1.0.0"
struct virtnet_stats {
@@ -78,6 +83,9 @@ struct receive_queue {
/* Chain pages by the private ptr. */
struct page *pages;
+ /* Average packet length for mergeable receive buffers. */
+ struct ewma mrg_avg_pkt_len;
+
/* Page frag for GFP_ATOMIC packet buffer allocation. */
struct page_frag atomic_frag;
@@ -339,13 +347,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
int num_buf = hdr->mhdr.num_buffers;
struct page *page = virt_to_head_page(buf);
int offset = buf - page_address(page);
- int truesize = max_t(int, len, MERGE_BUFFER_LEN);
- struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
+ struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, len);
struct sk_buff *curr_skb = head_skb;
if (unlikely(!curr_skb))
goto err_skb;
-
while (--num_buf) {
int num_skb_frags;
@@ -374,23 +380,40 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
head_skb->truesize += nskb->truesize;
num_skb_frags = 0;
}
- truesize = max_t(int, len, MERGE_BUFFER_LEN);
if (curr_skb != head_skb) {
head_skb->data_len += len;
head_skb->len += len;
- head_skb->truesize += truesize;
+ head_skb->truesize += len;
}
offset = buf - page_address(page);
if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
put_page(page);
skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
- len, truesize);
+ len, len);
} else {
skb_add_rx_frag(curr_skb, num_skb_frags, page,
- offset, len, truesize);
+ offset, len, len);
}
}
+ /* All frags before the last frag are fully used -- for those frags,
+ * truesize = len. Use the size of the most recent buffer allocation
+ * from the last frag's page to estimate the truesize of the last frag.
+ * EWMA with a weight of 64 makes the size adjustments quite small in
+ * the frags allocated on one page (even a order-3 one), and truesize
+ * doesn't need to be 100% accurate.
+ */
+ if (skb_is_nonlinear(head_skb)) {
+ u32 est_buffer_len = page_private(page);
+ if (est_buffer_len > len) {
+ u32 truesize_delta = est_buffer_len - len;
+
+ curr_skb->truesize += truesize_delta;
+ if (curr_skb != head_skb)
+ head_skb->truesize += truesize_delta;
+ }
+ }
+ ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
return head_skb;
@@ -578,24 +601,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
+ const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
struct page_frag *alloc_frag;
char *buf;
- int err, len, hole;
+ int err, hole;
+ u32 buflen;
+ buflen = hdr_len + clamp_t(u32, ewma_read(&rq->mrg_avg_pkt_len),
+ GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
+ buflen = ALIGN(buflen, L1_CACHE_BYTES);
alloc_frag = (gfp & __GFP_WAIT) ? &vi->sleep_frag : &rq->atomic_frag;
- if (unlikely(!skb_page_frag_refill(MERGE_BUFFER_LEN, alloc_frag, gfp)))
+ if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, gfp)))
return -ENOMEM;
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
get_page(alloc_frag->page);
- len = MERGE_BUFFER_LEN;
- alloc_frag->offset += len;
+ alloc_frag->offset += buflen;
+ set_page_private(alloc_frag->page, buflen);
hole = alloc_frag->size - alloc_frag->offset;
- if (hole < MERGE_BUFFER_LEN) {
- len += hole;
+ if (hole < buflen) {
+ buflen += hole;
alloc_frag->offset += hole;
}
- sg_init_one(rq->sg, buf, len);
+ sg_init_one(rq->sg, buf, buflen);
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
put_page(virt_to_head_page(buf));
@@ -1516,6 +1544,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
napi_weight);
sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
+ ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
}
--
1.8.5.1
Amos Kong
2013-12-30 10:14:46 UTC
Permalink
Post by Michael S. Tsirkin
Post by Michael Dalton
Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads. For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.
This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.
These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.
Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.
net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs): 13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s
Hi Michael Dalton,

In Autotest framework, we compare performance results by T-test [1], and generate
an abundant/meaningful HTML files. It's helpful to check if performance regression
exists significantly.

If you have interest, we also use this in your netperf testing.

Attached analysis results of netperf tests.
[1] https://github.com/autotest/virt-test/wiki/PerformanceTesting


Thanks, Amos
Post by Michael S. Tsirkin
Also I guess this 1% difference is in the noise, right?
Could you share data about host CPU utilization during
these runs please?
Michael S. Tsirkin
2014-01-08 17:41:17 UTC
Permalink
Post by Amos Kong
Post by Michael S. Tsirkin
Post by Michael Dalton
Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads. For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.
This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.
These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.
Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.
net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs): 13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s
Hi Michael Dalton,
In Autotest framework, we compare performance results by T-test [1], and generate
an abundant/meaningful HTML files. It's helpful to check if performance regression
exists significantly.
Nice. Specifically, this checks Throughput divided by CPU utilization
which is important I think.
Sometimes host scheduler gives us much more CPU and throughput
goes up, but if that's what happens, it won't scale with other work running on same host.
Post by Amos Kong
If you have interest, we also use this in your netperf testing.
Attached analysis results of netperf tests.
[1] https://github.com/autotest/virt-test/wiki/PerformanceTesting
Thanks, Amos
Post by Michael S. Tsirkin
Also I guess this 1% difference is in the noise, right?
Could you share data about host CPU utilization during
these runs please?
####1. Description of setup#1
qemu-kvm-0.12.1.2-2.209.el6.x86_64
host kernel: 3.4.0.zerocopy+
guest kernel:2.6.32-220.el6.x86_64
####2. Description of setup#2
qemu-kvm-0.12.1.2-2.209.el6.x86_64
host kernel: 3.4.0.zerocopy+
guest kernel:2.6.32-220.el6.x86_64
The tests are 60 seconds sessions of "Netperf". 'throughput' was taken from netperf's report.
other measurements were taken on the host.
- The Throughput is measured in Mbit/sec.
- io_exit: io exits of KVM.
- irq_inj: irq injections of KVM.
- Every Avg line represents the average value based on *5* repetitions of the same test,
and the following SD line represents the Standard Deviation between the *5* repetitions.
- The Standard deviation is displayed as a percentage of the average.
- The significance of the differences between the two averages is calculated using unpaired T-test that
takes into account the SD of the averages.
- The paired t-test is computed for the averages of same category.
┌──────────┬──────────────────────────────┬──────────┬────────┬────────┬────────┬────────┬─────────┬───────────┬────────┬────────┬────────┬────────┬────────┬─────────┬─────────┐
│== │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│Regression│ Category:TCP_STREAM │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│Testing: │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│netperf ==│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┬────────┬────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │thr/%CPU│#tx-pkts│#rx-pkts│#tx-byts │ #rx-byts │ # │#tx-intr│#rx-intr│#io_exit│#irq_inj│ #tpkt/# │ #rpkt/# │
│ │ │ │ │ │ │ │ │ │ │ │re-trans│ │ │ │ │ exit │ irq │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 3654.05 │ 22.30 │ 163.86 │2813996 │18928829│185871160│28658172593│ 0 │ 42 │2470996 │ 278105 │2530858 │ 10.12 │ 7.48 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 11% │ 14% │ 4% │ 38% │ 11% │ 38% │ 11% │ 0.0 │ 39% │ 52% │ 12% │ 51% │ 32% │ 324% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 4033.57 │ 23.08 │ 174.76 │2681221 │20895339│177070935│31633911136│ 0 │ 43 │2766747 │ 221023 │2825177 │ 12.13 │ 7.40 │
├──────────┼────────────┤ 4000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 22% │ 7% │ 27% │ 27% │ 22% │ 27% │ 22% │ 0.0 │ 32% │ 43% │ 51% │ 42% │ 478% │ 78% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ +10.386% │+3.498% │+6.652% │-4.718% │+10.389%│ -4.735% │ +10.384% │ +0.0 │+2.381% │+11.969%│-20.525%│+11.629%│+19.862% │ -1.070% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.585 │ 0.360 │ 0.406 │ 0.174 │ 0.585 │ 0.175 │ 0.585 │ 0.653 │ 0.065 │ 0.287 │ 0.690 │ 0.286 │ 0.648 │ 0.506 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 3953.32 │ 23.07 │ 171.36 │2623463 │20480990│173270716│31008001226│ 0 │ 54 │2406189 │ 206773 │2472555 │ 12.69 │ 8.28 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 16% │ 6% │ 16% │ 22% │ 16% │ 22% │ 16% │ 0.0 │ 50% │ 43% │ 68% │ 41% │ 186% │ 109% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 4818.95 │ 22.79 │ 211.45 │3171179 │24963857│209475231│37795242761│ 0 │ 48 │2754395 │ 371403 │2837031 │ 8.54 │ 8.80 │
├──────────┼────────────┤ 4000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 32% │ 7% │ 26% │ 57% │ 32% │ 57% │ 32% │ 0.0 │ 57% │ 78% │ 34% │ 76% │ 20% │ 104% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ +21.896% │-1.214% │+23.395%│+20.878%│+21.888%│+20.895% │ +21.889% │ +0.0 │-11.111%│+14.471%│+79.619%│+14.741%│-32.703% │ +6.280% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.724 │ 0.225 │ 0.797 │ 0.463 │ 0.724 │ 0.464 │ 0.724 │ 0.653 │ 0.256 │ 0.249 │ 0.914 │ 0.259 │ 0.846 │ 0.202 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4342.74 │ 26.09 │ 166.45 │3086160 │22499585│203857770│34064280140│ 0 │ 369 │2287182 │ 173759 │2392793 │ 17.76 │ 9.40 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 10% │ 14% │ 9% │ 42% │ 10% │ 42% │ 10% │ 0.0 │ 36% │ 34% │ 13% │ 34% │ 51% │ 31% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 3963.69 │ 24.11 │ 164.40 │1870501 │20537591│123620068│31093773242│ 0 │ 435 │2506725 │ 143191 │2581189 │ 13.06 │ 7.96 │
├──────────┼────────────┤ 4000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 4% │ 4% │ 3% │ 23% │ 4% │ 23% │ 4% │ 0.0 │ 29% │ 18% │ 33% │ 18% │ 37% │ 18% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -8.728% │-7.589% │-1.232% │-39.391%│-8.720% │-39.360% │ -8.720% │ +0.0 │+17.886%│+9.599% │-17.592%│+7.873% │-26.464% │-15.319% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.894 │ 0.716 │ 0.329 │ 0.918 │ 0.894 │ 0.918 │ 0.894 │ 0.653 │ 0.559 │ 0.399 │ 0.775 │ 0.339 │ 0.610 │ 0.799 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4442.45 │ 22.70 │ 195.70 │2269020 │23012158│149887965│34840362128│ 0 │ 34 │2187933 │ 273170 │2248565 │ 8.31 │ 10.23 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 12% │ 6% │ 18% │ 36% │ 12% │ 36% │ 12% │ 0.0 │ 36% │ 39% │ 4% │ 38% │ 33% │ 50% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 3784.41 │ 22.34 │ 169.40 │2839507 │19603918│187698543│29680297470│ 0 │ 43 │2188310 │ 282257 │2247171 │ 10.06 │ 8.72 │
├──────────┼────────────┤ 20000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 18% │ 16% │ 17% │ 40% │ 18% │ 40% │ 18% │ 0.0 │ 41% │ 60% │ 10% │ 59% │ 31% │ 266% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -14.813% │-1.586% │-13.439%│+25.142%│-14.811%│+25.226% │ -14.811% │ +0.0 │+26.471%│+0.017% │+3.326% │-0.062% │+21.059% │-14.761% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.874 │ 0.160 │ 0.767 │ 0.609 │ 0.874 │ 0.611 │ 0.874 │ 0.653 │ 0.600 │ 0.000 │ 0.467 │ 0.002 │ 0.560 │ 0.476 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4394.70 │ 22.29 │ 197.16 │2587799 │22768355│170940782│34470462005│ 0 │ 62 │3265444 │ 243982 │3339217 │ 10.61 │ 6.82 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 21% │ 15% │ 39% │ 19% │ 21% │ 19% │ 21% │ 0.0 │ 81% │ 28% │ 37% │ 27% │ 79% │ 23% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 3908.76 │ 23.63 │ 165.42 │2989759 │20249863│197517079│30657452074│ 0 │ 48 │2925685 │ 252785 │2992574 │ 11.83 │ 6.77 │
├──────────┼────────────┤ 20000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 20% │ 7% │ 17% │ 22% │ 20% │ 22% │ 20% │ 0.0 │ 29% │ 47% │ 52% │ 46% │ 558% │ 87% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -11.057% │+6.012% │-16.099%│+15.533%│-11.061%│+15.547% │ -11.062% │ +0.0 │-22.581%│-10.405%│+3.608% │-10.381%│+11.499% │ -0.733% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.599 │ 0.550 │ 0.695 │ 0.693 │ 0.599 │ 0.694 │ 0.599 │ 0.653 │ 0.426 │ 0.342 │ 0.095 │ 0.348 │ 0.589 │ 0.475 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4894.89 │ 25.13 │ 194.78 │3082568 │25360893│203681023│38396295385│ 0 │ 364 │3068186 │ 239536 │3172145 │ 12.87 │ 7.99 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 34% │ 5% │ 30% │ 85% │ 34% │ 85% │ 34% │ 0.0 │ 42% │ 30% │ 96% │ 31% │ 32% │ 5% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 5142.45 │ 26.12 │ 196.88 │3600457 │26643682│237950950│40338431022│ 0 │ 340 │2871917 │ 248898 │2992869 │ 14.47 │ 8.90 │
├──────────┼────────────┤ 20000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 32% │ 11% │ 33% │ 63% │ 32% │ 64% │ 32% │ 0.0 │ 58% │ 26% │ 91% │ 27% │ 59% │ 14% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ +5.058% │+3.940% │+1.078% │+16.801%│+5.058% │+16.825% │ +5.058% │ +0.0 │-6.593% │-6.397% │+3.908% │-5.652% │+12.432% │+11.389% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.183 │ 0.510 │ 0.090 │ 0.253 │ 0.183 │ 0.253 │ 0.183 │ 0.653 │ 0.165 │ 0.280 │ 0.050 │ 0.240 │ 0.601 │ 0.853 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4012.65 │ 21.78 │ 184.24 │2307643 │20785874│152551903│31469785004│ 0 │ 35 │1865340 │ 291537 │1925266 │ 7.92 │ 10.80 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 20% │ 16% │ 18% │ 49% │ 20% │ 49% │ 20% │ 0.0 │ 49% │ 65% │ 8% │ 63% │ 42% │ 218% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 4029.64 │ 22.23 │ 181.27 │2502225 │20873891│165368585│31603029704│ 0 │ 38 │1910923 │ 267096 │1970568 │ 9.37 │ 10.59 │
├──────────┼────────────┤ 40000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 21% │ 15% │ 17% │ 52% │ 21% │ 52% │ 21% │ 0.0 │ 52% │ 62% │ 10% │ 60% │ 42% │ 211% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ +0.423% │+2.066% │-1.612% │+8.432% │+0.423% │ +8.402% │ +0.423% │ +0.0 │+8.571% │+2.444% │-8.383% │+2.353% │+18.308% │ -1.944% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.025 │ 0.158 │ 0.131 │ 0.193 │ 0.025 │ 0.192 │ 0.025 │ 0.653 │ 0.196 │ 0.047 │ 0.821 │ 0.046 │ 0.419 │ 0.055 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4426.90 │ 23.58 │ 187.74 │2816686 │22934457│186110754│34721995043│ 0 │ 55 │4385070 │ 256976 │4457988 │ 10.96 │ 5.14 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 28% │ 5% │ 30% │ 43% │ 28% │ 43% │ 28% │ 0.0 │ 58% │ 43% │ 47% │ 42% │ 207% │ 14% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 3948.19 │ 22.83 │ 172.94 │2065324 │20453563│136432079│30966657915│ 0 │ 58 │2172531 │ 224001 │2241149 │ 9.22 │ 9.13 │
├──────────┼────────────┤ 40000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 11% │ 6% │ 7% │ 24% │ 11% │ 24% │ 11% │ 0.0 │ 74% │ 34% │ 54% │ 32% │ 53% │ 27% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -10.814% │-3.181% │-7.883% │-26.675%│-10.817%│-26.693% │ -10.815% │ +0.0 │+5.455% │-50.456%│-12.832%│-49.727%│-15.876% │+77.626% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.568 │ 0.606 │ 0.446 │ 0.764 │ 0.568 │ 0.765 │ 0.568 │ 0.653 │ 0.090 │ -0.961 │ 0.323 │ -0.960 │ 0.525 │ +0.995 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4205.99 │ 25.20 │ 166.90 │2596046 │21792738│171527637│32994109547│ 0 │ 308 │1914297 │ 205798 │2022008 │ 12.61 │ 10.78 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 10% │ 11% │ 6% │ 58% │ 10% │ 58% │ 10% │ 0.0 │ 21% │ 30% │ 33% │ 26% │ 26% │ 37% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 4054.46 │ 24.54 │ 165.22 │2241165 │21006264│148096081│31803374353│ 0 │ 430 │2268245 │ 171670 │2363749 │ 13.06 │ 8.89 │
├──────────┼────────────┤ 40000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 4% │ 6% │ 7% │ 42% │ 4% │ 42% │ 4% │ 0.0 │ 9% │ 22% │ 24% │ 20% │ 21% │ 21% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -3.603% │-2.619% │-1.007% │-13.670%│-3.609% │-13.661% │ -3.609% │ +0.0 │+39.610%│+18.490%│-16.583%│+16.901%│ +3.569% │-17.532% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.531 │ 0.344 │ 0.191 │ 0.334 │ 0.531 │ 0.334 │ 0.531 │ 0.653 │ +0.993 │ 0.674 │ 0.636 │ 0.691 │ 0.321 │ 0.738 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4249.31 │ 23.92 │ 177.65 │2662115 │22011879│175903720│33325960247│ 0 │ 40 │2453648 │ 281530 │2514216 │ 9.46 │ 8.75 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 13% │ 5% │ 16% │ 31% │ 13% │ 31% │ 13% │ 0.0 │ 31% │ 34% │ 7% │ 34% │ 25% │ 59% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 3432.07 │ 17.06 │ 201.18 │1927368 │17779003│127284944│26916454466│ 0 │ 29 │1900741 │ 215296 │1958147 │ 8.95 │ 9.08 │
├──────────┼────────────┤ 64000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 25% │ 58% │ 2667% │ 47% │ 25% │ 47% │ 25% │ 0.0 │ 48% │ 98% │ 52% │ 95% │ 546% │ 315% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -19.232% │-28.679%│+13.245%│-27.600%│-19.230%│-27.639% │ -19.233% │ +0.0 │-27.500%│-22.534%│-23.526%│-22.117%│ -5.391% │ +3.771% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.885 │ 0.839 │ 0.652 │ 0.784 │ 0.885 │ 0.785 │ 0.885 │ 0.653 │ 0.760 │ 0.437 │ 0.774 │ 0.440 │ 0.591 │ 0.812 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4384.12 │ 23.75 │ 184.59 │2236188 │22711421│147716182│34385017633│ 0 │ 51 │2469602 │ 214045 │2536198 │ 10.45 │ 8.95 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 15% │ 2% │ 14% │ 23% │ 15% │ 23% │ 15% │ 0.0 │ 41% │ 20% │ 60% │ 19% │ 118% │ 42% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 3917.46 │ 23.30 │ 168.13 │2229968 │20294602│147309151│30725795840│ 0 │ 37 │2789905 │ 253739 │2858678 │ 8.79 │ 7.10 │
├──────────┼────────────┤ 64000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 14% │ 7% │ 11% │ 17% │ 14% │ 17% │ 14% │ 0.0 │ 8% │ 31% │ 44% │ 30% │ 83% │ 35% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -10.644% │-1.895% │-8.917% │-0.278% │-10.641%│ -0.276% │ -10.642% │ +0.0 │-27.451%│+12.970%│+18.545%│+12.715%│-15.885% │-20.670% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.739 │ 0.439 │ 0.736 │ 0.017 │ 0.739 │ 0.017 │ 0.739 │ 0.653 │ 0.823 │ 0.506 │ 0.384 │ 0.512 │ 0.464 │ 0.626 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 4022.06 │ 24.92 │ 161.40 │2051216 │20840308│135569610│31552148131│ 0 │ 414 │2287715 │ 168291 │2380990 │ 12.19 │ 8.75 │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 9% │ 10% │ 4% │ 66% │ 9% │ 66% │ 9% │ 0.0 │ 43% │ 36% │ 55% │ 33% │ 52% │ 60% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 5250.56 │ 26.30 │ 199.64 │3490675 │27201806│230596578│41183428753│ 0 │ 405 │2560397 │ 227649 │2690396 │ 15.33 │ 10.11 │
├──────────┼────────────┤ 64000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 34% │ 11% │ 50% │ 23% │ 34% │ 23% │ 34% │ 0.0 │ 49% │ 37% │ 39% │ 35% │ 76% │ 25% │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ +30.544% │+5.538% │+23.693%│+70.176%│+30.525%│+70.095% │ +30.525% │ +0.0 │-2.174% │+11.919%│+35.271%│+12.995%│+25.759% │+15.543% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├──────────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.832 │ 0.553 │ 0.667 │ 0.924 │ 0.832 │ 0.924 │ 0.832 │ 0.653 │ 0.054 │ 0.360 │ 0.669 │ 0.412 │ 0.609 │ 0.132 │
├──────────┼────────────┼────────┼────────┼──────────┼────────┼────────┼────────┼────────┼─────────┼───────────┼────────┼────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ Total │ │ │ 0.245 │ 0.576 │ 0.181 │ 0.146 │ 0.245 │ 0.146 │ 0.245 │ 0.661 │ 0.597 │ 0.425 │ 0.161 │ 0.421 │ 0.102 │ 0.011 │
│ │Significance│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
└──────────┴────────────┴────────┴────────┴──────────┴────────┴────────┴────────┴────────┴─────────┴───────────┴────────┴────────┴────────┴────────┴────────┴─────────┴─────────┘
┌────────────────────────────┬────────┬──────────┬────────┬────────┬────────┬────────┬───────────┬───────────┬────────┬──────────┬────────┬────────┬────────┬─────────┬─────────┐
│ Category:TCP_RR │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┬────────────┬───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts │ #rx-byts │ # │ #tx-intr │#rx-intr│#io_exit│#irq_inj│ #tpkt/# │ #rpkt/# │
│ │ │ │ │ │ │ │ │ │ │ │re-trans│ │ │ │ │ exit │ irq │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 54076.24 │ 22.51 │2402.32 │9431729 │9866906 │13601026829│13629730150│ 12 │ 143 │ 986607 │ 515743 │1044580 │ 18.29 │ 9.45 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 15% │ 11% │ 7% │ 16% │ 13% │ 15% │ 15% │ 31% │ 16% │ 71% │ 27% │ 66% │ 38% │ 98% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 36935.31 │ 20.97 │1761.34 │6396858 │7372127 │9286901282 │9351261832 │ 5 │ 97 │ 247655 │ 148899 │ 287270 │ 42.96 │ 25.66 │
├───────┼────────────┤ 4000 │ 50 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 20% │ 6% │ 17% │ 19% │ 14% │ 20% │ 20% │ 55% │ 19% │ 50% │ 27% │ 43% │ 7% │ 42% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -31.698% │-6.841% │-26.682%│-32.177%│-25.284%│ -31.719% │ -31.391% │-58.333%│ -32.168% │-74.898%│-71.129%│-72.499%│+134.882%│+171.534%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ -0.992 │ 0.753 │ -0.997 │ -0.992 │ -0.992 │ -0.992 │ -0.992 │ -0.994 │ -0.992 │ -0.952 │ -0.999 │ -0.957 │ +1.000 │ +0.955 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 62327.45 │ 23.75 │2624.31 │10752984│11055030│15668725680│15688647646│ 10 │ 164 │1356548 │ 211331 │1416435 │ 50.88 │ 7.80 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 13% │ 10% │ 7% │ 14% │ 11% │ 13% │ 13% │ 67% │ 14% │ 45% │ 82% │ 43% │ 66% │ 109% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 42794.83 │ 22.66 │1888.56 │7308678 │8905297 │10753562439│10858930007│ 8 │ 112 │ 257954 │ 135412 │ 308370 │ 53.97 │ 28.88 │
├───────┼────────────┤ 4000 │ 100 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 11% │ 2% │ 12% │ 10% │ 5% │ 11% │ 11% │ 78% │ 11% │ 55% │ 46% │ 46% │ 43% │ 43% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -31.339% │-4.589% │-28.036%│-32.031%│-19.446%│ -31.369% │ -30.785% │-20.000%│ -31.707% │-80.985%│-35.924%│-78.229%│ +6.073% │+270.256%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ -0.998 │ 0.668 │ -0.999 │ -0.998 │ -0.994 │ -0.998 │ -0.998 │ 0.295 │ -0.998 │ -0.996 │ 0.614 │ -0.996 │ 0.406 │ +0.990 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 14703.32 │ 5.10 │2883.00 │2460591 │2605664 │3691923448 │3701796248 │ 0 │ 37 │ 139516 │ 16276 │ 154010 │ 151.18 │ 16.92 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 224% │ 213% │ 46% │ 224% │ 224% │ 224% │ 224% │ 0.0 │ 226% │ 223% │ 220% │ 220% │ 44% │ 44% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 9936.94 │ 4.98 │1995.37 │1668907 │2013806 │2495739547 │2518802907 │ 0 │ 1961 │ 222197 │ 4433 │ 236784 │ 376.47 │ 8.50 │
├───────┼────────────┤ 4000 │ 250 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 224% │ 212% │ 47% │ 223% │ 223% │ 224% │ 223% │ 0.0 │ 224% │ 223% │ 211% │ 221% │ 46% │ 43% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -32.417% │-2.353% │-30.788%│-32.175%│-22.714%│ -32.400% │ -31.957% │ +0.0 │+5200.000%│+59.263%│-72.764%│+53.746%│+149.021%│-49.764% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.205 │ 0.013 │ 0.192 │ 0.203 │ 0.138 │ 0.205 │ 0.202 │ 0.653 │ 0.645 │ 0.239 │ 0.505 │ 0.226 │ 0.422 │ 0.338 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 3.42 │ 0.26 │ 13.15 │ 836 │ 1078 │ 1103860 │ 1490637 │ 0 │ 0 │ 97 │ 260 │ 2968 │ 3.22 │ 0.36 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 15% │ 8% │ 7% │ 13% │ 12% │ 14% │ 11% │ 0.0 │ 0.0 │ 6% │ 7% │ 18% │ 10% │ 18% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 1.56 │ 0.27 │ 5.78 │ 900 │ 1175 │ 1190908 │ 1613970 │ 0 │ 0 │ 107 │ 259 │ 2702 │ 3.47 │ 0.43 │
├───────┼────────────┤ 4000 │ 500 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 137% │ 4% │ 142% │ 20% │ 21% │ 22% │ 21% │ 0.0 │ 0.0 │ 15% │ 9% │ 4% │ 16% │ 19% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -54.386% │+3.846% │-56.046%│+7.656% │+8.998% │ +7.886% │ +8.274% │ +0.0 │ +0.0 │+10.309%│-0.385% │-8.962% │ +7.764% │+19.444% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.905 │ 0.525 │ 0.905 │ 0.483 │ 0.545 │ 0.457 │ 0.508 │ 0.653 │ 0.653 │ 0.756 │ 0.059 │ 0.701 │ 0.585 │ 0.778 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 6034.32 │ 10.75 │ 561.33 │5153363 │5403272 │7582137362 │7598857558 │ 1 │ 6921 │ 715462 │ 15370 │ 747743 │ 335.29 │ 7.23 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 137% │ 134% │ 56% │ 137% │ 137% │ 137% │ 137% │ 195% │ 220% │ 137% │ 152% │ 137% │ 75% │ 52% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 4639.39 │ 9.59 │ 483.77 │3959334 │4172310 │5829152509 │5843461595 │ 1 │ 12782 │ 509659 │ 4209 │ 542761 │ 940.68 │ 7.69 │
├───────┼────────────┤ 20000 │ 50 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 137% │ 133% │ 55% │ 137% │ 137% │ 137% │ 137% │ 179% │ 137% │ 137% │ 130% │ 136% │ 56% │ 52% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -23.117% │-10.791%│-13.817%│-23.170%│-22.782%│ -23.120% │ -23.101% │+0.000% │ +84.684% │-28.765%│-72.615%│-27.413%│+180.557%│ +6.362% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.228 │ 0.104 │ 0.125 │ 0.228 │ 0.224 │ 0.228 │ 0.227 │ 0.130 │ 0.412 │ 0.288 │ 0.671 │ 0.274 │ 0.585 │ 0.066 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.58 │ 0.26 │ 2.23 │ 888 │ 1175 │ 1193268 │ 1657788 │ 0 │ 0 │ 97 │ 246 │ 2711 │ 3.61 │ 0.43 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 60% │ 6% │ 59% │ 18% │ 18% │ 18% │ 18% │ 0.0 │ 0.0 │ 19% │ 7% │ 3% │ 12% │ 16% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.63 │ 0.26 │ 2.42 │ 941 │ 1220 │ 1272993 │ 1722531 │ 0 │ 0 │ 92 │ 238 │ 3029 │ 3.95 │ 0.40 │
├───────┼────────────┤ 20000 │ 100 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 57% │ 7% │ 59% │ 12% │ 11% │ 12% │ 12% │ 0.0 │ 0.0 │ 11% │ 3% │ 17% │ 9% │ 13% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ +8.621% │+0.000% │+8.520% │+5.968% │+3.830% │ +6.681% │ +3.905% │ +0.0 │ +0.0 │-5.155% │-3.252% │+11.730%│ +9.418% │ -6.977% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.184 │ 0.144 │ 0.241 │ 0.444 │ 0.296 │ 0.473 │ 0.295 │ 0.653 │ 0.653 │ 0.346 │ 0.654 │ 0.779 │ 0.819 │ 0.507 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.85 │ 0.26 │ 3.27 │ 1031 │ 1350 │ 1391207 │ 1920218 │ 0 │ 0 │ 108 │ 249 │ 3066 │ 4.14 │ 0.44 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 8% │ 6% │ 13% │ 6% │ 6% │ 6% │ 7% │ 0.0 │ 0.0 │ 8% │ 4% │ 21% │ 4% │ 18% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.15 │ 0.26 │ 0.58 │ 893 │ 1173 │ 1209894 │ 1641494 │ 0 │ 0 │ 97 │ 233 │ 2666 │ 3.83 │ 0.44 │
├───────┼────────────┤ 20000 │ 250 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 229% │ 5% │ 237% │ 24% │ 24% │ 25% │ 25% │ 0.0 │ 0.0 │ 16% │ 7% │ 2% │ 17% │ 22% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -82.353% │+0.000% │-82.263%│-13.385%│-13.111%│ -13.033% │ -14.515% │ +0.0 │ +0.0 │-10.185%│-6.426% │-13.046%│ -7.488% │ +0.000% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ -0.998 │ 0.479 │ -0.997 │ 0.800 │ 0.786 │ 0.763 │ 0.812 │ 0.653 │ 0.653 │ 0.779 │ 0.906 │ 0.794 │ 0.705 │ 0.193 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.69 │ 0.26 │ 2.65 │ 1044 │ 1377 │ 1410784 │ 1952436 │ 0 │ 0 │ 109 │ 257 │ 3026 │ 4.06 │ 0.46 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 59% │ 6% │ 62% │ 11% │ 13% │ 12% │ 13% │ 0.0 │ 0.0 │ 9% │ 5% │ 18% │ 8% │ 8% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.46 │ 0.27 │ 1.70 │ 899 │ 1179 │ 1216410 │ 1649146 │ 0 │ 0 │ 98 │ 233 │ 2717 │ 3.86 │ 0.43 │
├───────┼────────────┤ 20000 │ 500 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 94% │ 6% │ 95% │ 14% │ 14% │ 15% │ 15% │ 0.0 │ 0.0 │ 13% │ 5% │ 4% │ 9% │ 14% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -33.333% │+3.846% │-35.849%│-13.889%│-14.379%│ -13.778% │ -15.534% │ +0.0 │ +0.0 │-10.092%│-9.339% │-10.212%│ -4.926% │ -6.522% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.589 │ 0.653 │ 0.654 │ 0.900 │ 0.889 │ 0.881 │ 0.908 │ 0.653 │ 0.653 │ 0.848 │ -0.988 │ 0.755 │ 0.644 │ 0.541 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.24 │ 0.26 │ 0.92 │ 961 │ 1231 │ 1273726 │ 1756207 │ 0 │ 0 │ 100 │ 247 │ 3022 │ 3.89 │ 0.41 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 92% │ 5% │ 93% │ 12% │ 15% │ 15% │ 15% │ 0.0 │ 0.0 │ 13% │ 5% │ 21% │ 15% │ 25% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.15 │ 0.27 │ 0.56 │ 920 │ 1178 │ 1218570 │ 1665604 │ 0 │ 0 │ 87 │ 224 │ 2657 │ 4.11 │ 0.44 │
├───────┼────────────┤ 40000 │ 50 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 137% │ 6% │ 145% │ 12% │ 11% │ 11% │ 12% │ 0.0 │ 0.0 │ 6% │ 4% │ 2% │ 12% │ 12% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -37.500% │+3.846% │-39.130%│-4.266% │-4.305% │ -4.330% │ -5.159% │ +0.0 │ +0.0 │-13.000%│-9.312% │-12.078%│ +5.656% │ +7.317% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.466 │ 0.677 │ 0.456 │ 0.410 │ 0.390 │ 0.393 │ 0.440 │ 0.653 │ 0.653 │ 0.919 │ -0.991 │ 0.763 │ 0.435 │ 0.343 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.21 │ 0.26 │ 0.81 │ 869 │ 1150 │ 1165925 │ 1635938 │ 0 │ 0 │ 95 │ 240 │ 2681 │ 3.62 │ 0.43 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 91% │ 7% │ 92% │ 8% │ 8% │ 9% │ 8% │ 0.0 │ 0.0 │ 6% │ 3% │ 2% │ 7% │ 7% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.13 │ 0.26 │ 0.50 │ 932 │ 1216 │ 1269201 │ 1724294 │ 0 │ 0 │ 86 │ 229 │ 2771 │ 4.07 │ 0.44 │
├───────┼────────────┤ 40000 │ 100 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 138% │ 7% │ 134% │ 14% │ 13% │ 15% │ 13% │ 0.0 │ 0.0 │ 11% │ 6% │ 4% │ 12% │ 14% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -38.095% │+0.000% │-38.272%│+7.250% │+5.739% │ +8.858% │ +5.401% │ +0.0 │ +0.0 │-9.474% │-4.583% │+3.357% │+12.431% │ +2.326% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.485 │ 0.269 │ 0.521 │ 0.627 │ 0.562 │ 0.686 │ 0.518 │ 0.653 │ 0.653 │ 0.883 │ 0.834 │ 0.847 │ 0.893 │ 0.342 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.37 │ 0.25 │ 1.48 │ 939 │ 1204 │ 1247126 │ 1708161 │ 0 │ 0 │ 88 │ 243 │ 2975 │ 3.86 │ 0.40 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 11% │ 6% │ 8% │ 12% │ 11% │ 12% │ 11% │ 0.0 │ 0.0 │ 11% │ 9% │ 22% │ 8% │ 15% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.14 │ 0.26 │ 0.54 │ 919 │ 1194 │ 1232652 │ 1689413 │ 0 │ 0 │ 87 │ 229 │ 2766 │ 4.01 │ 0.43 │
├───────┼────────────┤ 40000 │ 250 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 133% │ 11% │ 127% │ 7% │ 8% │ 7% │ 8% │ 0.0 │ 0.0 │ 7% │ 4% │ 3% │ 4% │ 12% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -62.162% │+4.000% │-63.514%│-2.130% │-0.831% │ -1.161% │ -1.098% │ +0.0 │ +0.0 │-1.136% │-5.761% │-7.025% │ +3.886% │ +7.500% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ -0.974 │ 0.709 │ -0.987 │ 0.259 │ 0.107 │ 0.141 │ 0.138 │ 0.653 │ 0.653 │ 0.241 │ 0.774 │ 0.507 │ 0.609 │ 0.452 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.22 │ 0.26 │ 0.85 │ 928 │ 1207 │ 1250884 │ 1717862 │ 0 │ 0 │ 90 │ 233 │ 2712 │ 3.98 │ 0.45 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 97% │ 5% │ 96% │ 21% │ 21% │ 22% │ 21% │ 0.0 │ 0.0 │ 17% │ 8% │ 2% │ 17% │ 19% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.31 │ 0.26 │ 1.19 │ 969 │ 1293 │ 1314718 │ 1839094 │ 0 │ 0 │ 92 │ 235 │ 2756 │ 4.12 │ 0.47 │
├───────┼────────────┤ 40000 │ 500 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 60% │ 7% │ 62% │ 18% │ 18% │ 19% │ 18% │ 0.0 │ 0.0 │ 13% │ 5% │ 5% │ 14% │ 20% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ +40.909% │+0.000% │+40.000%│+4.418% │+7.125% │ +5.103% │ +7.057% │ +0.0 │ +0.0 │+2.222% │+0.858% │+1.622% │ +3.518% │ +4.444% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.504 │ 0.000 │ 0.558 │ 0.270 │ 0.417 │ 0.291 │ 0.401 │ 0.653 │ 0.653 │ 0.192 │ 0.136 │ 0.473 │ 0.275 │ 0.363 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.24 │ 0.26 │ 0.92 │ 1000 │ 1293 │ 1357674 │ 1840914 │ 0 │ 0 │ 88 │ 235 │ 2741 │ 4.26 │ 0.47 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 14% │ 8% │ 12% │ 16% │ 14% │ 17% │ 14% │ 0.0 │ 0.0 │ 6% │ 5% │ 3% │ 12% │ 13% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.21 │ 0.26 │ 0.81 │ 846 │ 1111 │ 1126596 │ 1575802 │ 0 │ 0 │ 82 │ 219 │ 2776 │ 3.86 │ 0.40 │
├───────┼────────────┤ 64000 │ 50 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 22% │ 3% │ 25% │ 21% │ 21% │ 23% │ 21% │ 0.0 │ 0.0 │ 16% │ 5% │ 2% │ 16% │ 22% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -12.500% │+0.000% │-11.957%│-15.400%│-14.076%│ -17.020% │ -14.401% │ +0.0 │ +0.0 │-6.818% │-6.809% │+1.277% │ -9.390% │-14.894% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.782 │ 0.290 │ 0.789 │ 0.820 │ 0.797 │ 0.822 │ 0.804 │ 0.653 │ 0.653 │ 0.617 │ 0.923 │ 0.537 │ 0.721 │ 0.810 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.20 │ 0.26 │ 0.77 │ 841 │ 1091 │ 1110570 │ 1545252 │ 0 │ 0 │ 81 │ 225 │ 3017 │ 3.74 │ 0.36 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 12% │ 7% │ 16% │ 12% │ 10% │ 15% │ 11% │ 0.0 │ 0.0 │ 3% │ 5% │ 19% │ 11% │ 23% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.19 │ 0.27 │ 0.70 │ 1035 │ 1343 │ 1399732 │ 1913629 │ 0 │ 0 │ 95 │ 233 │ 2760 │ 4.44 │ 0.49 │
├───────┼────────────┤ 64000 │ 100 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 57% │ 7% │ 62% │ 19% │ 18% │ 21% │ 19% │ 0.0 │ 0.0 │ 10% │ 5% │ 2% │ 15% │ 18% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -5.000% │+3.846% │-9.091% │+23.068%│+23.098%│ +26.037% │ +23.839% │ +0.0 │ +0.0 │+17.284%│+3.556% │-8.518% │+18.717% │+36.111% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.184 │ 0.660 │ 0.293 │ 0.912 │ 0.929 │ 0.910 │ 0.927 │ 0.653 │ 0.653 │ +0.987 │ 0.696 │ 0.657 │ 0.917 │ 0.933 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.26 │ 0.26 │ 1.00 │ 1061 │ 1380 │ 1424087 │ 1970910 │ 0 │ 0 │ 99 │ 237 │ 2777 │ 4.48 │ 0.50 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 15% │ 7% │ 21% │ 12% │ 13% │ 13% │ 14% │ 0.0 │ 0.0 │ 14% │ 6% │ 3% │ 7% │ 13% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.25 │ 0.26 │ 0.96 │ 990 │ 1337 │ 1338042 │ 1903858 │ 0 │ 0 │ 89 │ 227 │ 3262 │ 4.36 │ 0.41 │
├───────┼────────────┤ 64000 │ 250 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 13% │ 9% │ 18% │ 12% │ 12% │ 13% │ 12% │ 0.0 │ 0.0 │ 14% │ 4% │ 24% │ 15% │ 32% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -3.846% │+0.000% │-4.000% │-6.692% │-3.116% │ -6.042% │ -3.402% │ +0.0 │ +0.0 │-10.101%│-4.219% │+17.465%│ -2.679% │-18.000% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.206 │ 0.114 │ 0.190 │ 0.607 │ 0.304 │ 0.536 │ 0.313 │ 0.653 │ 0.653 │ 0.759 │ 0.776 │ 0.793 │ 0.198 │ 0.627 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 1 │ Avg │ │ │ 0.19 │ 0.26 │ 0.73 │ 899 │ 1188 │ 1186040 │ 1686630 │ 0 │ 0 │ 93 │ 236 │ 2711 │ 3.81 │ 0.44 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 64% │ 8% │ 61% │ 30% │ 30% │ 31% │ 32% │ 0.0 │ 0.0 │ 16% │ 10% │ 6% │ 24% │ 29% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ 2 │ Avg │ │ │ 0.15 │ 0.24 │ 0.62 │ 845 │ 1106 │ 1129531 │ 1560339 │ 0 │ 0 │ 82 │ 220 │ 2947 │ 3.84 │ 0.38 │
├───────┼────────────┤ 64000 │ 500 ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ │ %SD │ │ │ 65% │ 11% │ 71% │ 28% │ 29% │ 30% │ 29% │ 0.0 │ 0.0 │ 17% │ 8% │ 17% │ 22% │ 36% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ %Diff │ │ │ -21.053% │-7.692% │-15.068%│-6.007% │-6.902% │ -4.765% │ -7.488% │ +0.0 │ +0.0 │-11.828%│-6.780% │+8.705% │ +0.787% │-13.636% │
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │Significance│ │ │ 0.434 │ 0.721 │ 0.175 │ 0.256 │ 0.285 │ 0.193 │ 0.299 │ 0.653 │ 0.653 │ 0.744 │ 0.766 │ 0.664 │ 0.039 │ 0.446 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼───────────┼────────┼──────────┼────────┼────────┼────────┼─────────┼─────────┤
│ - │ Total │ │ │ 0.893 │ 0.921 │ 0.927 │ 0.923 │ 0.933 │ 0.925 │ 0.925 │ 0.773 │ 0.776 │ 0.850 │ 0.775 │ 0.851 │ 0.808 │ 0.692 │
│ │Significance│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
└───────┴────────────┴───────┴────────┴──────────┴────────┴────────┴────────┴────────┴───────────┴───────────┴────────┴──────────┴────────┴────────┴────────┴─────────┴─────────┘
┌────────────────────────────┬────────┬──────────┬────────┬────────┬────────┬────────┬───────────┬────────┬─────────┬────────┬────────┬─────────┬────────┬─────────┬────────┐
│ Category:TCP_MAERTS │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┬────────────┬───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts │#rx-byts│#re-trans│#tx-intr│#rx-intr│#io_exit │#irq_inj│ #tpkt/# │#rpkt/# │
│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ exit │ irq │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 4025.16 │ 22.84 │ 176.23 │20851157│ 702802 │31567456210│46385250│ 0 │ 87079 │ 62340 │ 1193 │ 183511 │17477.92 │ 3.83 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 2% │ 2% │ 2% │ 5% │ 2% │ 5% │ 0.0 │ 1% │ 3% │ 6% │ 3% │ 6% │ 3% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3044.14 │ 20.50 │ 148.49 │15771221│ 598877 │23874038498│39526212│ 1 │ 67657 │ 26289 │ 1115 │ 115186 │14144.59 │ 5.20 │
├───────┼────────────┤ 4000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 3% │ 0.83% │ 3% │ 3% │ 1% │ 3% │ 1% │ 134% │ 2% │ 8% │ 8% │ 3% │ 6% │ 3% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -24.372% │-10.245%│-15.741%│-24.363%│-14.787%│ -24.371% │-14.787%│ +0.0 │-22.304%│-57.830%│ -6.538% │-37.232%│-19.072% │+35.770%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ 0.633 │ -1.000 │ -1.000 │ 0.821 │ -1.000 │ -1.000 │ +1.000 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3958.83 │ 23.40 │ 169.18 │20508673│ 854697 │31048301902│56410579│ 1 │ 85430 │ 59633 │ 621 │ 181895 │33025.24 │ 4.70 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 2% │ 2% │ 2% │ 2% │ 2% │ 2% │ 167% │ 2% │ 2% │ 7% │ 3% │ 8% │ 2% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3562.72 │ 20.90 │ 170.47 │18458461│ 857508 │27942035427│56596116│ 0 │ 78196 │ 31911 │ 1891 │ 136169 │ 9761.22 │ 6.30 │
├───────┼────────────┤ 4000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 37% │ 2% │ 39% │ 37% │ 21% │ 37% │ 21% │ 0.0 │ 34% │ 35% │ 113% │ 34% │ 71% │ 10% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -10.006% │-10.684%│+0.763% │-9.997% │+0.329% │ -10.005% │+0.329% │-100.000%│-8.468% │-46.488%│+204.509%│-25.139%│-70.443% │+34.043%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ 0.481 │ -1.000 │ 0.052 │ 0.480 │ 0.027 │ 0.481 │ 0.027 │ 0.805 │ 0.439 │ -0.999 │ 0.780 │ 0.940 │ -0.999 │ +1.000 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3901.40 │ 23.70 │ 164.62 │20211963│1267454 │30598575481│83653083│ 0 │ 82354 │ 58493 │ 629 │ 177376 │32133.49 │ 7.15 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 1% │ 2% │ 2% │ 3% │ 2% │ 3% │ 0.0 │ 3% │ 2% │ 22% │ 3% │ 17% │ 6% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 2910.91 │ 21.58 │ 134.89 │15082039│1160029 │22831307937│76563081│ 0 │ 63438 │ 29078 │ 535 │ 116572 │28190.73 │ 9.95 │
├───────┼────────────┤ 4000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 3% │ 2% │ 2% │ 3% │ 3% │ 3% │ 3% │ 0.0 │ 2% │ 4% │ 4% │ 3% │ 4% │ 5% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -25.388% │-8.945% │-18.060%│-25.381%│-8.476% │ -25.384% │-8.475% │ +0.0 │-22.969%│-50.288%│-14.944% │-34.280%│-12.270% │+39.161%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -0.997 │ -1.000 │ -0.997 │ 0.653 │ -1.000 │ -1.000 │ 0.838 │ -1.000 │ 0.907 │ +1.000 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3990.42 │ 24.09 │ 165.65 │20671202│ 664534 │31295001508│43859558│ 0 │ 85100 │ 62132 │ 5200 │ 203512 │ 3975.23 │ 3.27 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 15% │ 13% │ 2% │ 6% │ 2% │ 6% │ 0.0 │ 3% │ 4% │ 175% │ 28% │ 196% │ 23% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3064.89 │ 20.34 │ 150.68 │15881474│ 572534 │24036756364│37787582│ 0 │ 67629 │ 26538 │ 1091 │ 114112 │14556.80 │ 5.02 │
├───────┼────────────┤ 20000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 2% │ 3% │ 2% │ 0.29% │ 2% │ 0.29% │ 0.0 │ 2% │ 4% │ 4% │ 2% │ 4% │ 1% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -23.194% │-15.567%│-9.037% │-23.171%│-13.844%│ -23.193% │-13.844%│ +0.0 │-20.530%│-57.288%│-79.019% │-43.929%│+266.188%│+53.517%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ -1.000 │ -0.953 │ 0.880 │ -1.000 │ -0.999 │ -1.000 │ -0.999 │ 0.653 │ -1.000 │ -1.000 │ 0.658 │ -0.992 │ 0.067 │ +0.999 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3943.19 │ 22.83 │ 172.72 │20427150│ 842967 │30925722054│55636408│ 0 │ 85108 │ 59335 │ 842 │ 175785 │24260.27 │ 4.80 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 1% │ 3% │ 2% │ 4% │ 2% │ 4% │ 0.0 │ 2% │ 2% │ 37% │ 2% │ 29% │ 4% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 2978.75 │ 20.37 │ 146.23 │15432261│ 804840 │23362657135│53120047│ 0 │ 66416 │ 27420 │ 772 │ 112360 │19989.98 │ 7.16 │
├───────┼────────────┤ 20000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 1% │ 0.83% │ 0.83% │ 1% │ 4% │ 1% │ 4% │ 0.0 │ 1% │ 4% │ 14% │ 2% │ 12% │ 6% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -24.458% │-10.775%│-15.337%│-24.452%│-4.523% │ -24.456% │-4.523% │ +0.0 │-21.963%│-53.788%│ -8.314% │-36.081%│-17.602% │+49.167%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ 0.915 │ -1.000 │ 0.915 │ 0.653 │ -1.000 │ -1.000 │ 0.350 │ -1.000 │ 0.898 │ +1.000 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 4635.26 │ 23.14 │ 200.31 │24013988│1392787 │36354488763│91925099│ 1 │ 97818 │ 69897 │ 1217 │ 201387 │19732.12 │ 6.92 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 38% │ 1% │ 40% │ 38% │ 15% │ 38% │ 15% │ 303% │ 40% │ 38% │ 54% │ 38% │ 63% │ 17% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3502.37 │ 20.76 │ 168.71 │18146027│1274730 │27469437009│84133329│ 0 │ 76870 │ 35159 │ 702 │ 134911 │25849.04 │ 9.45 │
├───────┼────────────┤ 20000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 39% │ 0.50% │ 39% │ 39% │ 13% │ 39% │ 13% │ 0.0 │ 39% │ 30% │ 55% │ 36% │ 11% │ 17% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -24.441% │-10.285%│-15.776%│-24.436%│-8.476% │ -24.440% │-8.476% │-100.000%│-21.415%│-49.699%│-42.317% │-33.009%│+31.000% │+36.561%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ 0.713 │ -1.000 │ 0.494 │ 0.712 │ 0.639 │ 0.713 │ 0.639 │ 0.558 │ 0.635 │ -0.973 │ 0.833 │ 0.861 │ 0.324 │ +0.982 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 4022.67 │ 22.09 │ 182.10 │20838443│ 685924 │31548235695│45271283│ 0 │ 85465 │ 64027 │ 1180 │ 179593 │17659.70 │ 3.82 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 0.91% │ 0.59% │ 1% │ 0.91% │ 4% │ 0.91% │ 4% │ 0.0 │ 1% │ 3% │ 15% │ 0.65% │ 14% │ 4% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3036.88 │ 20.27 │ 149.82 │15735787│ 570734 │23817415383│37668804│ 0 │ 66697 │ 26397 │ 1098 │ 112629 │14331.32 │ 5.07 │
├───────┼────────────┤ 40000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 1% │ 3% │ 3% │ 1% │ 0.18% │ 1% │ 0.18% │ 0.0 │ 2% │ 5% │ 8% │ 2% │ 7% │ 2% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -24.506% │-8.239% │-17.727%│-24.487%│-16.793%│ -24.505% │-16.793%│ +0.0 │-21.960%│-58.772%│ -6.949% │-37.287%│-18.847% │+32.723%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ 0.793 │ -1.000 │ -1.000 │ 0.607 │ -1.000 │ -0.983 │ +1.000 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3959.74 │ 24.48 │ 161.75 │20513131│ 878602 │31055765278│57988350│ 0 │ 85255 │ 59696 │ 4569 │ 201114 │ 4489.63 │ 4.37 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 17% │ 14% │ 2% │ 4% │ 2% │ 4% │ 0.0 │ 3% │ 2% │ 190% │ 31% │ 299% │ 21% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3612.20 │ 20.41 │ 176.98 │18715784│ 848316 │28331890132│55989502│ 0 │ 79801 │ 32639 │ 1210 │ 135034 │15467.59 │ 6.28 │
├───────┼────────────┤ 40000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 38% │ 2% │ 39% │ 38% │ 19% │ 38% │ 19% │ 0.0 │ 37% │ 36% │ 35% │ 36% │ 28% │ 13% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -8.777% │-16.626%│+9.416% │-8.762% │-3.447% │ -8.771% │-3.447% │ +0.0 │-6.397% │-45.325%│-73.517% │-32.857%│+244.518%│+43.707%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ 0.416 │ 0.940 │ 0.291 │ 0.415 │ 0.302 │ 0.415 │ 0.302 │ 0.653 │ 0.307 │ -0.999 │ 0.587 │ 0.901 │ 0.774 │ +0.992 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3745.38 │ 22.95 │ 163.20 │19403080│1308077 │29375006522│86334201│ 0 │ 78885 │ 56216 │ 856 │ 161976 │22667.15 │ 8.08 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 1% │ 1% │ 2% │ 6% │ 2% │ 6% │ 0.0 │ 1% │ 2% │ 78% │ 3% │ 50% │ 9% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3493.96 │ 20.61 │ 169.53 │18102335│1299683 │27404210257│85780238│ 0 │ 76414 │ 34729 │ 989 │ 133336 │18303.68 │ 9.75 │
├───────┼────────────┤ 40000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 39% │ 1% │ 39% │ 39% │ 14% │ 39% │ 14% │ 0.0 │ 39% │ 30% │ 66% │ 36% │ 46% │ 16% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -6.713% │-10.196%│+3.879% │-6.704% │-0.642% │ -6.709% │-0.642% │ +0.0 │-3.132% │-38.222%│+15.537% │-17.682%│-19.250% │+20.668%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ 0.306 │ -1.000 │ 0.160 │ 0.306 │ 0.073 │ 0.306 │ 0.073 │ 0.653 │ 0.143 │ -0.998 │ 0.243 │ 0.780 │ 0.732 │ +0.975 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 4026.58 │ 22.20 │ 181.38 │20858458│ 686402 │31578612205│45302864│ 0 │ 85706 │ 63340 │ 1127 │ 178639 │18507.95 │ 3.84 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 1% │ 2% │ 2% │ 1% │ 4% │ 1% │ 4% │ 0.0 │ 2% │ 3% │ 0.81% │ 2% │ 0.96% │ 2% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3059.87 │ 20.17 │ 151.70 │15854408│ 571828 │23997011560│37740969│ 0 │ 67554 │ 26765 │ 1071 │ 113644 │14803.37 │ 5.03 │
├───────┼────────────┤ 64000 │ 1 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 3% │ 0.33% │ 2% │ 3% │ 0.43% │ 3% │ 0.43% │ 0.0 │ 3% │ 6% │ 3% │ 3% │ 2% │ 3% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -24.008% │-9.144% │-16.363%│-23.991%│-16.692%│ -24.009% │-16.692%│ +0.0 │-21.179%│-57.744%│ -4.969% │-36.383%│-20.016% │+30.990%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ -1.000 │ 0.653 │ -1.000 │ -1.000 │ -0.993 │ -1.000 │ -1.000 │ +1.000 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3947.77 │ 23.99 │ 164.56 │20450669│ 858583 │30961307511│56667066│ 0 │ 84716 │ 59427 │ 1769 │ 193705 │11560.58 │ 4.43 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 10% │ 9% │ 2% │ 2% │ 2% │ 2% │ 0.0 │ 2% │ 2% │ 133% │ 22% │ 112% │ 19% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 3597.82 │ 20.57 │ 174.91 │18641202│ 855720 │28218060821│56478120│ 0 │ 79043 │ 31734 │ 1075 │ 133925 │17340.65 │ 6.39 │
├───────┼────────────┤ 64000 │ 2 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 37% │ 2% │ 39% │ 37% │ 18% │ 37% │ 18% │ 0.0 │ 35% │ 35% │ 65% │ 35% │ 20% │ 13% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -8.864% │-14.256%│+6.289% │-8.848% │-0.333% │ -8.860% │-0.333% │ +0.0 │-6.696% │-46.600%│-39.231% │-30.861%│+49.998% │+44.244%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ 0.431 │ -0.988 │ 0.244 │ 0.430 │ 0.032 │ 0.431 │ 0.032 │ 0.653 │ 0.343 │ -0.999 │ 0.455 │ 0.932 │ 0.603 │ +0.995 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 1 │ Avg │ │ │ 3773.21 │ 23.40 │ 161.25 │19547537│1317873 │29593764943│86980763│ 0 │ 79685 │ 56543 │ 3466 │ 174616 │ 5639.80 │ 7.55 │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 2% │ 6% │ 7% │ 2% │ 5% │ 2% │ 5% │ 0.0 │ 3% │ 2% │ 185% │ 15% │ 268% │ 14% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ 2 │ Avg │ │ │ 2924.85 │ 21.20 │ 137.96 │15153638│1154835 │22940791452│76220282│ 0 │ 63344 │ 29439 │ 547 │ 113758 │27703.18 │ 10.15 │
├───────┼────────────┤ 64000 │ 4 ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ │ %SD │ │ │ 4% │ 2% │ 3% │ 4% │ 2% │ 4% │ 2% │ 0.0 │ 3% │ 4% │ 7% │ 3% │ 8% │ 3% │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ %Diff │ │ │ -22.484% │-9.402% │-14.443%│-22.478%│-12.371%│ -22.481% │-12.371%│ +0.0 │-20.507%│-47.935%│-84.218% │-34.852%│+391.209%│+34.437%│
│ │between Avg │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼────────────┤ │ ├──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │Significance│ │ │ -1.000 │ -0.993 │ -0.998 │ -1.000 │ -0.999 │ -1.000 │ -0.999 │ 0.653 │ -1.000 │ -1.000 │ 0.662 │ -0.999 │ 0.080 │ +0.999 │
├───────┼────────────┼───────┼────────┼──────────┼────────┼────────┼────────┼────────┼───────────┼────────┼─────────┼────────┼────────┼─────────┼────────┼─────────┼────────┤
│ - │ Total │ │ │ -1.000 │ -1.000 │ -0.985 │ -1.000 │ -0.999 │ -1.000 │ -0.999 │ 0.414 │ -1.000 │ -1.000 │ 0.909 │ -1.000 │ 0.183 │ +1.000 │
│ │Significance│ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
└───────┴────────────┴───────┴────────┴──────────┴────────┴────────┴────────┴────────┴───────────┴────────┴─────────┴────────┴────────┴─────────┴────────┴─────────┴────────┘
┌───────┬────────────────────────┬──────────┬───────┬───────┬────────┬────────┬───────────┬───────────┬────────┬───────┬───────┬───────┬───────┬────────┬───────┐
│== Raw │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│data of│ Category:TCP_STREAM │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│sample │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│1 == │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼───────┬───────┬────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │ thr/ │#tx-pkts│#rx-pkts│ #tx-byts │ #rx-byts │ # │ # │ # │ # │ # │#tpkt/# │#rpkt/#│
│ │ │ │ │ │ │ %CPU │ │ │ │ │re-trans│tx-intr│rx-intr│io_exit│irq_inj│ exit │ irq │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2914.69 │ 16.58 │175.80 │ 946015 │15098314│ 62478018 │22858578760│ 0 │ 14 │190786 │218640 │248844 │ 4.33 │ 60.67 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3846.89 │ 23.30 │165.10 │3384334 │19928213│ 223561788 │30171286582│ 0 │ 51 │3106036│289974 │3167495│ 11.67 │ 6.29 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3829.48 │ 23.28 │164.49 │3633348 │19837462│ 239959917 │30033885522│ 0 │ 55 │3126068│291581 │3186759│ 12.46 │ 6.22 │
├───────┼───────┤ 4000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3852.38 │ 24.17 │159.39 │3208800 │19956361│ 211964196 │30213902432│ 0 │ 48 │2923113│295547 │2983047│ 10.86 │ 6.69 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3826.82 │ 24.18 │158.26 │2897483 │19823797│ 191391882 │30013209670│ 0 │ 44 │3008981│294786 │3068148│ 9.83 │ 6.46 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3654.05 │ 22.30 │163.86 │2813996 │18928829│ 185871160 │28658172593│ 0 │ 42 │2470996│278105 │2530858│ 10.12 │ 7.48 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3150.22 │ 23.54 │133.82 │2801597 │16321786│ 185024490 │24710811662│ 0 │ 99 │3361056│ 46502 │3418283│ 60.25 │ 4.77 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3883.19 │ 20.74 │187.23 │3158250 │20118341│ 208596880 │30459134356│ 0 │ 48 │663771 │327600 │741974 │ 9.64 │ 27.11 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4561.23 │ 24.63 │185.19 │3084901 │23628392│ 203720814 │35773349852│ 0 │ 47 │2617111│288865 │2686249│ 10.68 │ 8.80 │
├───────┼───────┤ 4000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4594.03 │ 23.19 │198.10 │1751767 │23798542│ 115716366 │36030958376│ 0 │ 27 │2548977│309503 │2618148│ 5.66 │ 9.09 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3577.93 │ 23.25 │153.89 │2320804 │18537890│ 153295032 │28065751886│ 0 │ 49 │2840032│ 61399 │2898121│ 37.80 │ 6.40 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3953.32 │ 23.07 │171.36 │2623463 │20480990│ 173270716 │31008001226│ 0 │ 54 │2406189│206773 │2472555│ 12.69 │ 8.28 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3612.79 │ 21.11 │171.14 │2130367 │18718623│ 140797650 │28339815704│ 0 │ 390 │1288863│166504 │1365057│ 12.79 │ 13.71 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4460.59 │ 30.36 │146.92 │4553323 │23113273│ 300752346 │34993430332│ 0 │ 555 │3224635│140695 │3382961│ 32.36 │ 6.83 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4789.71 │ 28.39 │168.71 │4434046 │24814124│ 292821684 │37568519884│ 0 │ 387 │2232387│200883 │2350894│ 22.07 │ 10.56 │
├───────┼───────┤ 4000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4459.86 │ 23.46 │190.10 │1923225 │23105224│ 127036770 │34981220044│ 0 │ 330 │1842177│174892 │1912146│ 11.00 │ 12.08 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4390.76 │ 27.12 │161.90 │2389842 │22746683│ 157880400 │34438414736│ 0 │ 184 │2847848│185822 │2952908│ 12.86 │ 7.70 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4342.74 │ 26.09 │166.45 │3086160 │22499585│ 203857770 │34064280140│ 0 │ 369 │2287182│173759 │2392793│ 17.76 │ 9.40 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3841.00 │ 24.14 │159.11 │3093078 │19897547│ 204321312 │30124855328│ 0 │ 47 │3000038│286674 │3060205│ 10.79 │ 6.50 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3852.77 │ 24.42 │157.77 │3113317 │19958336│ 205666302 │30216890906│ 0 │ 47 │3239480│276424 │3299678│ 11.26 │ 6.05 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4876.24 │ 21.92 │222.46 │1827268 │25258600│ 120730740 │38241409172│ 0 │ 28 │1700264│264050 │1762228│ 6.92 │ 14.33 │
├───────┼───────┤ 20000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4813.63 │ 21.68 │222.03 │2043971 │24934583│ 135005358 │37750924418│ 0 │ 31 │1404942│262419 │1464728│ 7.79 │ 17.02 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4828.61 │ 21.34 │226.27 │1267470 │25011728│ 83716116 │37867730818│ 0 │ 19 │1594942│276285 │1655990│ 4.59 │ 15.10 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4442.45 │ 22.70 │195.70 │2269020 │23012158│ 149887965 │34840362128│ 0 │ 34 │2187933│273170 │2248565│ 8.31 │ 10.23 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4614.01 │ 23.50 │196.34 │2236705 │23901627│ 147759066 │36187019896│ 0 │ 34 │2619512│281045 │2687252│ 7.96 │ 8.89 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3012.82 │ 22.15 │136.02 │2290965 │15612079│ 151308010 │23636008424│ 0 │ 151 │2677904│ 83015 │2738943│ 27.60 │ 5.70 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4625.91 │ 24.23 │190.92 │2504936 │23963357│ 165589248 │36280488718│ 0 │ 38 │2723487│284637 │2791688│ 8.80 │ 8.58 │
├───────┼───────┤ 20000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4140.85 │ 25.07 │165.17 │2477142 │21450701│ 163668984 │32476328990│ 0 │ 37 │3598697│303891 │3665244│ 8.15 │ 5.85 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 5579.91 │ 16.52 │337.77 │3429251 │28914014│ 226378602 │43772464000│ 0 │ 52 │4707622│267324 │4812961│ 12.83 │ 6.01 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4394.70 │ 22.29 │197.16 │2587799 │22768355│ 170940782 │34470462005│ 0 │ 62 │3265444│243982 │3339217│ 10.61 │ 6.82 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3963.13 │ 24.37 │162.62 │1576022 │20535988│ 104202552 │31091368626│ 0 │ 516 │2566835│ 82331 │2635098│ 19.14 │ 7.79 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 7814.37 │ 26.41 │295.89 │7584132 │40484304│ 500952100 │61293157172│ 0 │ 119 │4694704│644533 │4901091│ 11.77 │ 8.26 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4264.56 │ 24.03 │177.47 │1736402 │22095527│ 114811464 │33452533114│ 0 │ 338 │2558984│146947 │2628159│ 11.82 │ 8.41 │
├───────┼───────┤ 20000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4394.78 │ 26.52 │165.72 │3126071 │22768346│ 206505030 │34471201720│ 0 │ 457 │2745724│178618 │2852158│ 17.50 │ 7.98 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4037.63 │ 24.32 │166.02 │1390213 │20920303│ 91933970 │31673216296│ 0 │ 394 │2774686│145251 │2844219│ 9.57 │ 7.36 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4894.89 │ 25.13 │194.78 │3082568 │25360893│ 203681023 │38396295385│ 0 │ 364 │3068186│239536 │3172145│ 12.87 │ 7.99 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4696.12 │ 21.45 │218.93 │1746717 │24325944│ 115428234 │36829461018│ 0 │ 27 │1412919│299272 │1473390│ 5.84 │ 16.51 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4857.96 │ 21.85 │222.33 │1877611 │25163730│ 124052082 │38097825926│ 0 │ 29 │1642255│281181 │1702943│ 6.68 │ 14.78 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3834.49 │ 24.77 │154.80 │3536398 │19863734│ 233871300 │30073660456│ 0 │ 54 │3141635│306823 │3202110│ 11.53 │ 6.20 │
├───────┼───────┤ 40000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2877.93 │ 16.28 │176.78 │ 965972 │14907628│ 63803040 │22570131676│ 0 │ 14 │180473 │254054 │238407 │ 3.80 │ 62.53 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3796.74 │ 24.57 │154.53 │3411520 │19668337│ 225604860 │29777845948│ 0 │ 52 │2949421│316357 │3009483│ 10.78 │ 6.54 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4012.65 │ 21.78 │184.24 │2307643 │20785874│ 152551903 │31469785004│ 0 │ 35 │1865340│291537 │1925266│ 7.92 │ 10.80 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4163.62 │ 24.39 │170.71 │2486527 │21568715│ 164397114 │32654995240│ 0 │ 38 │3494130│304520 │3560295│ 8.17 │ 6.06 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 6479.87 │ 22.83 │283.83 │4907786 │33573135│ 324164328 │50826111556│ 0 │ 75 │7684977│336261 │7792528│ 14.60 │ 4.31 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4168.32 │ 24.97 │166.93 │1769613 │21592962│ 116901894 │32691709288│ 0 │ 27 │3977691│295565 │4044685│ 5.99 │ 5.34 │
├───────┼───────┤ 40000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3192.35 │ 23.53 │135.67 │2660256 │16541052│ 175727580 │25043009116│ 0 │ 102 │3218560│ 44895 │3276488│ 59.26 │ 5.05 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4130.34 │ 22.16 │186.39 │2259249 │21396424│ 149362854 │32394150018│ 0 │ 34 │3549994│303642 │3615945│ 7.44 │ 5.92 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4426.90 │ 23.58 │187.74 │2816686 │22934457│ 186110754 │34721995043│ 0 │ 55 │4385070│256976 │4457988│ 10.96 │ 5.14 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3722.13 │ 22.56 │164.99 │1484605 │19286605│ 98125982 │29199830180│ 0 │ 306 │2041127│172603 │2110311│ 8.60 │ 9.14 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4075.12 │ 26.43 │154.19 │4216004 │21114946│ 278471452 │31967964016│ 0 │ 241 │1367095│293206 │1527824│ 14.38 │ 13.82 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4185.80 │ 23.41 │178.80 │1417592 │21688358│ 93727364 │32836027678│ 0 │ 358 │2554985│149408 │2623999│ 9.49 │ 8.27 │
├───────┼───────┤ 40000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4851.13 │ 29.52 │164.33 │4261781 │25135249│ 281515090 │38054707246│ 0 │ 250 │1267474│264201 │1437691│ 16.13 │ 17.48 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4195.77 │ 24.10 │174.10 │1600252 │21738536│ 105798300 │32912018618│ 0 │ 387 │2340807│149572 │2410218│ 10.70 │ 9.02 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4205.99 │ 25.20 │166.90 │2596046 │21792738│ 171527637 │32994109547│ 0 │ 308 │1914297│205798 │2022008│ 12.61 │ 10.78 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3863.66 │ 23.99 │161.05 │3050872 │20014608│ 201627408 │30302088890│ 0 │ 46 │2978200│287857 │3038123│ 10.60 │ 6.59 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4882.87 │ 21.95 │222.45 │1859234 │25293303│ 122793816 │38294025762│ 0 │ 28 │1648954│258666 │1709509│ 7.19 │ 14.80 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3861.97 │ 23.90 │161.59 │3245206 │20005782│ 214361940 │30288737126│ 0 │ 49 │3283337│284083 │3344212│ 11.42 │ 5.98 │
├───────┼───────┤ 64000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4849.67 │ 25.07 │193.44 │1702945 │25120820│ 112472070 │38032901884│ 0 │ 26 │1437275│266204 │1499082│ 6.40 │ 16.76 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3788.37 │ 24.67 │153.56 │3452319 │19624883│ 228263370 │29712047576│ 0 │ 52 │2920474│310841 │2980155│ 11.11 │ 6.59 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4249.31 │ 23.92 │177.65 │2662115 │22011879│ 175903720 │33325960247│ 0 │ 40 │2453648│281530 │2514216│ 9.46 │ 8.75 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3562.91 │ 23.23 │153.38 │2300483 │18457417│ 151960722 │27944416164│ 0 │ 58 │2876396│ 63094 │2934764│ 36.46 │ 6.29 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4613.51 │ 23.90 │193.03 │2448721 │23898855│ 161744190 │36182832826│ 0 │ 37 │2690384│285235 │2757277│ 8.58 │ 8.67 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 5257.18 │ 24.56 │214.05 │2793411 │27233581│ 184502818 │41231606500│ 0 │ 43 │1635722│348277 │1711248│ 8.02 │ 15.91 │
├───────┼───────┤ 64000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4584.42 │ 23.13 │198.20 │2219308 │23748720│ 146595708 │35955528220│ 0 │ 34 │2455753│285711 │2526131│ 7.77 │ 9.40 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3902.59 │ 23.93 │163.08 │1419017 │20218532│ 93777474 │30610704458│ 0 │ 85 │2689758│ 87910 │2751572│ 16.14 │ 7.35 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4384.12 │ 23.75 │184.59 │2236188 │22711421│ 147716182 │34385017633│ 0 │ 51 │2469602│214045 │2536198│ 10.45 │ 8.95 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3675.97 │ 22.57 │162.87 │1609139 │19045980│ 106378386 │28835553466│ 0 │ 347 │1925815│174777 │1996941│ 9.21 │ 9.54 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3793.56 │ 24.91 │152.29 │1729962 │19657863│ 114362928 │29761926386│ 0 │ 666 │3102645│ 73302 │3170057│ 23.60 │ 6.20 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4569.95 │ 29.07 │157.21 │4440539 │23678370│ 293327866 │35848984990│ 0 │ 171 │1045134│320589 │1232501│ 13.85 │ 19.21 │
├───────┼───────┤ 64000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4007.90 │ 24.24 │165.34 │1363391 │20766245│ 90158958 │31439979966│ 0 │ 463 │2860456│140066 │2931297│ 9.73 │ 7.08 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4062.92 │ 23.81 │170.64 │1113052 │21053084│ 73619916 │31874295850│ 0 │ 423 │2504525│132723 │2574155│ 8.39 │ 8.18 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4022.06 │ 24.92 │161.40 │2051216 │20840308│ 135569610 │31552148131│ 0 │ 414 │2287715│168291 │2380990│ 12.19 │ 8.75 │
├───────┴───────┴───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ Category:TCP_RR │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┬───────┬───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │ thr/ │#tx-pkts│#rx-pkts│ #tx-byts │ #rx-byts │ # │ # │ # │ # │ # │#tpkt/# │#rpkt/#│
│ │ │ │ │ │ │ %CPU │ │ │ │ │re-trans│tx-intr│rx-intr│io_exit│irq_inj│ exit │ irq │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 62063.30 │ 23.29 │2664.80│10898349│11138705│15614706386│15630551426│ 13 │ 166 │971977 │518626 │1023711│ 21.01 │ 10.88 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 44633.86 │ 19.54 │2284.23│7675417 │8425505 │11218936530│11268428118│ 10 │ 117 │316398 │740527 │379948 │ 10.36 │ 22.18 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 58846.04 │ 24.54 │2397.96│10361292│10595043│14807141312│14822541758│ 19 │ 158 │1708938│387095 │1763622│ 26.77 │ 6.01 │
├───────┼───────┤ 4000 │ 50 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 46379.40 │ 20.40 │2273.50│7934994 │8651555 │11654996612│11702275190│ 10 │ 121 │273345 │529932 │336016 │ 14.97 │ 25.75 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 58458.58 │ 24.79 │2358.15│10288597│10523722│14709353306│14724854260│ 12 │ 157 │1662381│402538 │1719603│ 25.56 │ 6.12 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 54076.24 │ 22.51 │2402.32│9431729 │9866906 │13601026829│13629730150│ 12 │ 143 │986607 │515743 │1044580│ 18.29 │ 9.45 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 47725.79 │ 20.04 │2386.28│8143711 │8926371 │11992102514│12043740430│ 17 │ 124 │280776 │521236 │345152 │ 15.62 │ 25.86 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 66147.84 │ 23.00 │2875.99│11417925│11646862│16629505298│16644593452│ 15 │ 175 │1664709│157810 │1725000│ 72.35 │ 6.75 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 67209.33 │ 25.35 │2651.26│11637388│11791745│16898728136│16908915442│ 0 │ 177 │1777024│113475 │1835680│ 102.55 │ 6.42 │
├───────┼───────┤ 4000 │ 100 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 66457.59 │ 25.03 │2655.12│11493181│11652590│16708810058│16719319572│ 8 │ 176 │1600205│133977 │1657520│ 85.78 │ 7.03 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 64096.68 │ 25.32 │2531.46│11072718│11257585│16114482396│16126669338│ 10 │ 168 │1460028│130160 │1518824│ 85.07 │ 7.41 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 62327.45 │ 23.75 │2624.31│10752984│11055030│15668725680│15688647646│ 10 │ 164 │1356548│211331 │1416435│ 50.88 │ 7.80 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.25 │ 0.00 │ 875 │ 1157 │ 1173070 │ 1606714 │ 0 │ 0 │ 100 │ 268 │ 2685 │ 3.26 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 73510.59 │ 24.49 │3001.66│12299712│13024092│18455336036│18503152896│ 3 │ 187 │697196 │ 80346 │759284 │ 153.08 │ 17.15 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 868 │ 1097 │ 1128200 │ 1518018 │ 0 │ 0 │ 105 │ 260 │ 2785 │ 3.34 │ 0.39 │
├───────┼───────┤ 4000 │ 250 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2.87 │ 0.24 │ 11.96 │ 722 │ 940 │ 948860 │ 1285684 │ 0 │ 0 │ 88 │ 246 │ 2621 │ 2.93 │ 0.36 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.15 │ 0.26 │ 12.12 │ 782 │ 1035 │ 1031076 │ 1417930 │ 0 │ 0 │ 92 │ 260 │ 2678 │ 3.01 │ 0.39 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 14703.32 │ 5.10 │2883.00│2460591 │2605664 │3691923448 │3701796248 │ 0 │ 37 │139516 │ 16276 │154010 │ 151.18 │ 16.92 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.08 │ 0.25 │ 12.32 │ 763 │ 995 │ 1004302 │ 1376328 │ 0 │ 0 │ 89 │ 241 │ 2774 │ 3.17 │ 0.36 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4.27 │ 0.30 │ 14.23 │ 998 │ 1276 │ 1334364 │ 1751516 │ 0 │ 0 │ 100 │ 264 │ 2866 │ 3.78 │ 0.45 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.00 │ 0.25 │ 12.00 │ 726 │ 954 │ 942116 │ 1304720 │ 0 │ 0 │ 97 │ 242 │ 2688 │ 3.00 │ 0.35 │
├───────┼───────┤ 4000 │ 500 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.35 │ 0.26 │ 12.88 │ 845 │ 1073 │ 1108666 │ 1493478 │ 0 │ 0 │ 105 │ 282 │ 2620 │ 3.00 │ 0.41 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.38 │ 0.25 │ 13.52 │ 851 │ 1094 │ 1129854 │ 1527144 │ 0 │ 1 │ 98 │ 273 │ 3895 │ 3.12 │ 0.28 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3.42 │ 0.26 │ 13.15 │ 836 │ 1078 │ 1103860 │ 1490637 │ 0 │ 0 │ 97 │ 260 │ 2968 │ 3.22 │ 0.36 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 662 │ 862 │ 872580 │ 1197944 │ 0 │ 0 │ 80 │ 234 │ 2570 │ 2.83 │ 0.34 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 14954.85 │ 26.51 │564.12 │12561026│13433340│18775875992│18833448696│ 4 │ 421 │1875885│ 53370 │1937836│ 235.36 │ 6.93 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 15216.05 │ 26.45 │575.28 │13203430│13579990│19131713484│19156583680│ 3 │ 34188 │1701172│ 22748 │1792951│ 580.42 │ 7.57 │
├───────┼───────┤ 20000 │ 50 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.68 │ 0.26 │ 2.62 │ 907 │ 1127 │ 1154494 │ 1587436 │ 0 │ 0 │ 92 │ 259 │ 2648 │ 3.50 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.27 │ 0.00 │ 792 │ 1044 │ 1070264 │ 1470038 │ 0 │ 0 │ 81 │ 241 │ 2713 │ 3.29 │ 0.38 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 6034.32 │ 10.75 │561.33 │5153363 │5403272 │7582137362 │7598857558 │ 1 │ 6921 │715462 │ 15370 │747743 │ 335.29 │ 7.23 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.60 │ 0.25 │ 2.40 │ 730 │ 978 │ 980108 │ 1369008 │ 0 │ 0 │ 74 │ 228 │ 2750 │ 3.20 │ 0.36 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.82 │ 0.27 │ 3.04 │ 974 │ 1318 │ 1316596 │ 1863320 │ 0 │ 0 │ 109 │ 254 │ 2684 │ 3.83 │ 0.49 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.58 │ 0.28 │ 2.07 │ 727 │ 939 │ 959446 │ 1323908 │ 0 │ 0 │ 81 │ 227 │ 2644 │ 3.20 │ 0.36 │
├───────┼───────┤ 20000 │ 100 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.88 │ 0.26 │ 3.38 │ 1096 │ 1425 │ 1469400 │ 2010448 │ 0 │ 0 │ 107 │ 263 │ 2854 │ 4.17 │ 0.50 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.24 │ 0.00 │ 915 │ 1219 │ 1240790 │ 1722258 │ 0 │ 0 │ 115 │ 261 │ 2626 │ 3.51 │ 0.46 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.58 │ 0.26 │ 2.23 │ 888 │ 1175 │ 1193268 │ 1657788 │ 0 │ 0 │ 97 │ 246 │ 2711 │ 3.61 │ 0.43 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.78 │ 0.28 │ 2.79 │ 988 │ 1247 │ 1310460 │ 1773674 │ 0 │ 0 │ 111 │ 236 │ 2859 │ 4.19 │ 0.44 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.93 │ 0.25 │ 3.72 │ 1120 │ 1455 │ 1520256 │ 2075640 │ 0 │ 0 │ 122 │ 254 │ 2780 │ 4.41 │ 0.52 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.80 │ 0.26 │ 3.08 │ 974 │ 1277 │ 1322540 │ 1818880 │ 0 │ 0 │ 101 │ 241 │ 2828 │ 4.04 │ 0.45 │
├───────┼───────┤ 20000 │ 250 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.85 │ 0.25 │ 3.40 │ 1032 │ 1364 │ 1378648 │ 1926622 │ 0 │ 0 │ 108 │ 259 │ 2647 │ 3.98 │ 0.52 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.90 │ 0.24 │ 3.75 │ 1042 │ 1408 │ 1424132 │ 2006276 │ 0 │ 0 │ 101 │ 258 │ 4216 │ 4.04 │ 0.33 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.85 │ 0.26 │ 3.27 │ 1031 │ 1350 │ 1391207 │ 1920218 │ 0 │ 0 │ 108 │ 249 │ 3066 │ 4.14 │ 0.44 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 1060 │ 1372 │ 1436688 │ 1942296 │ 0 │ 0 │ 120 │ 259 │ 2776 │ 4.09 │ 0.49 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 1.05 │ 0.24 │ 4.38 │ 1234 │ 1663 │ 1676340 │ 2359874 │ 0 │ 0 │ 116 │ 277 │ 3976 │ 4.45 │ 0.42 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.88 │ 0.27 │ 3.26 │ 1043 │ 1402 │ 1403574 │ 1993688 │ 0 │ 0 │ 108 │ 246 │ 2836 │ 4.24 │ 0.49 │
├───────┼───────┤ 20000 │ 500 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.77 │ 0.25 │ 3.08 │ 945 │ 1232 │ 1256882 │ 1736182 │ 0 │ 0 │ 95 │ 252 │ 2649 │ 3.75 │ 0.47 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.77 │ 0.26 │ 2.96 │ 940 │ 1217 │ 1280440 │ 1730142 │ 0 │ 0 │ 109 │ 252 │ 2893 │ 3.73 │ 0.42 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.69 │ 0.26 │ 2.65 │ 1044 │ 1377 │ 1410784 │ 1952436 │ 0 │ 0 │ 109 │ 257 │ 3026 │ 4.06 │ 0.46 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 802 │ 1025 │ 1055124 │ 1452832 │ 0 │ 0 │ 91 │ 245 │ 2743 │ 3.27 │ 0.37 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.35 │ 0.25 │ 1.40 │ 911 │ 1141 │ 1227270 │ 1620910 │ 0 │ 0 │ 96 │ 248 │ 2804 │ 3.67 │ 0.41 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.25 │ 0.00 │ 975 │ 1264 │ 1276454 │ 1798304 │ 0 │ 0 │ 123 │ 250 │ 2727 │ 3.90 │ 0.46 │
├───────┼───────┤ 40000 │ 50 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.37 │ 0.25 │ 1.48 │ 990 │ 1213 │ 1241336 │ 1728086 │ 0 │ 0 │ 94 │ 264 │ 4158 │ 3.75 │ 0.29 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.47 │ 0.27 │ 1.74 │ 1128 │ 1514 │ 1568448 │ 2180904 │ 0 │ 0 │ 96 │ 231 │ 2679 │ 4.88 │ 0.57 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.24 │ 0.26 │ 0.92 │ 961 │ 1231 │ 1273726 │ 1756207 │ 0 │ 0 │ 100 │ 247 │ 3022 │ 3.89 │ 0.41 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.29 │ 0.00 │ 981 │ 1290 │ 1334946 │ 1839232 │ 0 │ 0 │ 103 │ 241 │ 2753 │ 4.07 │ 0.47 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.37 │ 0.26 │ 1.42 │ 871 │ 1173 │ 1181286 │ 1672254 │ 0 │ 0 │ 90 │ 241 │ 2668 │ 3.61 │ 0.44 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.33 │ 0.26 │ 1.27 │ 807 │ 1082 │ 1063886 │ 1542784 │ 0 │ 0 │ 90 │ 236 │ 2705 │ 3.42 │ 0.40 │
├───────┼───────┤ 40000 │ 100 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.35 │ 0.25 │ 1.40 │ 871 │ 1151 │ 1152622 │ 1630252 │ 0 │ 0 │ 100 │ 250 │ 2660 │ 3.48 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.24 │ 0.00 │ 815 │ 1055 │ 1096886 │ 1495172 │ 0 │ 0 │ 92 │ 232 │ 2621 │ 3.51 │ 0.40 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.21 │ 0.26 │ 0.81 │ 869 │ 1150 │ 1165925 │ 1635938 │ 0 │ 0 │ 95 │ 240 │ 2681 │ 3.62 │ 0.43 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.42 │ 0.27 │ 1.56 │ 1080 │ 1350 │ 1414736 │ 1919280 │ 0 │ 0 │ 99 │ 248 │ 2797 │ 4.35 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.40 │ 0.25 │ 1.60 │ 1034 │ 1328 │ 1388932 │ 1882180 │ 0 │ 0 │ 95 │ 277 │ 4118 │ 3.73 │ 0.32 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.32 │ 0.24 │ 1.33 │ 797 │ 1031 │ 1061490 │ 1459482 │ 0 │ 0 │ 76 │ 224 │ 2635 │ 3.56 │ 0.39 │
├───────┼───────┤ 40000 │ 250 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.35 │ 0.25 │ 1.40 │ 911 │ 1152 │ 1207526 │ 1634406 │ 0 │ 0 │ 91 │ 233 │ 2640 │ 3.91 │ 0.44 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.35 │ 0.23 │ 1.52 │ 874 │ 1160 │ 1162948 │ 1645460 │ 0 │ 0 │ 83 │ 233 │ 2686 │ 3.75 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.37 │ 0.25 │ 1.48 │ 939 │ 1204 │ 1247126 │ 1708161 │ 0 │ 0 │ 88 │ 243 │ 2975 │ 3.86 │ 0.40 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.23 │ 0.28 │ 0.82 │ 598 │ 771 │ 788244 │ 1069924 │ 0 │ 0 │ 69 │ 208 │ 2669 │ 2.88 │ 0.29 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.24 │ 0.00 │ 983 │ 1265 │ 1284630 │ 1798626 │ 0 │ 0 │ 92 │ 223 │ 2632 │ 4.41 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 955 │ 1284 │ 1308294 │ 1831870 │ 0 │ 0 │ 87 │ 256 │ 2737 │ 3.73 │ 0.47 │
├───────┼───────┤ 40000 │ 500 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.42 │ 0.26 │ 1.62 │ 1022 │ 1346 │ 1404428 │ 1924040 │ 0 │ 0 │ 91 │ 247 │ 2745 │ 4.14 │ 0.49 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.43 │ 0.26 │ 1.65 │ 1084 │ 1372 │ 1468824 │ 1964852 │ 0 │ 0 │ 112 │ 235 │ 2780 │ 4.61 │ 0.49 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.22 │ 0.26 │ 0.85 │ 928 │ 1207 │ 1250884 │ 1717862 │ 0 │ 0 │ 90 │ 233 │ 2712 │ 3.98 │ 0.45 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.23 │ 0.25 │ 0.92 │ 968 │ 1250 │ 1308312 │ 1771312 │ 0 │ 0 │ 84 │ 222 │ 2810 │ 4.36 │ 0.44 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.20 │ 0.23 │ 0.87 │ 846 │ 1100 │ 1120164 │ 1573434 │ 0 │ 0 │ 85 │ 230 │ 2643 │ 3.68 │ 0.42 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.22 │ 0.28 │ 0.79 │ 879 │ 1161 │ 1184646 │ 1640878 │ 0 │ 0 │ 92 │ 235 │ 2752 │ 3.74 │ 0.42 │
├───────┼───────┤ 64000 │ 50 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.27 │ 0.25 │ 1.08 │ 1083 │ 1423 │ 1482126 │ 2042986 │ 0 │ 0 │ 85 │ 234 │ 2679 │ 4.63 │ 0.53 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.28 │ 0.28 │ 1.00 │ 1226 │ 1531 │ 1693124 │ 2175962 │ 0 │ 0 │ 97 │ 256 │ 2823 │ 4.79 │ 0.54 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.24 │ 0.26 │ 0.92 │ 1000 │ 1293 │ 1357674 │ 1840914 │ 0 │ 0 │ 88 │ 235 │ 2741 │ 4.26 │ 0.47 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.18 │ 0.28 │ 0.64 │ 733 │ 968 │ 934722 │ 1368346 │ 0 │ 0 │ 79 │ 227 │ 4022 │ 3.23 │ 0.24 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.18 │ 0.25 │ 0.72 │ 769 │ 1022 │ 988370 │ 1436872 │ 0 │ 0 │ 82 │ 222 │ 2804 │ 3.46 │ 0.36 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.23 │ 0.24 │ 0.96 │ 986 │ 1256 │ 1337620 │ 1774860 │ 0 │ 0 │ 85 │ 243 │ 2759 │ 4.06 │ 0.46 │
├───────┼───────┤ 64000 │ 100 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.22 │ 0.27 │ 0.81 │ 913 │ 1147 │ 1233458 │ 1639544 │ 0 │ 0 │ 79 │ 218 │ 2866 │ 4.19 │ 0.40 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.20 │ 0.24 │ 0.83 │ 808 │ 1066 │ 1058680 │ 1506640 │ 0 │ 0 │ 82 │ 218 │ 2638 │ 3.71 │ 0.40 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.20 │ 0.26 │ 0.77 │ 841 │ 1091 │ 1110570 │ 1545252 │ 0 │ 0 │ 81 │ 225 │ 3017 │ 3.74 │ 0.36 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.25 │ 0.28 │ 0.89 │ 1014 │ 1293 │ 1373748 │ 1841014 │ 0 │ 0 │ 97 │ 231 │ 2825 │ 4.39 │ 0.46 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.30 │ 0.25 │ 1.20 │ 1224 │ 1572 │ 1632904 │ 2263914 │ 0 │ 0 │ 113 │ 256 │ 2772 │ 4.78 │ 0.57 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.28 │ 0.24 │ 1.17 │ 1125 │ 1491 │ 1502834 │ 2122642 │ 0 │ 0 │ 116 │ 248 │ 2690 │ 4.54 │ 0.55 │
├───────┼───────┤ 64000 │ 250 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.20 │ 0.28 │ 0.71 │ 878 │ 1109 │ 1153140 │ 1570166 │ 0 │ 0 │ 84 │ 223 │ 2696 │ 3.94 │ 0.41 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.27 │ 0.25 │ 1.08 │ 1066 │ 1439 │ 1457812 │ 2056818 │ 0 │ 0 │ 89 │ 230 │ 2902 │ 4.63 │ 0.50 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.26 │ 0.26 │ 1.00 │ 1061 │ 1380 │ 1424087 │ 1970910 │ 0 │ 0 │ 99 │ 237 │ 2777 │ 4.48 │ 0.50 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.27 │ 0.29 │ 0.93 │ 1084 │ 1439 │ 1441392 │ 2054634 │ 0 │ 0 │ 107 │ 272 │ 2978 │ 3.99 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.32 │ 0.27 │ 1.19 │ 1247 │ 1670 │ 1667838 │ 2404040 │ 0 │ 0 │ 97 │ 240 │ 2668 │ 5.20 │ 0.63 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.24 │ 0.00 │ 624 │ 795 │ 806336 │ 1091926 │ 0 │ 0 │ 82 │ 213 │ 2638 │ 2.93 │ 0.30 │
├───────┼───────┤ 64000 │ 500 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.20 │ 0.27 │ 0.74 │ 863 │ 1114 │ 1126978 │ 1580894 │ 0 │ 0 │ 106 │ 242 │ 2665 │ 3.57 │ 0.42 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.17 │ 0.24 │ 0.71 │ 677 │ 924 │ 887658 │ 1301660 │ 0 │ 0 │ 74 │ 217 │ 2606 │ 3.12 │ 0.35 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.19 │ 0.26 │ 0.73 │ 899 │ 1188 │ 1186040 │ 1686630 │ 0 │ 0 │ 93 │ 236 │ 2711 │ 3.81 │ 0.44 │
├───────┴───────┴───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ Category:TCP_MAERTS │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┬───────┬───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │ thr/ │#tx-pkts│#rx-pkts│ #tx-byts │ #rx-byts │ # │ # │ # │ # │ # │#tpkt/# │#rpkt/#│
│ │ │ │ │ │ │ %CPU │ │ │ │ │re-trans│tx-intr│rx-intr│io_exit│irq_inj│ exit │ irq │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3988.85 │ 22.30 │178.87 │20663566│ 702609 │31283038628│ 46372460 │ 0 │ 87169 │ 61135 │ 1216 │180256 │16993.06│ 3.90 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4135.65 │ 23.64 │174.94 │21423513│ 741730 │32434120362│ 48954446 │ 0 │ 88264 │ 63359 │ 1166 │193022 │18373.51│ 3.84 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3960.26 │ 22.88 │173.09 │20514440│ 676178 │31058530968│ 44628032 │ 1 │ 87201 │ 60656 │ 1141 │179680 │17979.35│ 3.76 │
├───────┼───────┤ 4000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4066.22 │ 22.55 │180.32 │21064036│ 735381 │31889415224│ 48535460 │ 3 │ 87256 │ 65406 │ 1311 │184763 │16067.15│ 3.98 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3974.81 │ 22.83 │174.10 │20590232│ 658115 │31172175872│ 43435856 │ 0 │ 85508 │ 61146 │ 1135 │179836 │18141.17│ 3.66 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4025.16 │ 22.84 │176.23 │20851157│ 702802 │31567456210│ 46385250 │ 0 │ 87079 │ 62340 │ 1193 │183511 │17477.92│ 3.83 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4019.54 │ 23.36 │172.07 │20823401│ 877237 │31523871386│ 57898240 │ 4 │ 87464 │ 60938 │ 583 │182355 │35717.67│ 4.81 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3918.00 │ 23.03 │170.13 │20297940│ 848490 │30728910688│ 56000908 │ 2 │ 85495 │ 58819 │ 562 │178602 │36117.33│ 4.75 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4070.35 │ 23.91 │170.24 │21085277│ 871966 │31922564178│ 57550288 │ 0 │ 86792 │ 61284 │ 639 │190685 │32997.30│ 4.57 │
├───────┼───────┤ 4000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3906.16 │ 23.10 │169.10 │20235381│ 837776 │30635105466│ 55293766 │ 0 │ 84345 │ 58857 │ 666 │177246 │30383.45│ 4.73 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3880.11 │ 23.62 │164.27 │20101370│ 838017 │30431057796│ 55309696 │ 2 │ 83058 │ 58270 │ 657 │180588 │30595.69│ 4.64 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3958.83 │ 23.40 │169.18 │20508673│ 854697 │31048301902│ 56410579 │ 1 │ 85430 │ 59633 │ 621 │181895 │33025.24│ 4.70 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3870.63 │ 24.01 │161.21 │20051651│1193761 │30356283574│ 78789404 │ 0 │ 80351 │ 57986 │ 546 │178208 │36724.64│ 6.70 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3956.89 │ 23.74 │166.68 │20500511│1276891 │31033832414│ 84275894 │ 0 │ 83805 │ 59307 │ 572 │180002 │35840.05│ 7.09 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3761.69 │ 23.31 │161.38 │19488129│1305199 │29503136162│ 86144252 │ 0 │ 79995 │ 56420 │ 527 │167531 │36979.37│ 7.79 │
├───────┼───────┤ 4000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3961.93 │ 23.58 │168.02 │20524829│1288000 │31073196066│ 85009094 │ 0 │ 84571 │ 59466 │ 645 │176957 │31821.44│ 7.28 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3955.87 │ 23.88 │165.66 │20494696│1273419 │31026429192│ 84046772 │ 0 │ 83048 │ 59290 │ 859 │184184 │23858.78│ 6.91 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3901.40 │ 23.70 │164.62 │20211963│1267454 │30598575481│ 83653083 │ 0 │ 82354 │ 58493 │ 629 │177376 │32133.49│ 7.15 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3939.69 │ 22.47 │175.33 │20408390│ 636433 │30896854212│ 42004868 │ 0 │ 83640 │ 60558 │ 1086 │173872 │18792.26│ 3.66 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4050.50 │ 23.03 │175.88 │20982717│ 692047 │31766669514│ 45675398 │ 0 │ 86543 │ 61909 │ 1138 │183047 │18438.24│ 3.78 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4115.30 │ 22.36 │184.05 │21317685│ 717320 │32273842266│ 47343434 │ 0 │ 88329 │ 66337 │ 1164 │184388 │18314.16│ 3.89 │
├───────┼───────┤ 20000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3920.16 │ 22.16 │176.90 │20307155│ 636981 │30744055198│ 42041072 │ 0 │ 84683 │ 59989 │ 1135 │172394 │17891.77│ 3.69 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3926.43 │ 30.41 │129.12 │20340064│ 639890 │30793586352│ 42233018 │ 0 │ 82309 │ 61869 │ 21480 │303862 │ 946.93 │ 2.11 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3990.42 │ 24.09 │165.65 │20671202│ 664534 │31295001508│ 43859558 │ 0 │ 85100 │ 62132 │ 5200 │203512 │3975.23 │ 3.27 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3959.39 │ 22.58 │175.35 │20510479│ 805041 │31052727934│ 53133328 │ 0 │ 86778 │ 59644 │ 734 │178305 │27943.43│ 4.51 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4050.01 │ 22.78 │177.79 │20980461│ 877866 │31763343418│ 57939700 │ 0 │ 86554 │ 60885 │ 700 │178383 │29972.09│ 4.92 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4011.19 │ 23.07 │173.87 │20779629│ 871885 │31459062858│ 57544960 │ 0 │ 85376 │ 60277 │ 755 │177244 │27522.69│ 4.92 │
├───────┼───────┤ 20000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3848.50 │ 22.61 │170.21 │19937377│ 830338 │30183571434│ 54802870 │ 0 │ 83977 │ 57901 │ 627 │170302 │31798.05│ 4.88 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3846.84 │ 23.13 │166.31 │19927805│ 829706 │30169904626│ 54761182 │ 3 │ 82855 │ 57968 │ 1396 │174693 │14274.93│ 4.75 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3943.19 │ 22.83 │172.72 │20427150│ 842967 │30925722054│ 55636408 │ 0 │ 85108 │ 59335 │ 842 │175785 │24260.27│ 4.80 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3818.62 │ 23.43 │162.98 │19781592│1224374 │29948462512│ 80809886 │ 7 │ 79444 │ 58162 │ 2085 │170555 │9487.57 │ 7.18 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3762.95 │ 23.13 │162.69 │19494796│1309165 │29512793160│ 86406026 │ 0 │ 78322 │ 56323 │ 577 │163565 │33786.47│ 8.00 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3837.60 │ 23.06 │166.42 │19880774│1283988 │30097771036│ 84744350 │ 0 │ 80364 │ 57805 │ 1510 │166688 │13166.08│ 7.70 │
├───────┼───────┤ 20000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3967.47 │ 23.38 │169.70 │20553525│1383095 │31116665722│ 91285400 │ 2 │ 83863 │ 59523 │ 558 │168070 │36834.27│ 8.23 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 7789.67 │ 22.70 │343.16 │40359254│1763314 │61096751388│ 116379836 │ 0 │167101 │117674 │ 1359 │338058 │29697.76│ 5.22 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4635.26 │ 23.14 │200.31 │24013988│1392787 │36354488763│ 91925099 │ 1 │ 97818 │ 69897 │ 1217 │201387 │19732.12│ 6.92 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4052.75 │ 22.18 │182.72 │20994287│ 702295 │31784193038│ 46351760 │ 0 │ 86061 │ 65178 │ 1092 │181116 │19225.54│ 3.88 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4018.67 │ 22.03 │182.42 │20818007│ 687128 │31517188942│ 45350732 │ 0 │ 85422 │ 64407 │ 1122 │178529 │18554.37│ 3.85 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4033.66 │ 21.89 │184.27 │20895124│ 701588 │31634158688│ 46305074 │ 0 │ 86101 │ 64900 │ 1097 │178966 │19047.52│ 3.92 │
├───────┼───────┤ 40000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3961.68 │ 22.21 │178.37 │20522707│ 637825 │31070139166│ 42096764 │ 0 │ 83961 │ 60869 │ 1504 │178778 │13645.42│ 3.57 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4046.57 │ 22.13 │182.85 │20962093│ 700785 │31735498642│ 46252088 │ 0 │ 85783 │ 64782 │ 1087 │180576 │19284.35│ 3.88 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4022.67 │ 22.09 │182.10 │20838443│ 685924 │31548235695│ 45271283 │ 0 │ 85465 │ 64027 │ 1180 │179593 │17659.70│ 3.82 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4026.57 │ 22.74 │177.07 │20859499│ 854694 │31580307014│ 56410372 │ 0 │ 86596 │ 61059 │ 714 │176946 │29214.98│ 4.83 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3842.87 │ 22.33 │172.09 │19907513│ 888571 │30139156866│ 58646278 │ 0 │ 83138 │ 57684 │ 692 │166072 │28768.08│ 5.35 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3948.38 │ 22.72 │173.78 │20455465│ 854932 │30967374346│ 56426086 │ 0 │ 85990 │ 59397 │ 579 │174277 │35328.96│ 4.91 │
├───────┼───────┤ 40000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3946.23 │ 31.87 │123.82 │20441951│ 938273 │30948726710│ 61926556 │ 0 │ 82487 │ 59234 │ 20108 │311634 │1016.61 │ 3.01 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4034.63 │ 22.73 │177.50 │20901228│ 856543 │31643261456│ 56532460 │ 0 │ 88068 │ 61108 │ 756 │176644 │27647.13│ 4.85 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3959.74 │ 24.48 │161.75 │20513131│ 878602 │31055765278│ 57988350 │ 0 │ 85255 │ 59696 │ 4569 │201114 │4489.63 │ 4.37 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3743.58 │ 22.93 │163.26 │19393237│1321581 │29360433338│ 87225434 │ 0 │ 78689 │ 56048 │ 573 │160706 │33845.09│ 8.22 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3758.76 │ 23.39 │160.70 │19472185│1211187 │29480200370│ 79939478 │ 0 │ 79328 │ 56333 │ 578 │166718 │33688.90│ 7.26 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3729.28 │ 22.77 │163.78 │19320651│1433537 │29249089510│ 94614572 │ 0 │ 78158 │ 55876 │ 573 │158030 │33718.41│ 9.07 │
├───────┼───────┤ 40000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3829.35 │ 23.06 │166.06 │19837596│1271434 │30032803256│ 83915738 │ 0 │ 80307 │ 57932 │ 2044 │166569 │9705.28 │ 7.63 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3665.94 │ 22.59 │162.28 │18991734│1302647 │28752506140│ 85975784 │ 0 │ 77943 │ 54891 │ 512 │157857 │37093.23│ 8.25 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3745.38 │ 22.95 │163.20 │19403080│1308077 │29375006522│ 86334201 │ 0 │ 78885 │ 56216 │ 856 │161976 │22667.15│ 8.08 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3951.11 │ 22.41 │176.31 │20467334│ 641396 │30986757444│ 42332444 │ 0 │ 83827 │ 60678 │ 1124 │173757 │18209.37│ 3.69 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4022.83 │ 21.74 │185.04 │20839153│ 696117 │31548997058│ 45944006 │ 0 │ 87346 │ 63989 │ 1115 │178829 │18689.82│ 3.89 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4084.46 │ 22.88 │178.52 │21158723│ 707793 │32032701670│ 46714628 │ 1 │ 86788 │ 62586 │ 1140 │183126 │18560.28│ 3.87 │
├───────┼───────┤ 64000 │ 1 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4026.67 │ 22.02 │182.86 │20858734│ 700613 │31579477516│ 46240772 │ 0 │ 85411 │ 64538 │ 1127 │178786 │18508.19│ 3.92 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4047.81 │ 21.96 │184.33 │20968346│ 686094 │31745127340│ 45282470 │ 0 │ 85159 │ 64912 │ 1131 │178700 │18539.65│ 3.84 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4026.58 │ 22.20 │181.38 │20858458│ 686402 │31578612205│ 45302864 │ 0 │ 85706 │ 63340 │ 1127 │178639 │18507.95│ 3.84 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3876.48 │ 28.01 │138.40 │20081104│ 862098 │30402419344│ 56899012 │ 0 │ 82929 │ 58601 │ 5979 │269621 │3358.61 │ 3.20 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3946.32 │ 22.82 │172.93 │20443637│ 848843 │30950716154│ 56024182 │ 0 │ 86600 │ 59472 │ 558 │171770 │36637.34│ 4.94 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4037.20 │ 23.29 │173.34 │20913327│ 845757 │31661129038│ 55820536 │ 0 │ 85455 │ 60899 │ 960 │179602 │21784.72│ 4.71 │
├───────┼───────┤ 64000 │ 2 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4042.57 │ 23.58 │171.44 │20942024│ 842460 │31705740536│ 55602898 │ 0 │ 85844 │ 60672 │ 713 │184298 │29371.70│ 4.57 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3836.28 │ 22.24 │172.49 │19873255│ 893759 │30086532486│ 58988704 │ 0 │ 82755 │ 57491 │ 639 │163234 │31100.56│ 5.48 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3947.77 │ 23.99 │164.56 │20450669│ 858583 │30961307511│ 56667066 │ 0 │ 84716 │ 59427 │ 1769 │193705 │11560.58│ 4.43 │
├───────┼───────┼───────┼────────┼──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3673.98 │ 25.74 │142.73 │19034041│1300898 │28816134826│ 85860392 │ 0 │ 77098 │ 55140 │ 14927 │219874 │1275.14 │ 5.92 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3884.87 │ 22.52 │172.51 │20125917│1420081 │30469199442│ 93726458 │ 0 │ 83004 │ 58313 │ 776 │163768 │25935.46│ 8.67 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3742.00 │ 22.75 │164.48 │19385791│1302985 │29349125678│ 85998116 │ 0 │ 78659 │ 56041 │ 522 │160876 │37137.53│ 8.10 │
├───────┼───────┤ 64000 │ 4 ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3811.54 │ 22.81 │167.10 │19745934│1342980 │29894074932│ 88637774 │ 0 │ 80674 │ 57031 │ 536 │163672 │36839.43│ 8.21 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3753.64 │ 23.20 │161.79 │19446003│1222424 │29440289838│ 80681078 │ 0 │ 78992 │ 56190 │ 570 │164890 │34115.79│ 7.41 │
├───────┼───────┤ │ ├──────────┼───────┼───────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3773.21 │ 23.40 │161.25 │19547537│1317873 │29593764943│ 86980763 │ 0 │ 79685 │ 56543 │ 3466 │174616 │5639.80 │ 7.55 │
└───────┴───────┴───────┴────────┴──────────┴───────┴───────┴────────┴────────┴───────────┴───────────┴────────┴───────┴───────┴───────┴───────┴────────┴───────┘
┌───────┬────────────────────────┬──────────┬───────┬────────┬────────┬────────┬───────────┬───────────┬────────┬───────┬───────┬───────┬───────┬────────┬───────┐
│== Raw │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│data of│ Category:TCP_STREAM │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│sample │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
│2 == │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┼───────┬───────┬────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts │ #rx-byts │ # │ # │ # │ # │ # │#tpkt/# │#rpkt/#│
│ │ │ │ │ │ │ │ │ │ │ │re-trans│tx-intr│rx-intr│io_exit│irq_inj│ exit │ irq │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2704.11 │ 22.95 │ 117.83 │2907775 │14012723│ 191913618 │21207695978│ 0 │ 57 │4284600│ 20911 │4334295│ 139.05 │ 3.23 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3888.53 │ 24.51 │ 158.65 │3349580 │20143602│ 221261328 │30497373152│ 0 │ 51 │3179002│283982 │3239097│ 11.80 │ 6.22 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3838.67 │ 24.72 │ 155.29 │3346052 │19885195│ 221044149 │30106152476│ 0 │ 51 │3224560│282863 │3285371│ 11.83 │ 6.05 │
├───────┼───────┤ 4000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4851.42 │ 21.26 │ 228.19 │1825669 │25130037│ 120580914 │38046840086│ 0 │ 27 │1575381│273544 │1636227│ 6.67 │ 15.36 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4885.12 │ 21.95 │ 222.56 │1977029 │25305141│ 130554666 │38311493990│ 0 │ 30 │1570194│243815 │1630899│ 8.11 │ 15.52 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4033.57 │ 23.08 │ 174.76 │2681221 │20895339│ 177070935 │31633911136│ 0 │ 43 │2766747│221023 │2825177│ 12.13 │ 7.40 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3786.22 │ 20.80 │ 182.03 │2983686 │19615638│ 197086348 │29698041134│ 0 │ 45 │605132 │347951 │682299 │ 8.58 │ 28.75 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3652.15 │ 21.43 │ 170.42 │2053121 │18919870│ 135618378 │28644647774│ 0 │ 31 │1543871│310310 │1625181│ 6.62 │ 11.64 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 7441.98 │ 24.34 │ 305.75 │6316423 │38551769│ 417229998 │58367329674│ 0 │ 96 │6246031│593528 │6363931│ 10.64 │ 6.06 │
├───────┼───────┤ 4000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4567.63 │ 23.81 │ 191.84 │2528145 │23660920│ 167012418 │35822600626│ 0 │ 39 │2603634│300354 │2670947│ 8.42 │ 8.86 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4646.79 │ 23.57 │ 197.15 │1974521 │24071090│ 130429014 │36443594598│ 0 │ 30 │2773310│304875 │2842797│ 6.48 │ 8.47 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4818.95 │ 22.79 │ 211.45 │3171179 │24963857│ 209475231 │37795242761│ 0 │ 48 │2754395│371403 │2837031│ 8.54 │ 8.80 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4051.46 │ 24.45 │ 165.70 │1510339 │20993455│ 99856390 │31783956688│ 0 │ 409 │2713109│143234 │2784162│ 10.54 │ 7.54 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3866.81 │ 24.93 │ 155.11 │1574387 │20036882│ 104105382 │30335728454│ 0 │ 649 │2947421│ 69907 │3015449│ 22.52 │ 6.64 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3728.31 │ 22.59 │ 165.04 │1960031 │19317154│ 129472546 │29245991608│ 0 │ 364 │1965742│155990 │2034963│ 12.57 │ 9.49 │
├───────┼───────┤ 4000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4061.86 │ 24.28 │ 167.29 │1736036 │21046177│ 114746536 │31863847236│ 0 │ 328 │2856487│146552 │2925562│ 11.85 │ 7.19 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4110.01 │ 24.28 │ 169.28 │2571713 │21294288│ 169919486 │32239342226│ 0 │ 428 │2050867│200276 │2145812│ 12.84 │ 9.92 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3963.69 │ 24.11 │ 164.40 │1870501 │20537591│ 123620068 │31093773242│ 0 │ 435 │2506725│143191 │2581189│ 13.06 │ 7.96 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4744.82 │ 21.62 │ 219.46 │1881366 │24578233│ 124306620 │37211355172│ 0 │ 28 │1467339│253972 │1527534│ 7.41 │ 16.09 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3769.12 │ 24.58 │ 153.34 │3965906 │19525211│ 262103844 │29561150912│ 0 │ 60 │3072677│313512 │3130847│ 12.65 │ 6.24 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3781.89 │ 24.75 │ 152.80 │3544977 │19591576│ 234446274 │29661625644│ 0 │ 54 │3152734│302048 │3212757│ 11.74 │ 6.10 │
├───────┼───────┤ 20000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2858.49 │ 16.28 │ 175.58 │1355178 │14806747│ 89510196 │22417397168│ 0 │ 20 │191452 │247752 │247840 │ 5.47 │ 59.74 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3767.71 │ 24.45 │ 154.10 │3450112 │19517823│ 228125784 │29549958456│ 0 │ 53 │3057351│294002 │3116880│ 11.73 │ 6.26 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3784.41 │ 22.34 │ 169.40 │2839507 │19603918│ 187698543 │29680297470│ 0 │ 43 │2188310│282257 │2247171│ 10.06 │ 8.72 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4186.58 │ 24.53 │ 170.67 │2286900 │21687371│ 151181964 │32834613584│ 0 │ 35 │3576688│287746 │3643601│ 7.95 │ 5.95 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4248.25 │ 25.40 │ 167.25 │2899171 │22006813│ 191591154 │33318281676│ 0 │ 44 │4008122│307063 │4075157│ 9.44 │ 5.40 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3637.26 │ 21.54 │ 168.86 │2524675 │18842523│ 166796066 │28527543226│ 0 │ 38 │1534341│315457 │1611139│ 8.00 │ 11.70 │
├───────┼───────┤ 20000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2686.80 │ 22.35 │ 120.21 │3252341 │13923063│ 214677438 │21075480536│ 0 │ 66 │4184634│ 20728 │4237244│ 156.91 │ 3.29 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4784.92 │ 24.34 │ 196.59 │3985712 │24789549│ 263338776 │37531341348│ 0 │ 61 │1324642│332932 │1395732│ 11.97 │ 17.76 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3908.76 │ 23.63 │ 165.42 │2989759 │20249863│ 197517079 │30657452074│ 0 │ 48 │2925685│252785 │2992574│ 11.83 │ 6.77 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 8022.09 │ 25.90 │ 309.73 │7152232 │41566518│ 472811332 │62931646336│ 0 │ 123 │4113976│644581 │4331730│ 11.10 │ 9.60 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4712.27 │ 30.61 │ 153.95 │4575153 │24414463│ 302214850 │36963432360│ 0 │ 312 │2904389│156807 │3056868│ 29.18 │ 7.99 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3901.15 │ 24.50 │ 159.23 │1813769 │20214332│ 119904282 │30604350588│ 0 │ 637 │2674732│ 74532 │2741872│ 24.34 │ 7.37 │
├───────┼───────┤ 20000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4452.37 │ 23.25 │ 191.50 │1787954 │23065956│ 118163796 │34921688978│ 0 │ 410 │2121276│166916 │2191769│ 10.71 │ 10.52 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4624.39 │ 26.32 │ 175.70 │2673177 │23957143│ 176660494 │36271036848│ 0 │ 222 │2545215│201657 │2642107│ 13.26 │ 9.07 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 5142.45 │ 26.12 │ 196.88 │3600457 │26643682│ 237950950 │40338431022│ 0 │ 340 │2871917│248898 │2992869│ 14.47 │ 8.90 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2880.97 │ 16.61 │ 173.45 │1080285 │14923347│ 71344822 │22593863376│ 0 │ 17 │194394 │228813 │250948 │ 4.72 │ 59.47 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3772.68 │ 24.77 │ 152.31 │4077197 │19543225│ 269431086 │29588419476│ 0 │ 62 │2953852│292887 │3011843│ 13.92 │ 6.49 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4849.43 │ 24.25 │ 199.97 │1897426 │25119934│ 125365824 │38031556984│ 0 │ 29 │1716778│257484 │1779754│ 7.37 │ 14.11 │
├───────┼───────┤ 40000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4858.75 │ 21.38 │ 227.26 │1755987 │25168751│ 116029578 │38105440512│ 0 │ 27 │1602459│260765 │1664066│ 6.73 │ 15.12 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3786.37 │ 24.13 │ 156.92 │3700233 │19614201│ 244671618 │29695868172│ 0 │ 56 │3087133│295534 │3146233│ 12.52 │ 6.23 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4029.64 │ 22.23 │ 181.27 │2502225 │20873891│ 165368585 │31603029704│ 0 │ 38 │1910923│267096 │1970568│ 9.37 │ 10.59 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3639.98 │ 21.40 │ 170.09 │2343193 │18856650│ 154776390 │28548931822│ 0 │ 35 │1450223│305223 │1530321│ 7.68 │ 12.32 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3519.15 │ 21.09 │ 166.86 │2589904 │18230675│ 171086228 │27601210992│ 0 │ 40 │1406963│348234 │1483017│ 7.44 │ 12.29 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3897.87 │ 24.19 │ 161.14 │1395597 │20193214│ 92202474 │30572469616│ 0 │ 134 │3009384│ 84229 │3071018│ 16.57 │ 6.58 │
├───────┼───────┤ 40000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4088.77 │ 23.70 │ 172.52 │1725490 │21182211│ 113977944 │32069835208│ 0 │ 46 │2248564│103394 │2307795│ 16.69 │ 9.18 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4595.19 │ 23.77 │ 193.32 │2272436 │23805069│ 150117360 │36040841940│ 0 │ 35 │2747524│278929 │2813597│ 8.15 │ 8.46 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3948.19 │ 22.83 │ 172.94 │2065324 │20453563│ 136432079 │30966657915│ 0 │ 58 │2172531│224001 │2241149│ 9.22 │ 9.13 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4042.30 │ 25.90 │ 156.07 │3399939 │20941546│ 224574062 │31705416286│ 0 │ 438 │1571217│235937 │1719404│ 14.41 │ 12.18 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4073.20 │ 24.79 │ 164.31 │1623021 │21103387│ 107293106 │31950461424│ 0 │ 424 │2910404│142654 │2979999│ 11.38 │ 7.08 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4008.18 │ 25.92 │ 154.64 │3101260 │20767754│ 204855532 │31442279760│ 0 │ 485 │2372529│186924 │2491231│ 16.59 │ 8.34 │
├───────┼───────┤ 40000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4324.08 │ 23.84 │ 181.38 │1703347 │22403968│ 112636030 │33919514738│ 0 │ 371 │2409793│156562 │2479710│ 10.88 │ 9.03 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3824.55 │ 22.26 │ 171.81 │1378259 │19814667│ 91121678 │29999199560│ 0 │ 433 │2077285│136274 │2148403│ 10.11 │ 9.22 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4054.46 │ 24.54 │ 165.22 │2241165 │21006264│ 148096081 │31803374353│ 0 │ 430 │2268245│171670 │2363749│ 13.06 │ 8.89 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2875.73 │ 16.55 │ 173.76 │1046616 │14896281│ 69128628 │22552951756│ 0 │ 16 │186818 │234643 │242766 │ 4.46 │ 61.36 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2773.04 │ 22.77 │ 121.78 │2472146 │14367855│ 163162140 │21748293224│ 0 │ 39 │4464194│ 21370 │4514215│ 115.68 │ 3.18 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2920.04 │ 0.24 │12166.83│1010770 │15125856│ 66761964 │22900524916│ 0 │ 15 │200458 │249728 │260159 │ 4.05 │ 58.14 │
├───────┼───────┤ 64000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4798.54 │ 21.48 │ 223.40 │2009615 │24856224│ 132749118 │37632239906│ 0 │ 31 │1562879│272276 │1623563│ 7.38 │ 15.31 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3793.02 │ 24.26 │ 156.35 │3097696 │19648800│ 204622872 │29748262530│ 0 │ 47 │3089358│298467 │3150032│ 10.38 │ 6.24 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3432.07 │ 17.06 │ 201.18 │1927368 │17779003│ 127284944 │26916454466│ 0 │ 29 │1900741│215296 │1958147│ 8.95 │ 9.08 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4173.98 │ 24.44 │ 170.78 │2327586 │21623077│ 153753300 │32737291364│ 0 │ 35 │3475950│277360 │3541858│ 8.39 │ 6.11 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3133.37 │ 21.89 │ 143.14 │1591111 │16235089│ 105073710 │24578920852│ 0 │ 41 │2520960│ 65417 │2582043│ 24.32 │ 6.29 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3622.76 │ 21.53 │ 168.27 │2592610 │18768922│ 171253880 │28416104520│ 0 │ 40 │1555293│357017 │1637673│ 7.26 │ 11.46 │
├───────┼───────┤ 64000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4557.01 │ 23.66 │ 192.60 │2351702 │23606228│ 155409804 │35739794914│ 0 │ 36 │2630580│297393 │2698853│ 7.91 │ 8.75 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4100.16 │ 25.00 │ 164.01 │2286834 │21239696│ 151055064 │32156867554│ 0 │ 35 │3766742│271508 │3832963│ 8.42 │ 5.54 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3917.46 │ 23.30 │ 168.13 │2229968 │20294602│ 147309151 │30725795840│ 0 │ 37 │2789905│253739 │2858678│ 8.79 │ 7.10 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4385.07 │ 27.80 │ 157.74 │3430323 │22718377│ 226572970 │34395529388│ 0 │ 278 │1854662│208876 │1998101│ 16.42 │ 11.37 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4512.11 │ 26.14 │ 172.61 │3209923 │23376545│ 212052366 │35392003060│ 0 │ 292 │1536969│221988 │1675066│ 14.46 │ 13.96 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 8433.51 │ 21.98 │ 383.69 │2823699 │43691063│ 186569130 │66148048156│ 0 │ 237 │3906821│370898 │4035657│ 7.61 │ 10.83 │
├───────┼───────┤ 64000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4462.51 │ 29.99 │ 148.80 │4877568 │23119148│ 322251516 │35002322858│ 0 │ 701 │3029784│126916 │3184855│ 38.43 │ 7.26 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4459.60 │ 25.59 │ 174.27 │3111864 │23103899│ 205536912 │34979240304│ 0 │ 520 │2473750│209567 │2558301│ 14.85 │ 9.03 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 5250.56 │ 26.30 │ 199.64 │3490675 │27201806│ 230596578 │41183428753│ 0 │ 405 │2560397│227649 │2690396│ 15.33 │ 10.11 │
├───────┴───────┴───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ Category:TCP_RR │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┬───────┬───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts │ #rx-byts │ # │ # │ # │ # │ # │#tpkt/# │#rpkt/#│
│ │ │ │ │ │ │ │ │ │ │ │re-trans│tx-intr│rx-intr│io_exit│irq_inj│ exit │ irq │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 36356.50 │ 21.56 │1686.29 │6329707 │7305598 │9143557878 │9207958148 │ 6 │ 96 │368276 │140693 │408918 │ 44.99 │ 17.87 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 36851.57 │ 21.72 │1696.67 │6417742 │7522264 │9268180532 │9341066308 │ 9 │ 98 │394656 │142462 │433435 │ 45.05 │ 17.35 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 31402.25 │ 18.59 │1689.20 │5466008 │6562438 │7897527952 │7969889264 │ 2 │ 83 │159095 │124398 │190401 │ 43.94 │ 34.47 │
├───────┼───────┤ 4000 │ 50 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 49212.54 │ 21.74 │2263.69 │8415229 │8994054 │12366661314│12404856280│ 6 │ 128 │178364 │218356 │233510 │ 38.54 │ 38.52 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 30853.70 │ 21.25 │1451.93 │5355604 │6476282 │7758578736 │7832539160 │ 3 │ 81 │137886 │118588 │170090 │ 45.16 │ 38.08 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 36935.31 │ 20.97 │1761.34 │6396858 │7372127 │9286901282 │9351261832 │ 5 │ 97 │247655 │148899 │287270 │ 42.96 │ 25.66 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 41946.49 │ 23.08 │1817.44 │7201206 │8953859 │10542855364│10658525090│ 5 │ 110 │190063 │126137 │237546 │ 57.09 │ 37.69 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 37981.85 │ 22.51 │1687.33 │6517503 │8277587 │9546220070 │9662382286 │ 3 │ 99 │493987 │ 70974 │541265 │ 91.83 │ 15.29 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 50123.47 │ 22.06 │2272.14 │8432605 │9343192 │12586656774│12646735932│ 19 │ 130 │187073 │221880 │249952 │ 38.01 │ 37.38 │
├───────┼───────┤ 4000 │ 100 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 39227.62 │ 22.26 │1762.25 │6755439 │8640974 │9860901638 │9985339124 │ 7 │ 104 │135138 │ 84459 │179422 │ 79.98 │ 48.16 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 44694.70 │ 23.39 │1910.85 │7636640 │9310875 │11231178352│11341667606│ 8 │ 117 │283512 │173613 │333668 │ 43.99 │ 27.90 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 42794.83 │ 22.66 │1888.56 │7308678 │8905297 │10753562439│10858930007│ 8 │ 112 │257954 │135412 │308370 │ 53.97 │ 28.88 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 4.00 │ 0.28 │ 14.29 │ 957 │ 1227 │ 1273466 │ 1689722 │ 0 │ 0 │ 105 │ 261 │ 2797 │ 3.67 │ 0.44 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 899 │ 1160 │ 1191638 │ 1606878 │ 0 │ 0 │ 111 │ 263 │ 2695 │ 3.42 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.72 │ 0.26 │ 14.31 │ 909 │ 1170 │ 1202626 │ 1616872 │ 0 │ 0 │ 129 │ 253 │ 2691 │ 3.59 │ 0.43 │
├───────┼───────┤ 4000 │ 250 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 49676.99 │ 23.86 │2082.02 │8340981 │10064491│12473985274│12587736678│ 0 │ 9808 │1110558│ 21149 │1173132│ 394.39 │ 8.58 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 790 │ 986 │ 1044732 │ 1364386 │ 0 │ 0 │ 86 │ 239 │ 2609 │ 3.31 │ 0.38 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 9936.94 │ 4.98 │1995.37 │1668907 │2013806 │2495739547 │2518802907 │ 0 │ 1961 │222197 │ 4433 │236784 │ 376.47 │ 8.50 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 636 │ 824 │ 815040 │ 1120318 │ 0 │ 0 │ 83 │ 224 │ 2591 │ 2.84 │ 0.32 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.92 │ 0.26 │ 15.08 │ 939 │ 1240 │ 1256934 │ 1714956 │ 0 │ 0 │ 114 │ 248 │ 2730 │ 3.79 │ 0.45 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3.88 │ 0.26 │ 14.92 │ 972 │ 1236 │ 1277104 │ 1693474 │ 0 │ 0 │ 127 │ 274 │ 2845 │ 3.55 │ 0.43 │
├───────┼───────┤ 4000 │ 500 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 1132 │ 1496 │ 1535616 │ 2052668 │ 0 │ 0 │ 108 │ 273 │ 2735 │ 4.15 │ 0.55 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.27 │ 0.00 │ 825 │ 1081 │ 1069850 │ 1488438 │ 0 │ 0 │ 105 │ 278 │ 2612 │ 2.97 │ 0.41 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 1.56 │ 0.27 │ 5.78 │ 900 │ 1175 │ 1190908 │ 1613970 │ 0 │ 0 │ 107 │ 259 │ 2702 │ 3.47 │ 0.43 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 11344.08 │ 22.87 │ 496.02 │9824640 │10055813│14262352560│14277604386│ 4 │ 30659 │1226543│ 10261 │1305288│ 957.47 │ 7.70 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.65 │ 0.28 │ 2.32 │ 814 │ 1053 │ 1062260 │ 1454878 │ 0 │ 0 │ 91 │ 227 │ 2686 │ 3.59 │ 0.39 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.73 │ 0.23 │ 3.17 │ 896 │ 1161 │ 1186760 │ 1623804 │ 0 │ 0 │ 99 │ 230 │ 2625 │ 3.90 │ 0.44 │
├───────┼───────┤ 20000 │ 50 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 11850.73 │ 24.29 │ 487.89 │9969415 │10802298│14879924790│14934892060│ 2 │ 33254 │1321477│ 10110 │1400512│ 986.09 │ 7.71 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.77 │ 0.27 │ 2.85 │ 908 │ 1226 │ 1236176 │ 1732848 │ 0 │ 0 │ 87 │ 219 │ 2694 │ 4.15 │ 0.46 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 4639.39 │ 9.59 │ 483.77 │3959334 │4172310 │5829152509 │5843461595 │ 1 │ 12782 │509659 │ 4209 │542761 │ 940.68 │ 7.69 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.77 │ 0.26 │ 2.96 │ 948 │ 1229 │ 1275136 │ 1730702 │ 0 │ 0 │ 86 │ 237 │ 2773 │ 4.00 │ 0.44 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 868 │ 1113 │ 1170752 │ 1558662 │ 0 │ 0 │ 92 │ 238 │ 2844 │ 3.65 │ 0.39 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.67 │ 0.25 │ 2.68 │ 808 │ 1056 │ 1082280 │ 1475388 │ 0 │ 0 │ 83 │ 230 │ 2856 │ 3.51 │ 0.37 │
├───────┼───────┤ 20000 │ 100 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.88 │ 0.24 │ 3.67 │ 1084 │ 1395 │ 1481608 │ 1986746 │ 0 │ 0 │ 110 │ 247 │ 3965 │ 4.39 │ 0.35 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.83 │ 0.28 │ 2.96 │ 999 │ 1309 │ 1355190 │ 1861158 │ 0 │ 0 │ 93 │ 239 │ 2707 │ 4.18 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.63 │ 0.26 │ 2.42 │ 941 │ 1220 │ 1272993 │ 1722531 │ 0 │ 0 │ 92 │ 238 │ 3029 │ 3.95 │ 0.40 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 934 │ 1250 │ 1265012 │ 1757392 │ 0 │ 0 │ 100 │ 237 │ 2644 │ 3.94 │ 0.47 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 1200 │ 1563 │ 1653520 │ 2219130 │ 0 │ 0 │ 111 │ 253 │ 2738 │ 4.74 │ 0.57 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.25 │ 0.00 │ 803 │ 1014 │ 1083878 │ 1406896 │ 0 │ 0 │ 91 │ 229 │ 2616 │ 3.51 │ 0.39 │
├───────┼───────┤ 20000 │ 250 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.77 │ 0.25 │ 3.08 │ 915 │ 1224 │ 1233790 │ 1711772 │ 0 │ 0 │ 113 │ 239 │ 2691 │ 3.83 │ 0.45 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.27 │ 0.00 │ 616 │ 814 │ 813272 │ 1112280 │ 0 │ 0 │ 74 │ 211 │ 2641 │ 2.92 │ 0.31 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.15 │ 0.26 │ 0.58 │ 893 │ 1173 │ 1209894 │ 1641494 │ 0 │ 0 │ 97 │ 233 │ 2666 │ 3.83 │ 0.44 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 845 │ 1092 │ 1126410 │ 1516820 │ 0 │ 0 │ 93 │ 232 │ 2607 │ 3.64 │ 0.42 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.62 │ 0.28 │ 2.21 │ 749 │ 993 │ 1003666 │ 1390734 │ 0 │ 0 │ 82 │ 218 │ 2820 │ 3.44 │ 0.35 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.27 │ 0.00 │ 841 │ 1091 │ 1133178 │ 1516794 │ 0 │ 0 │ 93 │ 227 │ 2615 │ 3.70 │ 0.42 │
├───────┼───────┤ 20000 │ 500 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.85 │ 0.25 │ 3.40 │ 1037 │ 1350 │ 1429114 │ 1891752 │ 0 │ 0 │ 111 │ 243 │ 2690 │ 4.27 │ 0.50 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.85 │ 0.29 │ 2.93 │ 1027 │ 1369 │ 1389686 │ 1929630 │ 0 │ 0 │ 112 │ 246 │ 2854 │ 4.17 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.46 │ 0.27 │ 1.70 │ 899 │ 1179 │ 1216410 │ 1649146 │ 0 │ 0 │ 98 │ 233 │ 2717 │ 3.86 │ 0.43 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.40 │ 0.25 │ 1.60 │ 963 │ 1288 │ 1333486 │ 1838874 │ 0 │ 0 │ 92 │ 219 │ 2708 │ 4.40 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.29 │ 0.00 │ 888 │ 1143 │ 1200232 │ 1621170 │ 0 │ 0 │ 81 │ 223 │ 2700 │ 3.98 │ 0.42 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.35 │ 0.26 │ 1.35 │ 905 │ 1129 │ 1163978 │ 1578790 │ 0 │ 0 │ 82 │ 213 │ 2650 │ 4.25 │ 0.43 │
├───────┼───────┤ 40000 │ 50 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.27 │ 0.00 │ 772 │ 1005 │ 1028608 │ 1410784 │ 0 │ 0 │ 92 │ 233 │ 2628 │ 3.31 │ 0.38 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 1074 │ 1327 │ 1366548 │ 1878404 │ 0 │ 0 │ 90 │ 235 │ 2602 │ 4.57 │ 0.51 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.15 │ 0.27 │ 0.56 │ 920 │ 1178 │ 1218570 │ 1665604 │ 0 │ 0 │ 87 │ 224 │ 2657 │ 4.11 │ 0.44 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 1069 │ 1368 │ 1463354 │ 1961296 │ 0 │ 0 │ 88 │ 240 │ 2843 │ 4.45 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.30 │ 0.28 │ 1.07 │ 725 │ 978 │ 972450 │ 1369912 │ 0 │ 0 │ 76 │ 223 │ 2912 │ 3.25 │ 0.34 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 991 │ 1305 │ 1358838 │ 1839990 │ 0 │ 0 │ 96 │ 244 │ 2798 │ 4.06 │ 0.47 │
├───────┼───────┤ 40000 │ 100 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.24 │ 0.00 │ 985 │ 1286 │ 1333938 │ 1828972 │ 0 │ 0 │ 94 │ 231 │ 2640 │ 4.26 │ 0.49 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.35 │ 0.26 │ 1.35 │ 890 │ 1145 │ 1217428 │ 1621302 │ 0 │ 0 │ 78 │ 209 │ 2663 │ 4.26 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.13 │ 0.26 │ 0.50 │ 932 │ 1216 │ 1269201 │ 1724294 │ 0 │ 0 │ 86 │ 229 │ 2771 │ 4.07 │ 0.44 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.25 │ 0.00 │ 880 │ 1114 │ 1186088 │ 1578700 │ 0 │ 0 │ 77 │ 223 │ 2862 │ 3.95 │ 0.39 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.33 │ 0.31 │ 1.06 │ 845 │ 1111 │ 1147746 │ 1578258 │ 0 │ 0 │ 90 │ 227 │ 2869 │ 3.72 │ 0.39 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.24 │ 0.00 │ 1008 │ 1347 │ 1384200 │ 1906276 │ 0 │ 0 │ 93 │ 244 │ 2663 │ 4.13 │ 0.51 │
├───────┼───────┤ 40000 │ 250 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.35 │ 0.25 │ 1.40 │ 921 │ 1183 │ 1224106 │ 1661964 │ 0 │ 0 │ 91 │ 221 │ 2736 │ 4.17 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.27 │ 0.00 │ 941 │ 1215 │ 1221122 │ 1721868 │ 0 │ 0 │ 85 │ 233 │ 2704 │ 4.04 │ 0.45 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.14 │ 0.26 │ 0.54 │ 919 │ 1194 │ 1232652 │ 1689413 │ 0 │ 0 │ 87 │ 229 │ 2766 │ 4.01 │ 0.43 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.28 │ 0.26 │ 1.08 │ 722 │ 969 │ 949508 │ 1365904 │ 0 │ 0 │ 85 │ 221 │ 2938 │ 3.27 │ 0.33 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.29 │ 0.00 │ 1063 │ 1443 │ 1445038 │ 2055348 │ 0 │ 0 │ 111 │ 249 │ 2586 │ 4.27 │ 0.56 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.37 │ 0.25 │ 1.48 │ 898 │ 1200 │ 1221324 │ 1711300 │ 0 │ 0 │ 79 │ 230 │ 2747 │ 3.90 │ 0.44 │
├───────┼───────┤ 40000 │ 500 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.48 │ 0.26 │ 1.85 │ 1185 │ 1559 │ 1623266 │ 2227442 │ 0 │ 0 │ 96 │ 248 │ 2836 │ 4.78 │ 0.55 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.40 │ 0.24 │ 1.67 │ 981 │ 1297 │ 1334458 │ 1835478 │ 0 │ 0 │ 91 │ 230 │ 2677 │ 4.27 │ 0.48 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.31 │ 0.26 │ 1.19 │ 969 │ 1293 │ 1314718 │ 1839094 │ 0 │ 0 │ 92 │ 235 │ 2756 │ 4.12 │ 0.47 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.22 │ 0.26 │ 0.85 │ 903 │ 1151 │ 1209982 │ 1637342 │ 0 │ 0 │ 90 │ 223 │ 2840 │ 4.05 │ 0.41 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.18 │ 0.27 │ 0.67 │ 779 │ 1015 │ 1020414 │ 1439818 │ 0 │ 0 │ 78 │ 222 │ 2817 │ 3.51 │ 0.36 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.17 │ 0.26 │ 0.65 │ 691 │ 930 │ 898342 │ 1305624 │ 0 │ 0 │ 77 │ 213 │ 2811 │ 3.24 │ 0.33 │
├───────┼───────┤ 64000 │ 50 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.18 │ 0.27 │ 0.67 │ 732 │ 968 │ 960704 │ 1372004 │ 0 │ 0 │ 68 │ 204 │ 2695 │ 3.59 │ 0.36 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.28 │ 0.25 │ 1.12 │ 1126 │ 1495 │ 1543540 │ 2124226 │ 0 │ 0 │ 101 │ 236 │ 2720 │ 4.77 │ 0.55 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.21 │ 0.26 │ 0.81 │ 846 │ 1111 │ 1126596 │ 1575802 │ 0 │ 0 │ 82 │ 219 │ 2776 │ 3.86 │ 0.40 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.28 │ 0.24 │ 1.17 │ 1106 │ 1486 │ 1487636 │ 2122632 │ 0 │ 0 │ 103 │ 248 │ 2818 │ 4.46 │ 0.53 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.22 │ 0.29 │ 0.76 │ 899 │ 1161 │ 1223934 │ 1641038 │ 0 │ 0 │ 86 │ 219 │ 2681 │ 4.11 │ 0.43 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.26 │ 0.00 │ 1351 │ 1708 │ 1868726 │ 2457422 │ 0 │ 0 │ 108 │ 243 │ 2749 │ 5.56 │ 0.62 │
├───────┼───────┤ 64000 │ 100 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.22 │ 0.28 │ 0.79 │ 893 │ 1151 │ 1176338 │ 1638874 │ 0 │ 0 │ 87 │ 231 │ 2829 │ 3.87 │ 0.41 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.23 │ 0.27 │ 0.85 │ 929 │ 1210 │ 1242026 │ 1708182 │ 0 │ 0 │ 94 │ 226 │ 2726 │ 4.11 │ 0.44 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.19 │ 0.27 │ 0.70 │ 1035 │ 1343 │ 1399732 │ 1913629 │ 0 │ 0 │ 95 │ 233 │ 2760 │ 4.44 │ 0.49 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.25 │ 0.23 │ 1.09 │ 981 │ 1335 │ 1322786 │ 1918634 │ 0 │ 0 │ 91 │ 231 │ 4053 │ 4.25 │ 0.33 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.20 │ 0.29 │ 0.69 │ 794 │ 1071 │ 1043988 │ 1506348 │ 0 │ 0 │ 79 │ 241 │ 4190 │ 3.29 │ 0.26 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.28 │ 0.25 │ 1.12 │ 1102 │ 1488 │ 1496940 │ 2123758 │ 0 │ 0 │ 109 │ 228 │ 2693 │ 4.83 │ 0.55 │
├───────┼───────┤ 64000 │ 250 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.27 │ 0.26 │ 1.04 │ 1037 │ 1398 │ 1413514 │ 1985504 │ 0 │ 0 │ 80 │ 214 │ 2687 │ 4.85 │ 0.52 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.27 │ 0.28 │ 0.96 │ 1040 │ 1393 │ 1412984 │ 1985046 │ 0 │ 0 │ 87 │ 224 │ 2688 │ 4.64 │ 0.52 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.25 │ 0.26 │ 0.96 │ 990 │ 1337 │ 1338042 │ 1903858 │ 0 │ 0 │ 89 │ 227 │ 3262 │ 4.36 │ 0.41 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.22 │ 0.23 │ 0.96 │ 912 │ 1164 │ 1220760 │ 1641364 │ 0 │ 0 │ 84 │ 220 │ 2775 │ 4.15 │ 0.42 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.00 │ 0.28 │ 0.00 │ 1091 │ 1448 │ 1478750 │ 2057400 │ 0 │ 0 │ 92 │ 234 │ 2623 │ 4.66 │ 0.55 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.20 │ 0.21 │ 0.95 │ 794 │ 1069 │ 1056788 │ 1505966 │ 0 │ 0 │ 83 │ 224 │ 3825 │ 3.54 │ 0.28 │
├───────┼───────┤ 64000 │ 500 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.23 │ 0.24 │ 0.96 │ 964 │ 1257 │ 1309096 │ 1774516 │ 0 │ 0 │ 93 │ 231 │ 2740 │ 4.17 │ 0.46 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 0.10 │ 0.26 │ 0.38 │ 464 │ 596 │ 582264 │ 822450 │ 0 │ 0 │ 59 │ 191 │ 2775 │ 2.43 │ 0.21 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 0.15 │ 0.24 │ 0.62 │ 845 │ 1106 │ 1129531 │ 1560339 │ 0 │ 0 │ 82 │ 220 │ 2947 │ 3.84 │ 0.38 │
├───────┴───────┴───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ Category:TCP_MAERTS │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │
├───────┬───────┬───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ # │ Tile │ size │sessions│throughput│ %CPU │thr/%CPU│#tx-pkts│#rx-pkts│ #tx-byts │ #rx-byts │ # │ # │ # │ # │ # │#tpkt/# │#rpkt/#│
│ │ │ │ │ │ │ │ │ │ │ │re-trans│tx-intr│rx-intr│io_exit│irq_inj│ exit │ irq │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2924.86 │ 20.37 │ 143.59 │15153060│ 596993 │22939024640│ 39401828 │ 1 │ 65442 │ 23813 │ 1027 │110030 │14754.68│ 5.43 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3000.11 │ 20.66 │ 145.21 │15545222│ 591445 │23528740948│ 39035672 │ 3 │ 66561 │ 25393 │ 1059 │114212 │14679.15│ 5.18 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3136.45 │ 20.60 │ 152.25 │16248525│ 604735 │24598009138│ 39912824 │ 3 │ 69224 │ 28870 │ 1268 │119171 │12814.29│ 5.07 │
├───────┼───────┤ 4000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3078.63 │ 20.27 │ 151.88 │15947778│ 606104 │24144602852│ 40003130 │ 0 │ 69303 │ 25358 │ 1131 │114809 │14100.60│ 5.28 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3080.67 │ 20.61 │ 149.47 │15961524│ 595110 │24159814912│ 39277610 │ 1 │ 67755 │ 28011 │ 1094 │117709 │14590.06│ 5.06 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3044.14 │ 20.50 │ 148.49 │15771221│ 598877 │23874038498│ 39526212 │ 1 │ 67657 │ 26289 │ 1115 │115186 │14144.59│ 5.20 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2965.27 │ 20.74 │ 142.97 │15363170│ 790720 │23255408228│ 52188166 │ 0 │ 65418 │ 26707 │ 1468 │113316 │10465.37│ 6.98 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2947.36 │ 20.97 │ 140.55 │15268548│ 794787 │23115454336│ 52456486 │ 0 │ 66070 │ 26903 │ 700 │114160 │21812.21│ 6.96 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2960.73 │ 21.41 │ 138.29 │15339370│ 751896 │23221092668│ 49625686 │ 0 │ 65545 │ 25601 │ 811 │115892 │18914.14│ 6.49 │
├───────┼───────┤ 4000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 5907.73 │ 20.45 │ 288.89 │30610205│1177641 │46333281674│ 77724922 │ 2 │125770 │ 51857 │ 5667 │219300 │5401.48 │ 5.37 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3032.53 │ 20.92 │ 144.96 │15711016│ 772496 │23784940232│ 50985322 │ 0 │ 68181 │ 28487 │ 809 │118180 │19420.29│ 6.54 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3562.72 │ 20.90 │ 170.47 │18458461│ 857508 │27942035427│ 56596116 │ 0 │ 78196 │ 31911 │ 1891 │136169 │9761.22 │ 6.30 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2852.03 │ 21.36 │ 133.52 │14777845│1191964 │22369351906│ 78670742 │ 0 │ 62091 │ 28441 │ 556 │113287 │26578.86│ 10.52 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3040.04 │ 21.94 │ 138.56 │15751006│1139198 │23844389428│ 75188192 │ 0 │ 65927 │ 30407 │ 555 │122201 │28380.19│ 9.32 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2887.70 │ 21.77 │ 132.65 │14961461│1109226 │22649146810│ 73210034 │ 0 │ 62700 │ 29235 │ 539 │116775 │27757.81│ 9.50 │
├───────┼───────┤ 4000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2895.58 │ 21.13 │ 137.04 │15002968│1186035 │22711234744│ 78279392 │ 0 │ 63732 │ 29783 │ 518 │115563 │28963.26│ 10.26 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2879.21 │ 21.68 │ 132.80 │14916915│1173726 │22582416798│ 77467046 │ 0 │ 62743 │ 27524 │ 507 │115038 │29421.92│ 10.20 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 2910.91 │ 21.58 │ 134.89 │15082039│1160029 │22831307937│ 76563081 │ 0 │ 63438 │ 29078 │ 535 │116572 │28190.73│ 9.95 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3049.48 │ 20.40 │ 149.48 │15801215│ 572190 │23915846174│ 37764806 │ 0 │ 66915 │ 26969 │ 1112 │113792 │14209.73│ 5.03 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3105.77 │ 20.16 │ 154.06 │16092598│ 573938 │24356811972│ 37880198 │ 0 │ 68960 │ 27455 │ 1068 │115385 │15067.98│ 4.97 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3024.33 │ 20.97 │ 144.22 │15672214│ 571122 │23719355572│ 37694330 │ 0 │ 66545 │ 25304 │ 1147 │113705 │13663.66│ 5.02 │
├───────┼───────┤ 20000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3128.98 │ 20.36 │ 153.68 │16213305│ 574600 │24539185562│ 37923866 │ 0 │ 68775 │ 27714 │ 1085 │116210 │14943.14│ 4.94 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3015.91 │ 19.82 │ 152.16 │15628040│ 570824 │23652582544│ 37674710 │ 0 │ 66952 │ 25248 │ 1043 │111470 │14983.74│ 5.12 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3064.89 │ 20.34 │ 150.68 │15881474│ 572534 │24036756364│ 37787582 │ 0 │ 67629 │ 26538 │ 1091 │114112 │14556.80│ 5.02 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3017.86 │ 20.57 │ 146.71 │15635768│ 770376 │23670471288│ 50845462 │ 0 │ 66599 │ 28225 │ 971 │114210 │16102.75│ 6.75 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2915.46 │ 20.22 │ 144.19 │15104449│ 847355 │22865454450│ 55925980 │ 0 │ 65162 │ 25640 │ 730 │108818 │20691.03│ 7.79 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2989.10 │ 20.45 │ 146.17 │15485617│ 789257 │23444217290│ 52091512 │ 0 │ 66846 │ 27968 │ 739 │113575 │20954.83│ 6.95 │
├───────┼───────┤ 20000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2959.78 │ 20.17 │ 146.74 │15334182│ 823187 │23213729212│ 54330940 │ 0 │ 65965 │ 26989 │ 706 │111125 │21719.80│ 7.41 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3011.53 │ 20.44 │ 147.34 │15601291│ 794027 │23619413438│ 52406344 │ 0 │ 67510 │ 28280 │ 716 │114074 │21789.51│ 6.96 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 2978.75 │ 20.37 │ 146.23 │15432261│ 804840 │23362657135│ 53120047 │ 0 │ 66416 │ 27420 │ 772 │112360 │19989.98│ 7.16 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 5919.16 │ 20.64 │ 286.78 │30669866│1573766 │46423091212│ 103869704 │ 0 │129892 │ 53972 │ 1393 │221189 │22017.13│ 7.12 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2932.03 │ 20.71 │ 141.58 │15190875│1191597 │22995985998│ 78646496 │ 0 │ 64535 │ 31583 │ 519 │115086 │29269.51│ 10.35 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2883.39 │ 20.89 │ 138.03 │14937997│1211603 │22614678186│ 79966964 │ 3 │ 62928 │ 29990 │ 519 │112534 │28782.27│ 10.77 │
├───────┼───────┤ 20000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2902.41 │ 20.85 │ 139.20 │15037120│1181920 │22764686232│ 78007886 │ 0 │ 63619 │ 30671 │ 554 │113895 │27142.82│ 10.38 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2874.86 │ 20.73 │ 138.68 │14894278│1214765 │22548743420│ 80175596 │ 0 │ 63378 │ 29579 │ 529 │111854 │28155.53│ 10.86 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3502.37 │ 20.76 │ 168.71 │18146027│1274730 │27469437009│ 84133329 │ 0 │ 76870 │ 35159 │ 702 │134911 │25849.04│ 9.45 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3026.53 │ 21.15 │ 143.10 │15682830│ 570773 │23737051676│ 37671368 │ 3 │ 64978 │ 25555 │ 1230 │112859 │12750.27│ 5.06 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3007.50 │ 19.92 │ 150.98 │15582909│ 570840 │23586592498│ 37675730 │ 0 │ 66487 │ 26878 │ 1043 │112219 │14940.47│ 5.09 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3051.44 │ 20.19 │ 151.14 │15810348│ 571137 │23930723472│ 37695380 │ 0 │ 67459 │ 27639 │ 1055 │114241 │14986.11│ 5.00 │
├───────┼───────┤ 40000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3104.11 │ 20.32 │ 152.76 │16084451│ 571867 │24344925270│ 37743524 │ 1 │ 68005 │ 27596 │ 1153 │114806 │13950.09│ 4.98 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2994.81 │ 19.77 │ 151.48 │15518401│ 569056 │23487784002│ 37558022 │ 0 │ 66556 │ 24318 │ 1012 │109020 │15334.39│ 5.22 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3036.88 │ 20.27 │ 149.82 │15735787│ 570734 │23817415383│ 37668804 │ 0 │ 66697 │ 26397 │ 1098 │112629 │14331.32│ 5.07 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3002.65 │ 20.07 │ 149.61 │15555724│ 800442 │23549407192│ 52829722 │ 0 │ 66232 │ 28291 │ 777 │112836 │20020.24│ 7.09 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2993.13 │ 20.31 │ 147.37 │15506325│ 798478 │23473944914│ 52700182 │ 0 │ 66886 │ 28241 │ 775 │112960 │20008.16│ 7.07 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3071.56 │ 20.86 │ 147.25 │15913500│ 774753 │24089503224│ 51134320 │ 0 │ 67559 │ 26368 │ 1347 │114552 │11814.03│ 6.76 │
├───────┼───────┤ 40000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 6046.40 │ 20.19 │ 299.47 │31335834│1138243 │47431866492│ 75124678 │ 0 │132886 │ 53864 │ 1767 │223038 │17733.92│ 5.10 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2947.26 │ 20.63 │ 142.86 │15267540│ 729667 │23114728840│ 48158608 │ 2 │ 65443 │ 26431 │ 1386 │111787 │11015.54│ 6.53 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3612.20 │ 20.41 │ 176.98 │18715784│ 848316 │28331890132│ 55989502 │ 0 │ 79801 │ 32639 │ 1210 │135034 │15467.59│ 6.28 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2807.88 │ 20.79 │ 135.06 │14547494│1208116 │22024180476│ 79736750 │ 0 │ 61306 │ 29138 │ 572 │109567 │25432.68│ 11.03 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2917.26 │ 20.27 │ 143.92 │15114879│1255244 │22880404702│ 82847234 │ 0 │ 63728 │ 30183 │ 1436 │112031 │10525.68│ 11.20 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2916.93 │ 20.77 │ 140.44 │15111738│1197709 │22877637916│ 79049918 │ 0 │ 63861 │ 29963 │ 508 │113011 │29747.52│ 10.60 │
├───────┼───────┤ 40000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 5951.84 │ 20.76 │ 286.70 │30837735│1621963 │46682529726│ 107050718 │ 0 │129575 │ 53550 │ 1912 │219019 │16128.52│ 7.41 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2875.91 │ 20.45 │ 140.63 │14899829│1215386 │22556298466│ 80216570 │ 0 │ 63602 │ 30811 │ 517 │113055 │28819.79│ 10.75 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3493.96 │ 20.61 │ 169.53 │18102335│1299683 │27404210257│ 85780238 │ 0 │ 76414 │ 34729 │ 989 │133336 │18303.68│ 9.75 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2971.65 │ 20.18 │ 147.26 │15397768│ 569660 │23305187480│ 37597886 │ 0 │ 65136 │ 25160 │ 1025 │109390 │15022.21│ 5.21 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3039.73 │ 20.06 │ 151.53 │15749554│ 571308 │23838836620│ 37706594 │ 0 │ 67399 │ 27241 │ 1082 │113354 │14555.96│ 5.04 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3167.18 │ 20.24 │ 156.48 │16409368│ 575765 │24838314384│ 38000750 │ 0 │ 69998 │ 28670 │ 1083 │118643 │15151.77│ 4.85 │
├───────┼───────┤ 64000 │ 1 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3011.43 │ 20.18 │ 149.23 │15604231│ 569984 │23617080958│ 37619234 │ 0 │ 66146 │ 25188 │ 1053 │110588 │14818.83│ 5.15 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3109.37 │ 20.19 │ 154.01 │16111119│ 572426 │24385638358│ 37780382 │ 0 │ 69092 │ 27567 │ 1114 │116245 │14462.40│ 4.92 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3059.87 │ 20.17 │ 151.70 │15854408│ 571828 │23997011560│ 37740969 │ 0 │ 67554 │ 26765 │ 1071 │113644 │14803.37│ 5.03 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3002.80 │ 20.96 │ 143.26 │15557039│ 775172 │23551006550│ 51161914 │ 0 │ 66128 │ 26892 │ 807 │113574 │19277.62│ 6.83 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3054.32 │ 20.94 │ 145.86 │15824738│ 771426 │23955007524│ 50914678 │ 1 │ 67755 │ 26255 │ 751 │114455 │21071.56│ 6.74 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 5946.64 │ 19.96 │ 297.93 │30816873│1130615 │46642948666│ 74621242 │ 0 │128038 │ 51569 │ 2330 │216705 │13226.13│ 5.22 │
├───────┼───────┤ 64000 │ 2 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2932.97 │ 20.34 │ 144.20 │15194209│ 803587 │23002830338│ 53037304 │ 0 │ 65293 │ 26419 │ 686 │109847 │22148.99│ 7.32 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3052.39 │ 20.66 │ 147.74 │15813155│ 797801 │23938511030│ 52655464 │ 0 │ 68003 │ 27538 │ 802 │115045 │19717.15│ 6.93 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 3597.82 │ 20.57 │ 174.91 │18641202│ 855720 │28218060821│ 56478120 │ 0 │ 79043 │ 31734 │ 1075 │133925 │17340.65│ 6.39 │
├───────┼───────┼───────┼────────┼──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2838.78 │ 21.19 │ 133.97 │14708072│1132027 │22266304904│ 74714924 │ 0 │ 61319 │ 28323 │ 548 │110491 │26839.55│ 10.25 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2802.43 │ 20.80 │ 134.73 │14518769│1154549 │21980236458│ 76201340 │ 0 │ 61500 │ 28722 │ 540 │109217 │26886.61│ 10.57 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3050.89 │ 21.62 │ 141.11 │15807079│1131706 │23930034342│ 74693732 │ 0 │ 64955 │ 29623 │ 532 │117696 │29712.55│ 9.62 │
├───────┼───────┤ 64000 │ 4 ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 2929.79 │ 21.40 │ 136.91 │15179680│1166291 │22978798720│ 76976300 │ 0 │ 62978 │ 29165 │ 608 │114391 │24966.58│ 10.20 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ │ │ │ │ 3002.34 │ 21.01 │ 142.90 │15554591│1189606 │23548582838│ 78515114 │ 0 │ 65970 │ 31366 │ 511 │116996 │30439.51│ 10.17 │
├───────┼───────┤ │ ├──────────┼───────┼────────┼────────┼────────┼───────────┼───────────┼────────┼───────┼───────┼───────┼───────┼────────┼───────┤
│ - │ Avg │ │ │ 2924.85 │ 21.20 │ 137.96 │15153638│1154835 │22940791452│ 76220282 │ 0 │ 63344 │ 29439 │ 547 │113758 │27703.18│ 10.15 │
└───────┴───────┴───────┴────────┴──────────┴───────┴────────┴────────┴────────┴───────────┴───────────┴────────┴───────┴───────┴───────┴───────┴────────┴───────┘
Jason Wang
2013-12-26 07:33:01 UTC
Permalink
Post by Michael Dalton
Commit 2613af0ed18a ("virtio_net: migrate mergeable rx buffers to page frag
allocators") changed the mergeable receive buffer size from PAGE_SIZE to
MTU-size, introducing a single-stream regression for benchmarks with large
average packet size. There is no single optimal buffer size for all
workloads. For workloads with packet size <= MTU bytes, MTU + virtio-net
header-sized buffers are preferred as larger buffers reduce the TCP window
due to SKB truesize. However, single-stream workloads with large average
packet sizes have higher throughput if larger (e.g., PAGE_SIZE) buffers
are used.
This commit auto-tunes the mergeable receiver buffer packet size by
choosing the packet buffer size based on an EWMA of the recent packet
sizes for the receive queue. Packet buffer sizes range from MTU_SIZE +
virtio-net header len to PAGE_SIZE. This improves throughput for
large packet workloads, as any workload with average packet size >=
PAGE_SIZE will use PAGE_SIZE buffers.
These optimizations interact positively with recent commit
ba275241030c ("virtio-net: coalesce rx frags when possible during rx"),
which coalesces adjacent RX SKB fragments in virtio_net. The coalescing
optimizations benefit buffers of any size.
Benchmarks taken from an average of 5 netperf 30-second TCP_STREAM runs
between two QEMU VMs on a single physical machine. Each VM has two VCPUs
with all offloads & vhost enabled. All VMs and vhost threads run in a
single 4 CPU cgroup cpuset, using cgroups to ensure that other processes
in the system will not be scheduled on the benchmark CPUs. Trunk includes
SKB rx frag coalescing.
net-next w/ virtio_net before 2613af0ed18a (PAGE_SIZE bufs): 14642.85Gb/s
net-next (MTU-size bufs): 13170.01Gb/s
net-next + auto-tune: 14555.94Gb/s
The patch looks good to me and test this patch with mlx4, it help to
increase the rx performance from about 22Gb/s to about 26 Gb/s.
Michael Dalton
2013-12-26 20:06:23 UTC
Permalink
Post by Michael S. Tsirkin
OK so a high level benchmark shows it's worth it,
but how well does the logic work?
I think we should make the buffer size accessible in sysfs
or debugfs, and look at it, otherwise we don't really know.
Exporting the size sounds good to me, it is definitely an
important metric and would give more visibility to the admin.

Do you have a preference for implementation strategy? I was
thinking just add a DEVICE_ATTR to create a read-only sysfs file,
'mergeable_rx_buffer_size', and return a space-separated list of the
current buffer size (computed from the average packet size) for each
receive queue. -EINVAL or a similar error could be returned if the
netdev was not configured for mergeable rx buffers.
Post by Michael S. Tsirkin
I don't get the real motivation for this.
We have skbs A,B,C sharing a page, with chunk D being unused.
This randomly charges chunk D to an skb that ended up last
in the page.
Correct?
Why does this make sense?
The intent of this code is to adjust the SKB true size for
the packet. We should completely use each packet buffer except
for the last buffer. For all buffers except the last buffer, it
should be the case that 'len' (bytes received) = buffer size. For
the last buffer, this code adjusts the truesize by comparing the
approximated buffer size with the bytes received into the buffer,
and adding the difference to the SKB truesize if the buffer size
is greater than the number of bytes received.

We approximate the buffer size by using the last packet buffer size
from that same page, which as you have correctly noted may be a buffer
that belongs to a different packet on the same virtio-net device. This
buffer size should be very close to the actual buffer size because our
EWMA estimator uses a high weight (so the packet buffer size changes very
slowly) and there are only a handful packets on a page (even order-3).
Post by Michael S. Tsirkin
Why head_skb only? Why not full buffer size that comes from host?
This is simply len.
Sorry, I believe this code fragment should be clearer. Basically, we
have a corner case in that for packets with size <= GOOD_COPY_LEN, there
are no frags because page_to_skb() already unref'd the page and the entire
packet contents are copied to skb->data. In this case, the SKB truesize
is already accurate and should not be updated (and it would be unsafe to
access page->private as page is already unref'd).

I'll look at the above code again and cleanup (please let me know if you
have a preference) and/or add a comment to clarify.

Best,

Mike
Michael S. Tsirkin
2013-12-26 20:24:13 UTC
Permalink
Post by Michael Dalton
Post by Michael S. Tsirkin
OK so a high level benchmark shows it's worth it,
but how well does the logic work?
I think we should make the buffer size accessible in sysfs
or debugfs, and look at it, otherwise we don't really know.
Exporting the size sounds good to me, it is definitely an
important metric and would give more visibility to the admin.
Do you have a preference for implementation strategy? I was
thinking just add a DEVICE_ATTR to create a read-only sysfs file,
'mergeable_rx_buffer_size', and return a space-separated list of the
current buffer size (computed from the average packet size) for each
receive queue. -EINVAL or a similar error could be returned if the
netdev was not configured for mergeable rx buffers.
Not too important really. Maybe an attribute per queue? Might be
easier to parse and does not need tricky formatting.
Post by Michael Dalton
Post by Michael S. Tsirkin
I don't get the real motivation for this.
We have skbs A,B,C sharing a page, with chunk D being unused.
This randomly charges chunk D to an skb that ended up last
in the page.
Correct?
Why does this make sense?
The intent of this code is to adjust the SKB true size for
the packet. We should completely use each packet buffer except
for the last buffer. For all buffers except the last buffer, it
should be the case that 'len' (bytes received) = buffer size. For
the last buffer, this code adjusts the truesize by comparing the
approximated buffer size with the bytes received into the buffer,
and adding the difference to the SKB truesize if the buffer size
is greater than the number of bytes received.
We approximate the buffer size by using the last packet buffer size
from that same page, which as you have correctly noted may be a buffer
that belongs to a different packet on the same virtio-net device. This
buffer size should be very close to the actual buffer size because our
EWMA estimator uses a high weight (so the packet buffer size changes very
slowly) and there are only a handful packets on a page (even order-3).
I'll need to ponder it, don't understand it exactly.
Post by Michael Dalton
Post by Michael S. Tsirkin
Why head_skb only? Why not full buffer size that comes from host?
This is simply len.
Sorry, I believe this code fragment should be clearer. Basically, we
have a corner case in that for packets with size <= GOOD_COPY_LEN, there
are no frags because page_to_skb() already unref'd the page and the entire
packet contents are copied to skb->data. In this case, the SKB truesize
is already accurate and should not be updated (and it would be unsafe to
access page->private as page is already unref'd).
I'll look at the above code again and cleanup (please let me know if you
have a preference) and/or add a comment to clarify.
Best,
Mike
Jason Wang
2013-12-27 03:04:46 UTC
Permalink
Post by Michael Dalton
Post by Michael S. Tsirkin
OK so a high level benchmark shows it's worth it,
but how well does the logic work?
I think we should make the buffer size accessible in sysfs
or debugfs, and look at it, otherwise we don't really know.
Exporting the size sounds good to me, it is definitely an
important metric and would give more visibility to the admin.
Do you have a preference for implementation strategy? I was
thinking just add a DEVICE_ATTR to create a read-only sysfs file,
'mergeable_rx_buffer_size', and return a space-separated list of the
current buffer size (computed from the average packet size) for each
receive queue. -EINVAL or a similar error could be returned if the
netdev was not configured for mergeable rx buffers.
Post by Michael S. Tsirkin
I don't get the real motivation for this.
We have skbs A,B,C sharing a page, with chunk D being unused.
This randomly charges chunk D to an skb that ended up last
in the page.
Correct?
Why does this make sense?
The intent of this code is to adjust the SKB true size for
the packet. We should completely use each packet buffer except
for the last buffer. For all buffers except the last buffer, it
should be the case that 'len' (bytes received) = buffer size. For
the last buffer, this code adjusts the truesize by comparing the
approximated buffer size with the bytes received into the buffer,
and adding the difference to the SKB truesize if the buffer size
is greater than the number of bytes received.
But look like it does not take the 'hole' into account which may cause
under estimation of trusize.
Post by Michael Dalton
We approximate the buffer size by using the last packet buffer size
from that same page, which as you have correctly noted may be a buffer
that belongs to a different packet on the same virtio-net device. This
buffer size should be very close to the actual buffer size because our
EWMA estimator uses a high weight (so the packet buffer size changes very
slowly) and there are only a handful packets on a page (even order-3).
We can make this more accurate by using extra data structure to track
the real buf size and using it as token.
Post by Michael Dalton
Post by Michael S. Tsirkin
Why head_skb only? Why not full buffer size that comes from host?
This is simply len.
Sorry, I believe this code fragment should be clearer. Basically, we
have a corner case in that for packets with size <= GOOD_COPY_LEN, there
are no frags because page_to_skb() already unref'd the page and the entire
packet contents are copied to skb->data. In this case, the SKB truesize
is already accurate and should not be updated (and it would be unsafe to
access page->private as page is already unref'd).
I'll look at the above code again and cleanup (please let me know if you
have a preference) and/or add a comment to clarify.
Best,
Mike
Michael Dalton
2013-12-27 21:41:28 UTC
Permalink
I'm working on a followup patchset to address current feedback. I think
it will be cleaner to do a debugfs implementation for per-receive queue
packet buffer size exporting, so I'm trying that out.
Post by Jason Wang
We can make this more accurate by using extra data structure to track
the real buf size and using it as token.
I agree -- we can do precise buffer total len tracking. Something like
struct mergeable_packet_buffer_ctx {
void *buf;
unsigned int total_len;
};

Each receive queue could have a pointer to an array of N buffer contexts,
where N is queue size (kzalloc'd in init_vqs or similar). That would
allow us to allocate all of our buffer context data at startup.

Would this be preferred to the current approach or is there another
approach you would prefer? All other things being equal, having precise
length tracking is advantageous, so I'm inclined to try this out and
see how it goes.

I think this is a big design point - for example, if we have an extra
buffer context structure, then per-receive queue frag allocators are not
required for auto-tuning and we can reduce the number of patches in
this patchset.

I'm happy to implement either way. Thanks!

Best,

Mike
Jason Wang
2013-12-30 04:50:51 UTC
Permalink
Post by Michael Dalton
I'm working on a followup patchset to address current feedback. I think
it will be cleaner to do a debugfs implementation for per-receive queue
packet buffer size exporting, so I'm trying that out.
Post by Jason Wang
We can make this more accurate by using extra data structure to track
the real buf size and using it as token.
I agree -- we can do precise buffer total len tracking. Something like
struct mergeable_packet_buffer_ctx {
void *buf;
unsigned int total_len;
};
Each receive queue could have a pointer to an array of N buffer contexts,
where N is queue size (kzalloc'd in init_vqs or similar). That would
allow us to allocate all of our buffer context data at startup.
Would this be preferred to the current approach or is there another
approach you would prefer? All other things being equal, having precise
length tracking is advantageous, so I'm inclined to try this out and
see how it goes.
I think this is better since it was accurate and easier to be
implemented and understand.

Thanks
Post by Michael Dalton
I think this is a big design point - for example, if we have an extra
buffer context structure, then per-receive queue frag allocators are not
required for auto-tuning and we can reduce the number of patches in
this patchset.
I'm happy to implement either way. Thanks!
Best,
Mike
Jason Wang
2013-12-30 05:38:19 UTC
Permalink
Post by Michael Dalton
I'm working on a followup patchset to address current feedback. I think
it will be cleaner to do a debugfs implementation for per-receive queue
packet buffer size exporting, so I'm trying that out.
Post by Jason Wang
We can make this more accurate by using extra data structure to track
the real buf size and using it as token.
I agree -- we can do precise buffer total len tracking. Something like
struct mergeable_packet_buffer_ctx {
void *buf;
unsigned int total_len;
};
Each receive queue could have a pointer to an array of N buffer contexts,
where N is queue size (kzalloc'd in init_vqs or similar). That would
allow us to allocate all of our buffer context data at startup.
Would this be preferred to the current approach or is there another
approach you would prefer? All other things being equal, having precise
length tracking is advantageous, so I'm inclined to try this out and
see how it goes.
I think this is a big design point - for example, if we have an extra
buffer context structure, then per-receive queue frag allocators are not
required for auto-tuning and we can reduce the number of patches in
this patchset.
Not required but better keep it. Consider we may have multiple
virtio-net cards, using per-receive queue frag may increase the
possibility of coalescing.
Post by Michael Dalton
I'm happy to implement either way. Thanks!
Best,
Mike
Michael S. Tsirkin
2014-01-08 17:37:53 UTC
Permalink
Post by Michael Dalton
I'm working on a followup patchset to address current feedback. I think
it will be cleaner to do a debugfs implementation for per-receive queue
packet buffer size exporting, so I'm trying that out.
Post by Jason Wang
We can make this more accurate by using extra data structure to track
the real buf size and using it as token.
I agree -- we can do precise buffer total len tracking. Something like
struct mergeable_packet_buffer_ctx {
void *buf;
unsigned int total_len;
Maybe make total_len long so size is a power of 2.
Post by Michael Dalton
};
Hmm this doubles VQ cache footprint.
In the past when I tried increasong cache footprint
this hurt performance measureable. It's just a suggestion though,
YMMV, if numbers are good we don't need to argue about this.
Post by Michael Dalton
Each receive queue could have a pointer to an array of N buffer contexts,
where N is queue size (kzalloc'd in init_vqs or similar). That would
allow us to allocate all of our buffer context data at startup.
Would this be preferred to the current approach or is there another
approach you would prefer? All other things being equal, having precise
length tracking is advantageous, so I'm inclined to try this out and
see how it goes.
I think this is a big design point - for example, if we have an extra
buffer context structure, then per-receive queue frag allocators are not
required for auto-tuning and we can reduce the number of patches in
this patchset.
I'd be careful with adding even more stuff in
mergeable_packet_buffer_ctx for above reason.
Post by Michael Dalton
I'm happy to implement either way. Thanks!
Best,
Mike
David Miller
2013-12-19 19:58:15 UTC
Permalink
Can I get some reviews of this series from virtio folks?

Thanks.
Michael S. Tsirkin
2013-12-23 13:35:26 UTC
Permalink
Post by David Miller
Can I get some reviews of this series from virtio folks?
Thanks.
I think it's a good idea, certainly the best we can do
short term.
Would like a couple of comments addressed before it's applied.

Longer term one approach to consider is to make device
look aheads in the VQ until it finds a large
enough buffer.
--
MST
Jason Wang
2013-12-23 07:52:25 UTC
Permalink
Post by Michael Dalton
skb_page_frag_refill currently permits only order-0 page allocs
unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
higher-order page allocations whether or not GFP_WAIT is used. If
memory cannot be allocated, the allocator will fall back to
successively smaller page allocs (down to order-0 page allocs).
This change brings skb_page_frag_refill in line with the existing
page allocation strategy employed by netdev_alloc_frag, which attempts
higher-order page allocations whether or not GFP_WAIT is set, falling
back to successively lower-order page allocations on failure. Part
of migration of virtio-net to per-receive queue page frag allocators.
---
net/core/sock.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/net/core/sock.c b/net/core/sock.c
index ab20ed9..7383d23 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
put_page(pfrag->page);
}
- /* We restrict high order allocations to users that can afford to wait */
- order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
-
+ order = SKB_FRAG_PAGE_ORDER;
do {
gfp_t gfp = prio;
The original code seems try to avoid the high order allocation for
atomic allocation. This patch changes this, and looks like it will
introduce some extra cost when the memory is highly fragmented.
Eric Dumazet
2013-12-23 17:24:57 UTC
Permalink
Post by Jason Wang
Post by Michael Dalton
skb_page_frag_refill currently permits only order-0 page allocs
unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
higher-order page allocations whether or not GFP_WAIT is used. If
memory cannot be allocated, the allocator will fall back to
successively smaller page allocs (down to order-0 page allocs).
This change brings skb_page_frag_refill in line with the existing
page allocation strategy employed by netdev_alloc_frag, which attempts
higher-order page allocations whether or not GFP_WAIT is set, falling
back to successively lower-order page allocations on failure. Part
of migration of virtio-net to per-receive queue page frag allocators.
---
net/core/sock.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/net/core/sock.c b/net/core/sock.c
index ab20ed9..7383d23 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
put_page(pfrag->page);
}
- /* We restrict high order allocations to users that can afford to wait */
- order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
-
+ order = SKB_FRAG_PAGE_ORDER;
do {
gfp_t gfp = prio;
The original code seems try to avoid the high order allocation for
atomic allocation. This patch changes this, and looks like it will
introduce some extra cost when the memory is highly fragmented.
No noticeable extra cost that I could measure.

We use the same strategy in RX path nowadays.

Acked-by: Eric Dumazet <***@google.com>
Michael S. Tsirkin
2013-12-23 12:53:21 UTC
Permalink
Post by Michael Dalton
skb_page_frag_refill currently permits only order-0 page allocs
unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
higher-order page allocations whether or not GFP_WAIT is used. If
memory cannot be allocated, the allocator will fall back to
successively smaller page allocs (down to order-0 page allocs).
This change brings skb_page_frag_refill in line with the existing
page allocation strategy employed by netdev_alloc_frag, which attempts
higher-order page allocations whether or not GFP_WAIT is set, falling
back to successively lower-order page allocations on failure. Part
of migration of virtio-net to per-receive queue page frag allocators.
I don't get how this is related to patch 3/3 exactly.
That one seems to clamp the allocations from ewma to at most
PAGE_SIZE, so how to we get higher-order allocations here?
Could you clarify please?
Post by Michael Dalton
---
net/core/sock.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/net/core/sock.c b/net/core/sock.c
index ab20ed9..7383d23 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1865,9 +1865,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
put_page(pfrag->page);
}
- /* We restrict high order allocations to users that can afford to wait */
- order = (prio & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
-
+ order = SKB_FRAG_PAGE_ORDER;
do {
gfp_t gfp = prio;
--
1.8.5.1
Eric Dumazet
2013-12-23 17:30:38 UTC
Permalink
Post by Michael S. Tsirkin
Post by Michael Dalton
skb_page_frag_refill currently permits only order-0 page allocs
unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
higher-order page allocations whether or not GFP_WAIT is used. If
memory cannot be allocated, the allocator will fall back to
successively smaller page allocs (down to order-0 page allocs).
This change brings skb_page_frag_refill in line with the existing
page allocation strategy employed by netdev_alloc_frag, which attempts
higher-order page allocations whether or not GFP_WAIT is set, falling
back to successively lower-order page allocations on failure. Part
of migration of virtio-net to per-receive queue page frag allocators.
I don't get how this is related to patch 3/3 exactly.
That one seems to clamp the allocations from ewma to at most
PAGE_SIZE, so how to we get higher-order allocations here?
Could you clarify please?
If your ewma stabilizes at 2050 bytes per frag, using order-0 page will
waste ~50% of memory.
Michael S. Tsirkin
2013-12-23 19:19:27 UTC
Permalink
Post by Eric Dumazet
Post by Michael S. Tsirkin
Post by Michael Dalton
skb_page_frag_refill currently permits only order-0 page allocs
unless GFP_WAIT is used. Change skb_page_frag_refill to attempt
higher-order page allocations whether or not GFP_WAIT is used. If
memory cannot be allocated, the allocator will fall back to
successively smaller page allocs (down to order-0 page allocs).
This change brings skb_page_frag_refill in line with the existing
page allocation strategy employed by netdev_alloc_frag, which attempts
higher-order page allocations whether or not GFP_WAIT is set, falling
back to successively lower-order page allocations on failure. Part
of migration of virtio-net to per-receive queue page frag allocators.
I don't get how this is related to patch 3/3 exactly.
That one seems to clamp the allocations from ewma to at most
PAGE_SIZE, so how to we get higher-order allocations here?
Could you clarify please?
If your ewma stabilizes at 2050 bytes per frag, using order-0 page will
waste ~50% of memory.
Aha, got it. FWIW
Acked-by: Michael S. Tsirkin <***@redhat.com>
--
MST
David Miller
2013-12-24 22:46:47 UTC
Permalink
There is still feedback and/or minor adjustments being asked for wrt.
this series. These changes have been sitting for more than a week
which is a bit rediculous.

Please resubmit these changes once everything is resolved to
everyone's satisfaction, thanks.
Debabrata Banerjee
2014-01-03 00:42:00 UTC
Permalink
Currently because of how mm behaves (3.10.y) the code even before the
patch is a problem. I believe what may fix it is if instead of just
removing the conditional on __GFP_WAIT, the initial order > 0
allocation should be made GFP_ATOMIC, then fallback to the original
gfp mask for the order-0 allocations.

On systems that have highly fragmented main memory with pressure,
skb_page_frag_refill() causes problems. mm enters significant
compaction cycles on all cpu's which in itself is bad (add
considerable spinlock contention in isolate_migratepages_range() for
several seconds in kernel at 100% cpu), however even without this
happening basically we have large memory reclaimation when only
allocations from order-3 were necessary. For example, I might see half
the existing page cache on a system (2GB out of 8GB) reclaimed in a
burst, which effectively means the application has to wait even longer
after this compact/reclaim cycle for those pages to be read back from
disk. This is a significant reduction in useful memory from before
skb_page_frag_refill() existed, as one of our systems could run in
steady state will little free memory and 100% fragmentation. Now I see
10-30x more memory free (read: not utilized). Order > 0 allocations
were happening rarely before, now it happens consistently from this
function.

My suggestion above would avoid mm going through
__alloc_pages_direct_compact() and triggering the bad events above. It
will take me several days to try this experiment.

-Debabrata
Post by David Miller
There is still feedback and/or minor adjustments being asked for wrt.
this series. These changes have been sitting for more than a week
which is a bit rediculous.
Please resubmit these changes once everything is resolved to
everyone's satisfaction, thanks.
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
More majordomo info at http://vger.kernel.org/majordomo-info.html
Eric Dumazet
2014-01-03 00:56:31 UTC
Permalink
Post by Debabrata Banerjee
Currently because of how mm behaves (3.10.y) the code even before the
patch is a problem. I believe what may fix it is if instead of just
removing the conditional on __GFP_WAIT, the initial order > 0
allocation should be made GFP_ATOMIC, then fallback to the original
gfp mask for the order-0 allocations.
On systems that have highly fragmented main memory with pressure,
skb_page_frag_refill() causes problems. mm enters significant
compaction cycles on all cpu's which in itself is bad (add
considerable spinlock contention in isolate_migratepages_range() for
several seconds in kernel at 100% cpu), however even without this
happening basically we have large memory reclaimation when only
allocations from order-3 were necessary. For example, I might see half
the existing page cache on a system (2GB out of 8GB) reclaimed in a
burst, which effectively means the application has to wait even longer
after this compact/reclaim cycle for those pages to be read back from
disk. This is a significant reduction in useful memory from before
skb_page_frag_refill() existed, as one of our systems could run in
steady state will little free memory and 100% fragmentation. Now I see
10-30x more memory free (read: not utilized). Order > 0 allocations
were happening rarely before, now it happens consistently from this
function.
My suggestion above would avoid mm going through
__alloc_pages_direct_compact() and triggering the bad events above. It
will take me several days to try this experiment.
My suggestion is to use a recent kernel, and/or eventually backport the
mm fixes if any.

order-3 allocations should not reclaim 2GB out of 8GB.

There is a reason PAGE_ALLOC_COSTLY_ORDER exists and is 3
Eric Dumazet
2014-01-03 01:26:36 UTC
Permalink
Post by Eric Dumazet
My suggestion is to use a recent kernel, and/or eventually backport the
mm fixes if any.
order-3 allocations should not reclaim 2GB out of 8GB.
There is a reason PAGE_ALLOC_COSTLY_ORDER exists and is 3
Hmm... it looks like I missed __GFP_NORETRY



diff --git a/net/core/sock.c b/net/core/sock.c
index 5393b4b719d7..5f42a4d70cb2 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
gfp_t gfp = prio;

if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
+ gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
pfrag->page = alloc_pages(gfp, order);
if (likely(pfrag->page)) {
pfrag->offset = 0;
Debabrata Banerjee
2014-01-03 01:59:13 UTC
Permalink
Post by Eric Dumazet
Post by Eric Dumazet
My suggestion is to use a recent kernel, and/or eventually backport the
mm fixes if any.
order-3 allocations should not reclaim 2GB out of 8GB.
There is a reason PAGE_ALLOC_COSTLY_ORDER exists and is 3
Sorry 2GB cache out of 8GB phys, ~1GB gets reclaimed. Regardless the
reclaimation of cache is minor compared to the compaction event that
precedes it, I haven't spotted something addressing that yet -
isolate_migratepages_range ()/compact_checklock_irqsave(). If even
more of memory was unmoveable, the compaction routines would be hit
even harder as reclaimation wouldn't do anything - mm would have to
get very very smart about unmoveable pages being freed and just fail
allocations/oom kill if nothing has changed without running through
compaction/reclaim fruitlessly.

I guess this is a bit of a tangent since what I'm saying proves the
patch from Michael doesn't make this behavior worse.
Post by Eric Dumazet
Hmm... it looks like I missed __GFP_NORETRY
diff --git a/net/core/sock.c b/net/core/sock.c
index 5393b4b719d7..5f42a4d70cb2 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
gfp_t gfp = prio;
if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
+ gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
pfrag->page = alloc_pages(gfp, order);
if (likely(pfrag->page)) {
pfrag->offset = 0;
Yes this seems like it will make the situation better, but one send()
may still cause a direct_compact and direct_reclaim() cycle to happen,
followed immediately by another direct_compact() if direct_reclaim()
didn't free an order-3. Now have all cpu's doing a send(), you can
still get heavy spinlock contention in the routines described above.
The major change I see here is that allocations > order-0 used to be
rare, now it's on every send().

I can try your patch to see how much things improve.

-Debabrata
Debabrata Banerjee
2014-01-03 22:47:20 UTC
Permalink
Post by Eric Dumazet
Hmm... it looks like I missed __GFP_NORETRY
diff --git a/net/core/sock.c b/net/core/sock.c
index 5393b4b719d7..5f42a4d70cb2 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
gfp_t gfp = prio;
if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
+ gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
pfrag->page = alloc_pages(gfp, order);
if (likely(pfrag->page)) {
pfrag->offset = 0;
There is another patch needed (looks like good stable fixes):
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 06e72d3..d42d48c 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -378,7 +378,7 @@ refill:
gfp_t gfp = gfp_mask;

if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
+ gfp |= __GFP_COMP | __GFP_NOWARN |
__GFP_NORETRY;
nc->frag.page = alloc_pages(gfp, order);
if (likely(nc->frag.page))
break;

This reduces the really pathological compact/reclaim behavior but
doesn't fix it. Actually it still really quite bad because the whole
thing loops until it gets to order-0 so it's effectively trying the
allocation 4 times anyway. I typically see non-zero order allocations
very rarely without these two pieces of code. I hotpatched a running
system to get results from this quickly. Even setting the max order to
order-1 I still see bad behavior. If anything this behavior should be
conditional until this is ironed out.

Performance data: http://pastebin.ubuntu.com/6687527/

-Debabrata
Eric Dumazet
2014-01-03 22:54:53 UTC
Permalink
Post by Debabrata Banerjee
Post by Eric Dumazet
Hmm... it looks like I missed __GFP_NORETRY
diff --git a/net/core/sock.c b/net/core/sock.c
index 5393b4b719d7..5f42a4d70cb2 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1872,7 +1872,7 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
gfp_t gfp = prio;
if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
+ gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
pfrag->page = alloc_pages(gfp, order);
if (likely(pfrag->page)) {
pfrag->offset = 0;
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 06e72d3..d42d48c 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
gfp_t gfp = gfp_mask;
if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
+ gfp |= __GFP_COMP | __GFP_NOWARN |
__GFP_NORETRY;
nc->frag.page = alloc_pages(gfp, order);
if (likely(nc->frag.page))
break;
This is in GFP_ATOMIC cases, I dont think it can ever start compaction.
Post by Debabrata Banerjee
This reduces the really pathological compact/reclaim behavior but
doesn't fix it. Actually it still really quite bad because the whole
thing loops until it gets to order-0 so it's effectively trying the
allocation 4 times anyway. I typically see non-zero order allocations
very rarely without these two pieces of code. I hotpatched a running
system to get results from this quickly. Even setting the max order to
order-1 I still see bad behavior. If anything this behavior should be
conditional until this is ironed out.
Performance data: http://pastebin.ubuntu.com/6687527/
It seems that you shoot the messenger : If memory is fragmented, then
one order-1 allocation is going to start compaction.

It can be a simple fork().

If your workload never fork(), then yes, you never needed compaction.

It doesn't really matter to say that which memory allocation triggered
compaction, which is a normal step in mm layer.

If you believe its badly done, you should ask to mm guys to fix/improve
it, not netdev...

We are not trying to optimize the kernel behavior for hosts in deep
memory pressure.

Using order-3 pages in TCP stack improves performance for 99% of the
hosts, there might be something wrong on your side ?
Debabrata Banerjee
2014-01-03 23:27:11 UTC
Permalink
Post by Eric Dumazet
This is in GFP_ATOMIC cases, I dont think it can ever start compaction.
I think that's right I probably finally got it back to normal behavior
with order-0 allocations.
Post by Eric Dumazet
It seems that you shoot the messenger : If memory is fragmented, then
one order-1 allocation is going to start compaction.
It can be a simple fork().
If your workload never fork(), then yes, you never needed compaction.
Sure but the rate of network packets in and out and subsequent
allocations would be more equivalent to a fork bomb than normal
forking. I understand mm should work more sanely in this scenario but
at the same time we see a bad regression with this code, I see we're
not alone.
Post by Eric Dumazet
We are not trying to optimize the kernel behavior for hosts in deep
memory pressure.
We're leaving about half for the kernel so I wouldn't call it "deep".
Any server application that is using page cache and mlocked memory
will run into similar issues.
Post by Eric Dumazet
It doesn't really matter to say that which memory allocation triggered
compaction, which is a normal step in mm layer.
If you believe its badly done, you should ask to mm guys to fix/improve
it, not netdev...
Using order-3 pages in TCP stack improves performance for 99% of the
hosts, there might be something wrong on your side ?
Having lots of memory mlocked is bad right now yes, but not
necessarily an uncommon scenario. We're handing mm an almost
intractable problem. I see compaction of mlocked pages has been
discussed a few times over there, but no patch has actually made it
in.

-Debabrata
Continue reading on narkive:
Loading...