root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# git show HEAD | head
commit 2636311e5e2894bd7c7800939a3b9b68e7a93bcc
Author: Gioh Kim <gi-oh.kim@ionos.com>
Date:   Tue Apr 13 14:00:27 2021 +0200

    swap likely and unlikely

diff --git a/rtrs/rtrs-clt.c b/rtrs/rtrs-clt.c
index 1b4b3e6..6235827 100644
--- a/rtrs/rtrs-clt.c
+++ b/rtrs/rtrs-clt.c


141 root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# make clean && make
make[1]: Entering directory '/usr/src/linux-5.4.86-pserver'
  CLEAN   /tmp/ddd/gkim/ibnbd2/Module.symvers
make[1]: Leaving directory '/usr/src/linux-5.4.86-pserver'
make[1]: Entering directory '/usr/src/linux-5.4.86-pserver'
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-clt.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-clt-sysfs.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-common.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-client.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-srv.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-srv-dev.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-srv-sysfs.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-server.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-core.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-clt.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-clt-stats.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-clt-sysfs.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-client.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-srv.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-srv-stats.o
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-srv-sysfs.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-server.o
  AR      /tmp/ddd/gkim/ibnbd2/built-in.a 
  Building modules, stage 2.
  MODPOST 5 modules
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-client.mod.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-client.ko
  CC [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-server.mod.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rnbd/rnbd-server.ko
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-client.mod.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-client.ko
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-core.mod.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-core.ko
  CC [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-server.mod.o
  LD [M]  /tmp/ddd/gkim/ibnbd2/rtrs/rtrs-server.ko
make[1]: Leaving directory '/usr/src/linux-5.4.86-pserver'
root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# rmmod rnbd-client
root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# rmmod rtrs-client
root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# rmmod rtrs-core
root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# insmod rtrs/rtrs-core.ko
root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# insmod rtrs/rtrs-client.ko
root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# insmod rnbd/rnbd-client.ko


fio start   : Di 13. Apr 12:10:30 UTC 2021
kernel info : Linux ps401a-914 5.4.86-pserver #5.4.86-3~deb10 SMP Fri Mar 5 12:29:36 UTC 2021 x86_64 GNU/Linux
fio version : fio-3.12
gcc: gcc (Debian 8.3.0-6) 8.3.0
Copyright (C) 2018 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
Start fio test
fiotest: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.12
Starting 64 processes
Jobs: 64 (f=1814): [r(4),f(2),r(5),f(1),r(11),f(1),r(2),f(1),r(4),f(2),r(5),f(1),r(4),f(1),r(12),f(1),r(5),f(1),r(1)][100.0%][r=3244MiB/s][r=830k IOPS][eta 00m:00s]
fiotest: (groupid=0, jobs=64): err= 0: pid=37528: Tue Apr 13 12:13:32 2021
  read: IOPS=829k, BW=3238MiB/s (3395MB/s)(569GiB/180025msec)
    slat (usec): min=165, max=195365, avg=1271.82, stdev=1671.22
    clat (nsec): min=1080, max=32693k, avg=8544062.13, stdev=2411682.41
     lat (usec): min=407, max=206880, avg=9815.94, stdev=2694.31
    clat percentiles (usec):
     |  1.00th=[ 3949],  5.00th=[ 5211], 10.00th=[ 5800], 20.00th=[ 6587],
     | 30.00th=[ 7177], 40.00th=[ 7701], 50.00th=[ 8225], 60.00th=[ 8848],
     | 70.00th=[ 9503], 80.00th=[10421], 90.00th=[11731], 95.00th=[12911],
     | 99.00th=[15270], 99.50th=[16319], 99.90th=[18482], 99.95th=[19530],
     | 99.99th=[22152]
   bw (  KiB/s): min=29696, max=254980, per=1.56%, avg=51775.19, stdev=3418.36, samples=22980
   iops        : min= 7424, max=63745, avg=12943.76, stdev=854.59, samples=22980
  lat (usec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.04%
  lat (usec)   : 100=0.04%, 250=0.02%, 500=0.01%, 750=0.01%, 1000=0.01%
  lat (msec)   : 2=0.05%, 4=0.90%, 10=74.58%, 20=24.31%, 50=0.04%
  cpu          : usr=1.00%, sys=4.97%, ctx=82399686, majf=0, minf=3717209
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=100.0%
     complete  : 0=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=100.0%
     issued rwts: total=149229295,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
   READ: bw=3238MiB/s (3395MB/s), 3238MiB/s-3238MiB/s (3395MB/s-3395MB/s), io=569GiB (611GB), run=180025-180025msec

Disk stats (read/write):
  rnbd0: ios=4661956/0, merge=0/0, ticks=10212115/0, in_queue=323800, util=99.67%
  rnbd1: ios=4661956/0, merge=0/0, ticks=10489817/0, in_queue=340940, util=99.70%
  rnbd2: ios=4661959/0, merge=0/0, ticks=10510042/0, in_queue=342180, util=99.70%
  rnbd3: ios=4661958/0, merge=0/0, ticks=10702745/0, in_queue=353350, util=99.73%
  rnbd4: ios=4661964/0, merge=0/0, ticks=10793914/0, in_queue=359190, util=99.74%
  rnbd5: ios=4661968/0, merge=0/0, ticks=10913714/0, in_queue=369150, util=99.74%
  rnbd6: ios=4661960/0, merge=0/0, ticks=10958094/0, in_queue=370730, util=99.74%
  rnbd7: ios=4661968/0, merge=0/0, ticks=10976320/0, in_queue=370090, util=99.76%
  rnbd8: ios=4661964/0, merge=0/0, ticks=11014804/0, in_queue=375780, util=99.79%
  rnbd9: ios=4661964/0, merge=0/0, ticks=11031969/0, in_queue=376760, util=99.80%
  rnbd10: ios=4661966/0, merge=0/0, ticks=11047729/0, in_queue=375450, util=99.81%
  rnbd11: ios=4661975/0, merge=0/0, ticks=11053595/0, in_queue=378140, util=99.83%
  rnbd12: ios=4661972/0, merge=1/0, ticks=11087759/0, in_queue=376570, util=99.83%
  rnbd13: ios=4661975/0, merge=0/0, ticks=11066221/0, in_queue=381940, util=99.85%
  rnbd14: ios=4661967/0, merge=0/0, ticks=11092973/0, in_queue=381730, util=99.85%
  rnbd15: ios=4661981/0, merge=0/0, ticks=11056803/0, in_queue=382830, util=99.86%
  rnbd16: ios=4661985/0, merge=0/0, ticks=9447901/0, in_queue=280700, util=99.89%
  rnbd17: ios=4661978/0, merge=0/0, ticks=10506961/0, in_queue=348500, util=99.90%
  rnbd18: ios=4661983/0, merge=0/0, ticks=10702411/0, in_queue=364060, util=99.92%
  rnbd19: ios=4661977/0, merge=0/0, ticks=10777160/0, in_queue=374250, util=99.92%
  rnbd20: ios=4661982/0, merge=0/0, ticks=10780637/0, in_queue=371820, util=99.93%
  rnbd21: ios=4661980/0, merge=0/0, ticks=10841533/0, in_queue=376750, util=99.95%
  rnbd22: ios=4661984/0, merge=0/0, ticks=10869817/0, in_queue=378430, util=99.95%
  rnbd23: ios=4661985/0, merge=0/0, ticks=10966341/0, in_queue=387410, util=99.96%
  rnbd24: ios=4661987/0, merge=0/0, ticks=10957613/0, in_queue=390960, util=99.96%
  rnbd25: ios=4661988/0, merge=0/0, ticks=11015585/0, in_queue=390920, util=99.97%
  rnbd26: ios=4661980/0, merge=0/0, ticks=11074411/0, in_queue=398090, util=100.00%
  rnbd27: ios=4661985/0, merge=0/0, ticks=11122911/0, in_queue=404760, util=100.00%
  rnbd28: ios=4661993/0, merge=0/0, ticks=11095077/0, in_queue=402480, util=100.00%
  rnbd29: ios=4661991/0, merge=0/0, ticks=11170485/0, in_queue=408370, util=100.00%
  rnbd30: ios=4661992/0, merge=0/0, ticks=11213819/0, in_queue=409730, util=100.00%
  rnbd31: ios=4661989/0, merge=0/0, ticks=11263063/0, in_queue=420640, util=100.00%


root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# bash go_32dev_128proc.sh
fio start   : Di 13. Apr 12:42:42 UTC 2021
kernel info : Linux ps401a-914 5.4.86-pserver #5.4.86-3~deb10 SMP Fri Mar 5 12:29:36 UTC 2021 x86_64 GNU/Linux
fio version : fio-3.12
gcc: gcc (Debian 8.3.0-6) 8.3.0
Copyright (C) 2018 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
Start fio test
fiotest: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.12
Starting 128 processes
Jobs: 123 (f=2946): [r(3),f(1),r(8),f(1),r(4),f(2),r(1),f(2),r(1),f(1),r(1),f(1),r(5),f(1),r(1),f(1),r(3),_(2),r(2),f(2),r(4),f(2),r(5),f(2),r(6),f(3),r(1),f(2),r(1),_(1),r(7),f(1),r(2),f(1),r(5),f(1),r(5),f
(5),r(1),f(1),r(1),_(2),f(1),r(2),f(1),r(22)][16.8%][r=3190MiB/s][r=817k IOPS][eta 15m:00s]
fiotest: (groupid=0, jobs=128): err= 0: pid=39254: Tue Apr 13 12:45:45 2021
  read: IOPS=817k, BW=3191MiB/s (3346MB/s)(561GiB/180029msec)
    slat (usec): min=69, max=412616, avg=7725.43, stdev=5595.18
    clat (nsec): min=1054, max=39850k, avg=12233217.00, stdev=3542424.84
     lat (usec): min=179, max=421190, avg=19958.71, stdev=5791.92
    clat percentiles (usec):
     |  1.00th=[ 4555],  5.00th=[ 6849], 10.00th=[ 7963], 20.00th=[ 9372],
     | 30.00th=[10290], 40.00th=[11207], 50.00th=[11994], 60.00th=[12911],
     | 70.00th=[13829], 80.00th=[15008], 90.00th=[16712], 95.00th=[18220],
     | 99.00th=[21627], 99.50th=[22938], 99.90th=[26084], 99.95th=[27395],
     | 99.99th=[30540]
   bw (  KiB/s): min= 3072, max=146432, per=0.78%, avg=25495.37, stdev=2758.09, samples=45992
   iops        : min=  768, max=36608, avg=6373.81, stdev=689.53, samples=45992
  lat (usec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.02%
  lat (usec)   : 100=0.02%, 250=0.03%, 500=0.04%, 750=0.03%, 1000=0.03%
  lat (msec)   : 2=0.09%, 4=0.43%, 10=25.69%, 20=71.33%, 50=2.29%
  cpu          : usr=0.57%, sys=2.20%, ctx=75748318, majf=0, minf=6305149
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=0.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.1%, >=64=100.0%
     complete  : 0=0.0%, 4=0.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.1%, >=64=100.0%
     issued rwts: total=147045041,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
   READ: bw=3191MiB/s (3346MB/s), 3191MiB/s-3191MiB/s (3346MB/s-3346MB/s), io=561GiB (602GB), run=180029-180029msec


Disk stats (read/write):
  rnbd0: ios=4589511/0, merge=0/0, ticks=13423743/0, in_queue=1931790, util=99.58%
  rnbd1: ios=4589508/0, merge=0/0, ticks=14476127/0, in_queue=2228970, util=99.60%
  rnbd2: ios=4589517/0, merge=0/0, ticks=14624196/0, in_queue=2271880, util=99.61%
  rnbd3: ios=4589504/0, merge=0/0, ticks=14686013/0, in_queue=2286760, util=99.63%
  rnbd4: ios=4595158/0, merge=0/0, ticks=14772465/0, in_queue=2316590, util=99.62%
  rnbd5: ios=4595158/0, merge=0/0, ticks=14805361/0, in_queue=2314670, util=99.65%
  rnbd6: ios=4595158/0, merge=0/0, ticks=14817116/0, in_queue=2324300, util=99.65%
  rnbd7: ios=4595158/0, merge=0/0, ticks=14833164/0, in_queue=2320360, util=99.66%
  rnbd8: ios=4595158/0, merge=0/0, ticks=14900960/0, in_queue=2340200, util=99.68%
  rnbd9: ios=4595158/0, merge=0/0, ticks=14917077/0, in_queue=2345260, util=99.70%
  rnbd10: ios=4595158/0, merge=0/0, ticks=14931826/0, in_queue=2344540, util=99.71%
  rnbd11: ios=4595158/0, merge=0/0, ticks=14963132/0, in_queue=2345350, util=99.72%
  rnbd12: ios=4595158/0, merge=0/0, ticks=14978944/0, in_queue=2371930, util=99.73%
  rnbd13: ios=4595158/0, merge=0/0, ticks=14953823/0, in_queue=2349200, util=99.75%
  rnbd14: ios=4595157/0, merge=0/0, ticks=14991909/0, in_queue=2361030, util=99.75%
  rnbd15: ios=4595157/0, merge=0/0, ticks=15039741/0, in_queue=2379400, util=99.76%
  rnbd16: ios=4595157/0, merge=0/0, ticks=15057599/0, in_queue=2387550, util=99.79%
  rnbd17: ios=4595157/0, merge=0/0, ticks=15052981/0, in_queue=2378570, util=99.80%
  rnbd18: ios=4595157/0, merge=0/0, ticks=15364367/0, in_queue=2455030, util=99.83%
  rnbd19: ios=4595157/0, merge=0/0, ticks=15369998/0, in_queue=2462130, util=99.84%
  rnbd20: ios=4595157/0, merge=0/0, ticks=14953262/0, in_queue=2354080, util=99.84%
  rnbd21: ios=4595157/0, merge=0/0, ticks=15116061/0, in_queue=2404290, util=99.86%
  rnbd22: ios=4595157/0, merge=0/0, ticks=15190489/0, in_queue=2419870, util=99.86%
  rnbd23: ios=4595157/0, merge=0/0, ticks=15212165/0, in_queue=2414980, util=99.88%
  rnbd24: ios=4595157/0, merge=0/0, ticks=15225716/0, in_queue=2429100, util=99.88%
  rnbd25: ios=4595157/0, merge=0/0, ticks=15239578/0, in_queue=2428240, util=99.89%
  rnbd26: ios=4595157/0, merge=0/0, ticks=15251628/0, in_queue=2427670, util=99.92%
  rnbd27: ios=4595157/0, merge=0/0, ticks=13955168/0, in_queue=2127300, util=99.93%
  rnbd28: ios=4595157/0, merge=0/0, ticks=14694941/0, in_queue=2329440, util=99.95%
  rnbd29: ios=4595157/0, merge=0/0, ticks=14804318/0, in_queue=2355090, util=99.96%
  rnbd30: ios=4595157/0, merge=0/0, ticks=15183672/0, in_queue=2421440, util=99.96%
  rnbd31: ios=4595157/0, merge=0/0, ticks=11575822/0, in_queue=1492750, util=99.98%
  
  
root@ps401a-914.nst:/tmp/ddd/gkim/ibnbd2# bash go_32dev_128proc.sh
fio start   : Di 13. Apr 12:51:40 UTC 2021
kernel info : Linux ps401a-914 5.4.86-pserver #5.4.86-3~deb10 SMP Fri Mar 5 12:29:36 UTC 2021 x86_64 GNU/Linux
fio version : fio-3.12
gcc: gcc (Debian 8.3.0-6) 8.3.0
Copyright (C) 2018 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
Start fio test
fiotest: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.12
Starting 128 processes
Jobs: 70 (f=470): [f(4),_(1),f(2),_(1),f(1),_(1),f(1),_(1),f(1),_(5),f(1),_(2),f(1),_(1),f(2),_(3),f(2),_(1),f(3),r(1),_(4),f(1),_(1),f(2),r(1),_(1),f(2),_(1),f(1),_(1),f(1),_(5),f(3),_(2),f(4),_(5),f(1),_(1
),f(1),_(2),f(3),_(2),f(1),_(1),f(1),_(1),f(2),_(1),r(1),_(1),f(5),_(1),f(1),_(3),f(5),_(1),f(1),_(2),f(1),_(1),f(1),_(2),f(9),_(1),f(2),_(1),f(1),_(1)][1.7%][r=3210MiB/s][r=822k IOPS][eta 02h:54m:00s]
fiotest: (groupid=0, jobs=128): err= 0: pid=40166: Tue Apr 13 12:54:43 2021
  read: IOPS=817k, BW=3193MiB/s (3348MB/s)(561GiB/180023msec)
    slat (usec): min=7, max=292298, avg=7787.34, stdev=5758.03
    clat (nsec): min=1586, max=50911k, avg=12167070.96, stdev=3539907.52
     lat (usec): min=206, max=300176, avg=19954.48, stdev=5924.10
    clat percentiles (usec):
     |  1.00th=[ 4490],  5.00th=[ 6783], 10.00th=[ 7898], 20.00th=[ 9241],
     | 30.00th=[10290], 40.00th=[11076], 50.00th=[11994], 60.00th=[12780],
     | 70.00th=[13829], 80.00th=[15008], 90.00th=[16712], 95.00th=[18220],
     | 99.00th=[21627], 99.50th=[22938], 99.90th=[26084], 99.95th=[27395],
     | 99.99th=[30540]
   bw (  KiB/s): min=10240, max=156672, per=0.78%, avg=25505.77, stdev=3163.58, samples=45970
   iops        : min= 2560, max=39168, avg=6376.42, stdev=790.91, samples=45970
  lat (usec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.02%
  lat (usec)   : 100=0.02%, 250=0.02%, 500=0.02%, 750=0.02%, 1000=0.02%
  lat (msec)   : 2=0.11%, 4=0.49%, 10=26.37%, 20=70.68%, 50=2.22%
  lat (msec)   : 100=0.01%
  cpu          : usr=0.55%, sys=2.16%, ctx=75466111, majf=0, minf=5747705
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=0.1%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=100.0%
     complete  : 0=0.0%, 4=0.1%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=100.0%
     issued rwts: total=147144833,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
   READ: bw=3193MiB/s (3348MB/s), 3193MiB/s-3193MiB/s (3348MB/s-3348MB/s), io=561GiB (603GB), run=180023-180023msec

Disk stats (read/write):
  rnbd0: ios=4598269/0, merge=0/0, ticks=14961482/0, in_queue=2294850, util=99.60%