vitiofs 9pfs: fio comparsion

• qemu + virtiofs(cache=auto, dax=0) a.ka. kata-qemu-virtiofs

• qemu + 9pfs a.k.a kata-qemu

Platform

Packet : c1.small.x86-01

PROC1 x Intel E3-1240 v3 RAM32GB
DISK2 x 120GB SSD
NIC2 x 1Gbps Bonded Port
Nproc: 8

fio workload:

fio --direct=1 --gtod_reduce=1 --name=test --filename=random_read_write.fio --bs=4k --iodepth=64 --size=4G --readwrite=randrw --rwmixread=75 --output=/output/fio.tx

Results:

kata-qemu(9pfs):

READ: bw=211MiB/s (222MB/s), 211MiB/s-211MiB/s (222MB/s-222MB/s), io=3070MiB (3219MB), run=14532-14532msec

WRITE: bw=70.6MiB/s (74.0MB/s), 70.6MiB/s-70.6MiB/s (74.0MB/s-74.0MB/s), io=1026MiB (1076MB), run=14532-14532msec

kata-qemu-virtiofs:

Run status group 0 (all jobs):
   READ: bw=159MiB/s (167MB/s), 159MiB/s-159MiB/s (167MB/s-167MB/s), io=3070MiB (3219MB), run=19321-19321msec
  WRITE: bw=53.1MiB/s (55.7MB/s), 53.1MiB/s-53.1MiB/s (55.7MB/s-55.7MB/s), io=1026MiB (1076MB), run=19321-19321msec

Some other useful information:

Qemu command:

• virtiofs

/opt/kata/bin/qemu-virtiofs-system-x86_64 
-name sandbox-6da5fc42e640c8e9bb4a0104a379c69b5a97d0074c8e88b27e1b58342f20aefb 
-uuid 6dfcd0cf-4d02-45a0-88d6-b5a3b3f88e52 
-machine pc,accel=kvm,kernel_irqchip,nvdimm 
-cpu host,pmu=off 
-qmp unix:/run/vc/vm/6da5fc42e640c8e9bb4a0104a379c69b5a97d0074c8e88b27e1b58342f20aefb/qmp.sock,server,nowait 
-m 2048M,slots=10,maxmem=33139M 
-device pci-bridge,bus=pci.0,id=pci-bridge-0,chassis_nr=1,shpc=on,addr=2,romfile= 
-device virtio-serial-pci,disable-modern=false,id=serial0,romfile= 
-device virtconsole,chardev=charconsole0,id=console0 
-chardev socket,id=charconsole0,path=/run/vc/vm/6da5fc42e640c8e9bb4a0104a379c69b5a97d0074c8e88b27e1b58342f20aefb/console.sock,server,nowait 
-device nvdimm,id=nv0,memdev=mem0 
-object memory-backend-file,id=mem0,mem-path=/opt/kata/share/kata-containers/kata-containers-image_clearlinux_1.12.0-alpha1_agent_8c9bbadcd4.img,size=268435456 
-device virtio-scsi-pci,id=scsi0,disable-modern=false,romfile= 
-object rng-random,id=rng0,filename=/dev/urandom 
-device virtio-rng-pci,rng=rng0,romfile= 
-device virtserialport,chardev=charch0,id=channel0,name=agent.channel.0 
-chardev socket,id=charch0,path=/run/vc/vm/6da5fc42e640c8e9bb4a0104a379c69b5a97d0074c8e88b27e1b58342f20aefb/kata.sock,server,nowait 
-chardev socket,id=char-31162971885078d3,path=/run/vc/vm/6da5fc42e640c8e9bb4a0104a379c69b5a97d0074c8e88b27e1b58342f20aefb/vhost-fs.sock 
-device vhost-user-fs-pci,chardev=char-31162971885078d3,tag=kataShared,romfile= 
-netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4 
-device 
driver=virtio-net-pci,netdev=network-0,mac=02:42:ac:11:00:02,disable-modern=false,mq=on,vectors=4,romfile= 
-rtc base=utc,driftfix=slew,clock=host -global kvm-pit.lost_tick_policy=discard 
-vga none -no-user-config -nodefaults 
-nographic 
--no-reboot 
-daemonize 
-object memory-backend-file,id=dimm1,size=2048M,mem-path=/dev/shm,share=on 
-numa node,memdev=dimm1 -kernel /opt/kata/share/kata-containers/vmlinux-5.4.60-88 
-append 'tsc=reliable no_timer_check rcupdate.rcu_expedited=1 i8042.direct=1 i8042.dumbkbd=1 i8042.nopnp=1 i8042.noaux=1 noreplace-smp reboot=k console=hvc0 console=hvc1 cryptomgr.notests net.ifnames=0 pci=lastbus=0 iommu=off root=/dev/pmem0p1 rootflags=dax,data=ordered,errors=remount-ro ro rootfstype=ext4 quiet systemd.show_status=false panic=1 nr_cpus=8 agent.use_vsock=false systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none'
-pidfile /run/vc/vm/6da5fc42e640c8e9bb4a0104a379c69b5a97d0074c8e88b27e1b58342f20aefb/pid 
-smp 1,cores=1,threads=1,sockets=8,maxcpus=8

virtiofsd command:

/opt/kata/bin/virtiofsd 
--fd=3 
-o source=/run/kata-containers/shared/sandboxes/6da5fc42e640c8e9bb4a0104a379c69b5a97d0074c8e88b27e1b58342f20aefb/shared 
-o cache=auto 
--syslog 
-o no_posix_lock 
-f

qemu-9pfs:

/opt/kata/bin/qemu-system-x86_64 
-name sandbox-0c7f830064ca21e4b48d759b808eab11d8daaae6513192b533443ab8ce383970 
-uuid 56fce320-3e1d-4763-ae94-d8c539ee3e5f 

-machine pc,accel=kvm,kernel_irqchip,nvdimm 
-cpu host,pmu=off 
-qmp unix:/run/vc/vm/0c7f830064ca21e4b48d759b808eab11d8daaae6513192b533443ab8ce383970/qmp.sock,server,nowait 
-m 2048M,slots=10,maxmem=33139M 
-device pci-bridge,bus=pci.0,id=pci-bridge-0,chassis_nr=1,shpc=on,addr=2,romfile= 
-device virtio-serial-pci,disable-modern=false,id=serial0,romfile= 
-device virtconsole,chardev=charconsole0,id=console0 
-chardev socket,id=charconsole0,path=/run/vc/vm/0c7f830064ca21e4b48d759b808eab11d8daaae6513192b533443ab8ce383970/console.sock,server,nowait 
-device nvdimm,id=nv0,memdev=mem0 
-object memory-backend-file,id=mem0,mem-path=/opt/kata/share/kata-containers/kata-containers-image_clearlinux_1.12.0-alpha1_agent_8c9bbadcd4.img,size=268435456 
-device virtio-scsi-pci,id=scsi0,disable-modern=false,romfile= 
-object rng-random,id=rng0,filename=/dev/urandom 
-device virtio-rng-pci,rng=rng0,romfile= 
-device virtserialport,chardev=charch0,id=channel0,name=agent.channel.0 
-chardev socket,id=charch0,path=/run/vc/vm/0c7f830064ca21e4b48d759b808eab11d8daaae6513192b533443ab8ce383970/kata.sock,server,nowait 
-device virtio-9p-pci,disable-modern=false,fsdev=extra-9p-kataShared,mount_tag=kataShared,romfile= 
-fsdev local,id=extra-9p-kataShared,path=/shared,security_model=none,multidevs=remap 
-netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4 
-device driver=virtio-net-pci,netdev=network-0,mac=02:42:ac:11:00:02,disable-modern=false,mq=on,vectors=4,romfile= 
-rtc base=utc,driftfix=slew,clock=host 
-global kvm-pit.lost_tick_policy=discard 
-vga none -no-user-config -nodefaults -nographic 
--no-reboot -daemonize -object memory-backend-ram,id=dimm1,size=2048M 
-numa node,memdev=dimm1 -kernel /opt/kata/share/kata-containers/vmlinux-5.4.60-88 
-append 'tsc=reliable no_timer_check rcupdate.rcu_expedited=1 i8042.direct=1 i8042.dumbkbd=1 i8042.nopnp=1 i8042.noaux=1 noreplace-smp reboot=k console=hvc0 console=hvc1 cryptomgr.notests net.ifnames=0 pci=lastbus=0 iommu=off root=/dev/pmem0p1 rootflags=dax,data=ordered,errors=remount-ro ro rootfstype=ext4 quiet systemd.show_status=false panic=1 nr_cpus=8 agent.use_vsock=false systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none' 
-pidfile /run/vc/vm/0c7f830064ca21e4b48d759b808eab11d8daaae6513192b533443ab8ce383970/pid 
-smp 1,cores=1,threads=1,sockets=8,maxcpus=8

-fsdev local,id=extra-9p-kataShared,path=/shared,security_model=none,multidevs=remap -netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4

-device virtio-9p-pci,disable-modern=false,fsdev=extra-9p-kataShared,mount_tag=kataShared,romfile= -fsdev local,id=extra-9p-kataShared,path=/shared,security_model=none,multidevs=remap

If you want to try by yourself

Please use the following repository: https://github.com/jcvenegas/mrunner/

If you dont find useful the repository:

Here is a summary about what it does:

#!/bin/bash
set -x
set -e

(
cd workloads/fio/dockerfile/
docker build -f Dockerfile -t large-files-4gb .
)

# virtiofsd
results_dir="${PWD}"
sudo crudini --set --existing /opt/kata/share/defaults/kata-containers/configuration-qemu-virtiofs.toml hypervisor.qemu virtio_fs_cache '"auto"'
sudo crudini --set --existing /opt/kata/share/defaults/kata-containers/configuration-qemu-virtiofs.toml hypervisor.qemu virtio_fs_cache_size 0
sudo crudini --set --existing /opt/kata/share/defaults/kata-containers/configuration-qemu-virtiofs.toml hypervisor.qemu virtio_fs_extra_args '[]'
sudo crudini --set --existing /opt/kata/share/defaults/kata-containers/configuration-qemu-virtiofs.toml hypervisor.qemu kernel '"/opt/kata/share/kata-containers/vmlinux.container"'
/opt/kata/bin/kata-qemu-virtiofs kata-env



docker run -dti --runtime kata-qemu-virtiofs -v ${results_dir}:/output --name large-files-4gb large-files-4gb
docker exec -i large-files-4gb sh -c 'fio --direct=1 --gtod_reduce=1 --name=test --filename=random_read_write.fio --bs=4k --iodepth=64 --size=4G --readwrite=randrw --rwmixread=75 --output=/output/fio-virtiofs.txt'
docker rm -f large-files-4gb

# 9pfs
docker run -dti --runtime kata-qemu -v ${results_dir}:/output --name large-files-4gb large-files-4gb
docker exec -i large-files-4gb sh -c 'fio --direct=1 --gtod_reduce=1 --name=test --filename=random_read_write.fio --bs=4k --iodepth=64 --size=4G --readwrite=randrw --rwmixread=75 --output=/output/fio-9pfs.txt'
docker rm -f large-files-4gb