* [PATCH blktests v2] rdma: Use rdma link instead of /sys/class/infiniband/*/parent
@ 2021-02-28 22:34 Bart Van Assche
2021-03-07 20:40 ` Omar Sandoval
0 siblings, 1 reply; 4+ messages in thread
From: Bart Van Assche @ 2021-02-28 22:34 UTC (permalink / raw)
To: Omar Sandoval; +Cc: linux-block, Yi Zhang, Bart Van Assche, Jason Gunthorpe
The approach of verifying whether or not an RDMA interface is associated
with the rdma_rxe interface by looking up its parent device is deprecated
and will be removed soon from the Linux kernel. Hence this patch that uses
the rdma link command instead.
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Yi Zhang <yi.zhang@redhat.com>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
v2: Added a _have_program check for 'rdma' as requested by Omar.
---
common/multipath-over-rdma | 111 +++++++++++--------------------------
tests/nvmeof-mp/rc | 2 +-
tests/srp/rc | 12 +---
3 files changed, 35 insertions(+), 90 deletions(-)
diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma
index 9d9d2b27af83..9e68189b56fd 100644
--- a/common/multipath-over-rdma
+++ b/common/multipath-over-rdma
@@ -57,6 +57,9 @@ _multipathd_version_ge() {
}
get_ipv4_addr() {
+ if [ ! -e "/sys/class/net/$1" ]; then
+ echo "get_ipv4_addr(): $1 is not a network interface" 1>&2
+ fi
ip -4 -o addr show dev "$1" |
sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p'
}
@@ -76,41 +79,11 @@ is_number() {
[ "$1" -eq "0$1" ] 2>/dev/null
}
-# Check whether a device is an RDMA device. An example argument:
-# /sys/devices/pci0000:00/0000:00:03.0/0000:04:00.0
-is_rdma_device() {
- local d i inode1 inode2
-
- inode1=$(stat -c %i "$1")
- # echo "inode1 = $inode1"
- for i in /sys/class/infiniband/*; do
- d=/sys/class/infiniband/"$(readlink "$i")"
- d=$(dirname "$(dirname "$d")")
- inode2=$(stat -c %i "$d")
- # echo "inode2 = $inode2"
- if [ "$inode1" = "$inode2" ]; then
- return
- fi
- done
- false
-}
-
# Lists RDMA capable network interface names, e.g. ib0 ib1.
rdma_network_interfaces() {
- (
- cd /sys/class/net &&
- for i in *; do
- [ -e "$i" ] || continue
- # Skip IPoIB (ARPHRD_INFINIBAND) network
- # interfaces.
- [ "$(<"$i"/type)" = 32 ] && continue
- [ -L "$i/device" ] || continue
- d=$(readlink "$i/device" 2>/dev/null)
- if [ -n "$d" ] && is_rdma_device "$i/$d"; then
- echo "$i"
- fi
- done
- )
+ rdma link show |
+ sed -n 's/^.*[[:blank:]]netdev[[:blank:]]\+\([^[:blank:]]*\)[[:blank:]]*/\1/p' |
+ sort -u
}
# Check whether any stacked block device holds block device $1. If so, echo
@@ -411,47 +384,36 @@ all_primary_gids() {
done
}
-# Check whether or not an rdma_rxe instance has been associated with network
-# interface $1.
-has_rdma_rxe() {
- local f
-
- for f in /sys/class/infiniband/*/parent; do
- if [ -e "$f" ] && [ "$(<"$f")" = "$1" ]; then
- return 0
- fi
- done
-
- return 1
+# Check whether or not an rdma_rxe or siw instance has been associated with
+# network interface $1.
+has_soft_rdma() {
+ rdma link | grep -q " netdev $1[[:blank:]]*\$"
}
# Load the rdma_rxe or siw kernel module and associate it with all network
# interfaces.
start_soft_rdma() {
+ local type
+
{
if [ -n "$use_siw" ]; then
modprobe siw || return $?
- (
- cd /sys/class/net &&
- for i in *; do
- [ -e "$i" ] || continue
- [ -e "/sys/class/infiniband/${i}_siw" ] && continue
- rdma link add "${i}_siw" type siw netdev "$i" ||
- echo "Failed to bind the siw driver to $i"
- done
- )
+ type=siw
else
modprobe rdma_rxe || return $?
- (
- cd /sys/class/net &&
- for i in *; do
- if [ -e "$i" ] && ! has_rdma_rxe "$i"; then
- echo "$i" > /sys/module/rdma_rxe/parameters/add ||
- echo "Failed to bind the rdma_rxe driver to $i"
- fi
- done
- )
+ type=rxe
fi
+ (
+ cd /sys/class/net &&
+ for i in *; do
+ [ -e "$i" ] || continue
+ [ "$i" = "lo" ] && continue
+ [ "$(<"$i/addr_len")" = 6 ] || continue
+ has_soft_rdma "$i" && continue
+ rdma link add "${i}_$type" type $type netdev "$i" ||
+ echo "Failed to bind the $type driver to $i"
+ done
+ )
} >>"$FULL"
}
@@ -459,27 +421,16 @@ start_soft_rdma() {
# unload the rdma_rxe kernel module.
stop_soft_rdma() {
{
- (
- cd /sys/class/net &&
- for i in *; do
- if [ -e "$i" ] && has_rdma_rxe "$i"; then
- { echo "$i" > /sys/module/rdma_rxe/parameters/remove; } \
- 2>/dev/null
- fi
- done
- )
+ rdma link |
+ sed -n 's,^link[[:blank:]]*\([^/]*\)/.* netdev .*,\1,p' |
+ while read -r i; do
+ echo "$i ..."
+ rdma link del "${i}" || echo "Failed to remove ${i}"
+ done
if ! unload_module rdma_rxe 10; then
echo "Unloading rdma_rxe failed"
return 1
fi
- (
- cd /sys/class/net &&
- for i in *_siw; do
- [ -e "$i" ] || continue
- rdma link del "${i}" ||
- echo "Failed to unbind the siw driver from ${i%_siw}"
- done
- )
if ! unload_module siw 10; then
echo "Unloading siw failed"
return 1
diff --git a/tests/nvmeof-mp/rc b/tests/nvmeof-mp/rc
index ab7770f6aac0..0a12825e2c11 100755
--- a/tests/nvmeof-mp/rc
+++ b/tests/nvmeof-mp/rc
@@ -42,7 +42,7 @@ and multipathing has been enabled in the nvme_core kernel module"
)
_have_modules "${required_modules[@]}" || return
- for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof; do
+ for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof rdma; do
_have_program "$p" || return
done
diff --git a/tests/srp/rc b/tests/srp/rc
index 700cd71ea155..2986bfd5439d 100755
--- a/tests/srp/rc
+++ b/tests/srp/rc
@@ -59,7 +59,8 @@ group_requires() {
)
_have_modules "${required_modules[@]}" || return
- for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof sg_reset; do
+ for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof rdma \
+ sg_reset; do
_have_program "$p" || return
done
@@ -142,14 +143,7 @@ do_ib_cm_login() {
}
rdma_dev_to_net_dev() {
- local b d rdma_dev=$1
-
- b=/sys/class/infiniband/$rdma_dev/parent
- if [ -e "$b" ]; then
- echo "$(<"$b")"
- else
- echo "${rdma_dev%_siw}"
- fi
+ rdma link show "$1/1" | sed 's/.* netdev //;s/[[:blank:]]*$//'
}
# Tell the SRP initiator to log in to an SRP target using the RDMA/CM.
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH blktests v2] rdma: Use rdma link instead of /sys/class/infiniband/*/parent
2021-02-28 22:34 [PATCH blktests v2] rdma: Use rdma link instead of /sys/class/infiniband/*/parent Bart Van Assche
@ 2021-03-07 20:40 ` Omar Sandoval
0 siblings, 0 replies; 4+ messages in thread
From: Omar Sandoval @ 2021-03-07 20:40 UTC (permalink / raw)
To: Bart Van Assche; +Cc: Omar Sandoval, linux-block, Yi Zhang, Jason Gunthorpe
On Sun, Feb 28, 2021 at 02:34:03PM -0800, Bart Van Assche wrote:
> The approach of verifying whether or not an RDMA interface is associated
> with the rdma_rxe interface by looking up its parent device is deprecated
> and will be removed soon from the Linux kernel. Hence this patch that uses
> the rdma link command instead.
>
> Cc: Jason Gunthorpe <jgg@nvidia.com>
> Cc: Yi Zhang <yi.zhang@redhat.com>
> Signed-off-by: Bart Van Assche <bvanassche@acm.org>
> ---
> v2: Added a _have_program check for 'rdma' as requested by Omar.
> ---
> common/multipath-over-rdma | 111 +++++++++++--------------------------
> tests/nvmeof-mp/rc | 2 +-
> tests/srp/rc | 12 +---
> 3 files changed, 35 insertions(+), 90 deletions(-)
Thanks, applied.
^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCH blktests] tests/block/031: Add a test for sharing a tag set across hardware queues
@ 2021-03-28 23:12 Bart Van Assche
2021-03-28 23:12 ` [PATCH blktests v2] rdma: Use rdma link instead of /sys/class/infiniband/*/parent Bart Van Assche
0 siblings, 1 reply; 4+ messages in thread
From: Bart Van Assche @ 2021-03-28 23:12 UTC (permalink / raw)
To: Omar Sandoval
Cc: linux-block, Bart Van Assche, John Garry, Don Brace, Douglas Gilbert
Support for sharing a tag set across hardware queues has been added
recently to the Linux kernel. See also the BLK_MQ_F_TAG_HCTX_SHARED flag,
Linux kernel commit 32bc15afed04 ("blk-mq: Facilitate a shared sbitmap per
tagset"; v5.10) and commit 0905053bdb5b ("null_blk: Support shared tag
bitmap"; v5.10). Add a test that triggers the shared tag set code in the
block layer core.
Cc: John Garry <john.garry@huawei.com>
Cc: Don Brace<don.brace@microsemi.com>
Cc: Douglas Gilbert <dgilbert@interlog.com>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
tests/block/031 | 43 +++++++++++++++++++++++++++++++++++++++++++
tests/block/031.out | 1 +
2 files changed, 44 insertions(+)
create mode 100755 tests/block/031
create mode 100644 tests/block/031.out
diff --git a/tests/block/031 b/tests/block/031
new file mode 100755
index 000000000000..7940f9fa1114
--- /dev/null
+++ b/tests/block/031
@@ -0,0 +1,43 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright 2021 Google LLC
+#
+# Test support for sharing a tag set across hardware queues ("host tag set").
+
+. tests/block/rc
+. common/null_blk
+
+DESCRIPTION="test host tagset support"
+TIMED=1
+
+requires() {
+ _have_null_blk && _have_module_param null_blk shared_tag_bitmap
+}
+
+test() {
+ local fio_status bs=512
+
+ : "${TIMEOUT:=30}"
+ if ! _init_null_blk nr_devices=0 queue_mode=2 shared_tag_bitmap=1; then
+ echo "Loading null_blk failed"
+ return 1
+ fi
+ if ! _configure_null_blk nullb0 completion_nsec=0 blocksize=$bs size=1\
+ submit_queues="$(nproc)" memory_backed=1 power=1; then
+ echo "Configuring null_blk failed"
+ return 1
+ fi
+ fio --verify=md5 --rw=randwrite --bs=$bs --loops=$((10**6)) \
+ --iodepth=64 --group_reporting --sync=1 --direct=1 \
+ --ioengine=libaio --runtime="${TIMEOUT}" --thread \
+ --name=block-031 --filename=/dev/nullb0 \
+ --output="${RESULTS_DIR}/block/fio-output-031.txt" \
+ >>"$FULL"
+ fio_status=$?
+ rmdir /sys/kernel/config/nullb/nullb0
+ _exit_null_blk
+ case $fio_status in
+ 0) echo "Passed";;
+ *) echo "Failed (fio status = $fio_status)";;
+ esac
+}
diff --git a/tests/block/031.out b/tests/block/031.out
new file mode 100644
index 000000000000..863339fb8ced
--- /dev/null
+++ b/tests/block/031.out
@@ -0,0 +1 @@
+Passed
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH blktests v2] rdma: Use rdma link instead of /sys/class/infiniband/*/parent
2021-03-28 23:12 [PATCH blktests] tests/block/031: Add a test for sharing a tag set across hardware queues Bart Van Assche
@ 2021-03-28 23:12 ` Bart Van Assche
2021-03-28 23:41 ` Bart Van Assche
0 siblings, 1 reply; 4+ messages in thread
From: Bart Van Assche @ 2021-03-28 23:12 UTC (permalink / raw)
To: Omar Sandoval
Cc: linux-block, Bart Van Assche, John Garry, Don Brace,
Douglas Gilbert, Jason Gunthorpe, Yi Zhang
The approach of verifying whether or not an RDMA interface is associated
with the rdma_rxe interface by looking up its parent device is deprecated
and will be removed soon from the Linux kernel. Hence this patch that uses
the rdma link command instead.
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Yi Zhang <yi.zhang@redhat.com>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
v2: Added a _have_program check for 'rdma' as requested by Omar.
---
common/multipath-over-rdma | 111 +++++++++++--------------------------
tests/nvmeof-mp/rc | 2 +-
tests/srp/rc | 12 +---
3 files changed, 35 insertions(+), 90 deletions(-)
diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma
index 9d9d2b27af83..9e68189b56fd 100644
--- a/common/multipath-over-rdma
+++ b/common/multipath-over-rdma
@@ -57,6 +57,9 @@ _multipathd_version_ge() {
}
get_ipv4_addr() {
+ if [ ! -e "/sys/class/net/$1" ]; then
+ echo "get_ipv4_addr(): $1 is not a network interface" 1>&2
+ fi
ip -4 -o addr show dev "$1" |
sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p'
}
@@ -76,41 +79,11 @@ is_number() {
[ "$1" -eq "0$1" ] 2>/dev/null
}
-# Check whether a device is an RDMA device. An example argument:
-# /sys/devices/pci0000:00/0000:00:03.0/0000:04:00.0
-is_rdma_device() {
- local d i inode1 inode2
-
- inode1=$(stat -c %i "$1")
- # echo "inode1 = $inode1"
- for i in /sys/class/infiniband/*; do
- d=/sys/class/infiniband/"$(readlink "$i")"
- d=$(dirname "$(dirname "$d")")
- inode2=$(stat -c %i "$d")
- # echo "inode2 = $inode2"
- if [ "$inode1" = "$inode2" ]; then
- return
- fi
- done
- false
-}
-
# Lists RDMA capable network interface names, e.g. ib0 ib1.
rdma_network_interfaces() {
- (
- cd /sys/class/net &&
- for i in *; do
- [ -e "$i" ] || continue
- # Skip IPoIB (ARPHRD_INFINIBAND) network
- # interfaces.
- [ "$(<"$i"/type)" = 32 ] && continue
- [ -L "$i/device" ] || continue
- d=$(readlink "$i/device" 2>/dev/null)
- if [ -n "$d" ] && is_rdma_device "$i/$d"; then
- echo "$i"
- fi
- done
- )
+ rdma link show |
+ sed -n 's/^.*[[:blank:]]netdev[[:blank:]]\+\([^[:blank:]]*\)[[:blank:]]*/\1/p' |
+ sort -u
}
# Check whether any stacked block device holds block device $1. If so, echo
@@ -411,47 +384,36 @@ all_primary_gids() {
done
}
-# Check whether or not an rdma_rxe instance has been associated with network
-# interface $1.
-has_rdma_rxe() {
- local f
-
- for f in /sys/class/infiniband/*/parent; do
- if [ -e "$f" ] && [ "$(<"$f")" = "$1" ]; then
- return 0
- fi
- done
-
- return 1
+# Check whether or not an rdma_rxe or siw instance has been associated with
+# network interface $1.
+has_soft_rdma() {
+ rdma link | grep -q " netdev $1[[:blank:]]*\$"
}
# Load the rdma_rxe or siw kernel module and associate it with all network
# interfaces.
start_soft_rdma() {
+ local type
+
{
if [ -n "$use_siw" ]; then
modprobe siw || return $?
- (
- cd /sys/class/net &&
- for i in *; do
- [ -e "$i" ] || continue
- [ -e "/sys/class/infiniband/${i}_siw" ] && continue
- rdma link add "${i}_siw" type siw netdev "$i" ||
- echo "Failed to bind the siw driver to $i"
- done
- )
+ type=siw
else
modprobe rdma_rxe || return $?
- (
- cd /sys/class/net &&
- for i in *; do
- if [ -e "$i" ] && ! has_rdma_rxe "$i"; then
- echo "$i" > /sys/module/rdma_rxe/parameters/add ||
- echo "Failed to bind the rdma_rxe driver to $i"
- fi
- done
- )
+ type=rxe
fi
+ (
+ cd /sys/class/net &&
+ for i in *; do
+ [ -e "$i" ] || continue
+ [ "$i" = "lo" ] && continue
+ [ "$(<"$i/addr_len")" = 6 ] || continue
+ has_soft_rdma "$i" && continue
+ rdma link add "${i}_$type" type $type netdev "$i" ||
+ echo "Failed to bind the $type driver to $i"
+ done
+ )
} >>"$FULL"
}
@@ -459,27 +421,16 @@ start_soft_rdma() {
# unload the rdma_rxe kernel module.
stop_soft_rdma() {
{
- (
- cd /sys/class/net &&
- for i in *; do
- if [ -e "$i" ] && has_rdma_rxe "$i"; then
- { echo "$i" > /sys/module/rdma_rxe/parameters/remove; } \
- 2>/dev/null
- fi
- done
- )
+ rdma link |
+ sed -n 's,^link[[:blank:]]*\([^/]*\)/.* netdev .*,\1,p' |
+ while read -r i; do
+ echo "$i ..."
+ rdma link del "${i}" || echo "Failed to remove ${i}"
+ done
if ! unload_module rdma_rxe 10; then
echo "Unloading rdma_rxe failed"
return 1
fi
- (
- cd /sys/class/net &&
- for i in *_siw; do
- [ -e "$i" ] || continue
- rdma link del "${i}" ||
- echo "Failed to unbind the siw driver from ${i%_siw}"
- done
- )
if ! unload_module siw 10; then
echo "Unloading siw failed"
return 1
diff --git a/tests/nvmeof-mp/rc b/tests/nvmeof-mp/rc
index ab7770f6aac0..0a12825e2c11 100755
--- a/tests/nvmeof-mp/rc
+++ b/tests/nvmeof-mp/rc
@@ -42,7 +42,7 @@ and multipathing has been enabled in the nvme_core kernel module"
)
_have_modules "${required_modules[@]}" || return
- for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof; do
+ for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof rdma; do
_have_program "$p" || return
done
diff --git a/tests/srp/rc b/tests/srp/rc
index 700cd71ea155..2986bfd5439d 100755
--- a/tests/srp/rc
+++ b/tests/srp/rc
@@ -59,7 +59,8 @@ group_requires() {
)
_have_modules "${required_modules[@]}" || return
- for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof sg_reset; do
+ for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof rdma \
+ sg_reset; do
_have_program "$p" || return
done
@@ -142,14 +143,7 @@ do_ib_cm_login() {
}
rdma_dev_to_net_dev() {
- local b d rdma_dev=$1
-
- b=/sys/class/infiniband/$rdma_dev/parent
- if [ -e "$b" ]; then
- echo "$(<"$b")"
- else
- echo "${rdma_dev%_siw}"
- fi
+ rdma link show "$1/1" | sed 's/.* netdev //;s/[[:blank:]]*$//'
}
# Tell the SRP initiator to log in to an SRP target using the RDMA/CM.
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2021-03-28 23:43 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-02-28 22:34 [PATCH blktests v2] rdma: Use rdma link instead of /sys/class/infiniband/*/parent Bart Van Assche
2021-03-07 20:40 ` Omar Sandoval
2021-03-28 23:12 [PATCH blktests] tests/block/031: Add a test for sharing a tag set across hardware queues Bart Van Assche
2021-03-28 23:12 ` [PATCH blktests v2] rdma: Use rdma link instead of /sys/class/infiniband/*/parent Bart Van Assche
2021-03-28 23:41 ` Bart Van Assche
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).