[PATCH iproute2-next 0/3] Global per-type support for QP counters

[PATCH iproute2-next 0/3] Global per-type support for QP counters

[Leon Romanovsky]

From: Leon Romanovsky <leonro@mellanox.com>

Changelog:
 * Update first patch to latest rdma_netlink.h file.
 * Drop RFC, the kernel part was accepted.
https://lore.kernel.org/linux-rdma/20200726112011.75905-1-leon@kernel.org

---------------------------------------------------

Hi,

This is complimentary part of kernel series [1] that extends netlink
interface to allow automatic binding of QP counters based on their type,
in very similar manner to already existing per-PID ability.

Thanks

[1] https://lore.kernel.org/lkml/20200702082933.424537-1-leon@kernel.org

Mark Zhang (3):
  rdma: update uapi headers
  rdma: Add "PID" criteria support for statistic counter auto mode
  rdma: Document the new "pid" criteria for auto mode

 man/man8/rdma-statistic.8             | 23 +++++--
 rdma/include/uapi/rdma/rdma_netlink.h |  9 +--
 rdma/res-cmid.c                       |  6 --
 rdma/res.c                            |  6 ++
 rdma/res.h                            |  2 +-
 rdma/stat.c                           | 95 ++++++++++++++++++++-------
 6 files changed, 102 insertions(+), 39 deletions(-)

--
2.26.2

[PATCH iproute2-next 1/3] rdma: update uapi headers

[Leon Romanovsky]

From: Mark Zhang <markz@mellanox.com>

Update rdma_netlink.h file upto kernel commit 76251e15ea73
("RDMA/counter: Add PID category support in auto mode")

Signed-off-by: Mark Zhang <markz@mellanox.com>
Reviewed-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
 rdma/include/uapi/rdma/rdma_netlink.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
index fe127b88..4aef76ae 100644
--- a/rdma/include/uapi/rdma/rdma_netlink.h
+++ b/rdma/include/uapi/rdma/rdma_netlink.h
@@ -287,11 +287,11 @@ enum rdma_nldev_command {
 
 	RDMA_NLDEV_CMD_STAT_DEL,
 
-	RDMA_NLDEV_CMD_RES_QP_GET_RAW, /* can dump */
+	RDMA_NLDEV_CMD_RES_QP_GET_RAW,
 
-	RDMA_NLDEV_CMD_RES_CQ_GET_RAW, /* can dump */
+	RDMA_NLDEV_CMD_RES_CQ_GET_RAW,
 
-	RDMA_NLDEV_CMD_RES_MR_GET_RAW, /* can dump */
+	RDMA_NLDEV_CMD_RES_MR_GET_RAW,
 
 	RDMA_NLDEV_NUM_OPS
 };
@@ -531,7 +531,7 @@ enum rdma_nldev_attr {
 	 */
 	RDMA_NLDEV_ATTR_DEV_DIM,                /* u8 */
 
-	RDMA_NLDEV_ATTR_RES_RAW,                /* binary */
+	RDMA_NLDEV_ATTR_RES_RAW,	/* binary */
 
 	/*
 	 * Always the end
@@ -569,5 +569,6 @@ enum rdma_nl_counter_mode {
  */
 enum rdma_nl_counter_mask {
 	RDMA_COUNTER_MASK_QP_TYPE = 1,
+	RDMA_COUNTER_MASK_PID = 1 << 1,
 };
 #endif /* _RDMA_NETLINK_H */
-- 
2.26.2

[PATCH iproute2-next 2/3] rdma: Add "PID" criteria support for statistic counter auto mode

[Leon Romanovsky]

From: Mark Zhang <markz@mellanox.com>

With this new criteria, QPs have different PIDs will be bound to
different counters in auto mode. This can be used in combination with
other criteria like "type". Examples:

$ rdma statistic qp set link mlx5_2/1 auto pid on
$ rdma statistic qp set link mlx5_2/1 auto type,pid on
$ rdma statistic qp set link mlx5_2/1 auto off
$ rdma statistic qp show link mlx5_0 qp-type UD

Signed-off-by: Mark Zhang <markz@mellanox.com>
Reviewed-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
 rdma/res-cmid.c |  6 ----
 rdma/res.c      |  6 ++++
 rdma/res.h      |  2 +-
 rdma/stat.c     | 95 ++++++++++++++++++++++++++++++++++++-------------
 4 files changed, 78 insertions(+), 31 deletions(-)

diff --git a/rdma/res-cmid.c b/rdma/res-cmid.c
index f167800f..bfaa47b5 100644
--- a/rdma/res-cmid.c
+++ b/rdma/res-cmid.c
@@ -7,12 +7,6 @@
 #include "res.h"
 #include <inttypes.h>
 
-static void print_qp_type(struct rd *rd, uint32_t val)
-{
-	print_color_string(PRINT_ANY, COLOR_NONE, "qp-type", "qp-type %s ",
-			   qp_types_to_str(val));
-}
-
 static const char *cm_id_state_to_str(uint8_t idx)
 {
 	static const char *const cm_id_states_str[] = {
diff --git a/rdma/res.c b/rdma/res.c
index 251f5041..c99a1fcb 100644
--- a/rdma/res.c
+++ b/rdma/res.c
@@ -179,6 +179,12 @@ void print_link(struct rd *rd, uint32_t idx, const char *name, uint32_t port,
 		print_color_uint(PRINT_ANY, COLOR_NONE, "port", "%u ", port);
 }
 
+void print_qp_type(struct rd *rd, uint32_t val)
+{
+	print_color_string(PRINT_ANY, COLOR_NONE, "qp-type", "qp-type %s ",
+			   qp_types_to_str(val));
+}
+
 char *get_task_name(uint32_t pid)
 {
 	char *comm;
diff --git a/rdma/res.h b/rdma/res.h
index 70ce5758..707941da 100644
--- a/rdma/res.h
+++ b/rdma/res.h
@@ -165,5 +165,5 @@ void res_print_uint(struct rd *rd, const char *name, uint64_t val,
 		    struct nlattr *nlattr);
 void print_comm(struct rd *rd, const char *str, struct nlattr **nla_line);
 const char *qp_types_to_str(uint8_t idx);
-
+void print_qp_type(struct rd *rd, uint32_t val);
 #endif /* _RDMA_TOOL_RES_H_ */
diff --git a/rdma/stat.c b/rdma/stat.c
index 8d4b7a11..a2b5da1c 100644
--- a/rdma/stat.c
+++ b/rdma/stat.c
@@ -48,6 +48,7 @@ struct counter_param {
 
 static struct counter_param auto_params[] = {
 	{ "type", RDMA_COUNTER_MASK_QP_TYPE, },
+	{ "pid", RDMA_COUNTER_MASK_PID, },
 	{ NULL },
 };
 
@@ -56,6 +57,7 @@ static int prepare_auto_mode_str(struct nlattr **tb, uint32_t mask,
 {
 	char s[] = "qp auto";
 	int i, outlen = strlen(s);
+	bool first = true;
 
 	memset(output, 0, len);
 	snprintf(output, len, "%s", s);
@@ -66,7 +68,12 @@ static int prepare_auto_mode_str(struct nlattr **tb, uint32_t mask,
 				outlen += strlen(auto_params[i].name) + 1;
 				if (outlen >= len)
 					return -EINVAL;
-				strcat(output, " ");
+				if (first) {
+					strcat(output, " ");
+					first = false;
+				} else
+					strcat(output, ",");
+
 				strcat(output, auto_params[i].name);
 			}
 		}
@@ -202,7 +209,7 @@ int res_get_hwcounters(struct rd *rd, struct nlattr *hwc_table, bool print)
 static int res_counter_line(struct rd *rd, const char *name, int index,
 		       struct nlattr **nla_line)
 {
-	uint32_t cntn, port = 0, pid = 0, qpn;
+	uint32_t cntn, port = 0, pid = 0, qpn, qp_type = 0;
 	struct nlattr *hwc_table, *qp_table;
 	struct nlattr *nla_entry;
 	const char *comm = NULL;
@@ -223,6 +230,13 @@ static int res_counter_line(struct rd *rd, const char *name, int index,
 				nla_line[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]))
 		return MNL_CB_OK;
 
+	if (nla_line[RDMA_NLDEV_ATTR_RES_TYPE])
+		qp_type = mnl_attr_get_u8(nla_line[RDMA_NLDEV_ATTR_RES_TYPE]);
+
+	if (rd_is_string_filtered_attr(rd, "qp-type", qp_types_to_str(qp_type),
+				       nla_line[RDMA_NLDEV_ATTR_RES_TYPE]))
+		return MNL_CB_OK;
+
 	if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) {
 		pid = mnl_attr_get_u32(nla_line[RDMA_NLDEV_ATTR_RES_PID]);
 		comm = get_task_name(pid);
@@ -257,6 +271,8 @@ static int res_counter_line(struct rd *rd, const char *name, int index,
 	open_json_object(NULL);
 	print_link(rd, index, name, port, nla_line);
 	print_color_uint(PRINT_ANY, COLOR_NONE, "cntn", "cntn %u ", cntn);
+	if (nla_line[RDMA_NLDEV_ATTR_RES_TYPE])
+		print_qp_type(rd, qp_type);
 	res_print_uint(rd, "pid", pid, nla_line[RDMA_NLDEV_ATTR_RES_PID]);
 	print_comm(rd, comm, nla_line);
 	res_get_hwcounters(rd, hwc_table, true);
@@ -321,6 +337,7 @@ static const struct filters stat_valid_filters[MAX_NUMBER_OF_FILTERS] = {
 	{ .name = "cntn", .is_number = true },
 	{ .name = "lqpn", .is_number = true },
 	{ .name = "pid", .is_number = true },
+	{ .name = "qp-type", .is_number = false },
 };
 
 static int stat_qp_show_one_link(struct rd *rd)
@@ -382,37 +399,67 @@ static int stat_qp_set_link_auto_sendmsg(struct rd *rd, uint32_t mask)
 	return rd_sendrecv_msg(rd, seq);
 }
 
-static int stat_one_qp_set_link_auto_off(struct rd *rd)
+static int stat_get_auto_mode_mask(struct rd *rd)
 {
-	return stat_qp_set_link_auto_sendmsg(rd, 0);
-}
+	char *modes = rd_argv(rd), *mode, *saved_ptr;
+	const char *delim = ",";
+	int mask = 0, found, i;
 
-static int stat_one_qp_set_auto_type_on(struct rd *rd)
-{
-	return stat_qp_set_link_auto_sendmsg(rd, RDMA_COUNTER_MASK_QP_TYPE);
-}
+	if (!modes)
+		return mask;
 
-static int stat_one_qp_set_link_auto_type(struct rd *rd)
-{
-	const struct rd_cmd cmds[] = {
-		{ NULL,		stat_help },
-		{ "on",		stat_one_qp_set_auto_type_on },
-		{ 0 }
-	};
+	mode = strtok_r(modes, delim, &saved_ptr);
+	do {
+		if (!mode)
+			break;
 
-	return rd_exec_cmd(rd, cmds, "parameter");
+		found = false;
+		for (i = 0;  auto_params[i].name != NULL; i++) {
+			if (!strcmp(mode, auto_params[i].name)) {
+				mask |= auto_params[i].attr;
+				found = true;
+				break;
+			}
+		}
+
+		if (!found) {
+			pr_err("Unknown auto mode '%s'.\n", mode);
+			mask = 0;
+			break;
+		}
+
+		mode = strtok_r(NULL, delim, &saved_ptr);
+	} while(1);
+
+	if (mask)
+		rd_arg_inc(rd);
+
+	return mask;
 }
 
 static int stat_one_qp_set_link_auto(struct rd *rd)
 {
-	const struct rd_cmd cmds[] = {
-		{ NULL,		stat_one_qp_link_get_mode },
-		{ "off",	stat_one_qp_set_link_auto_off },
-		{ "type",	stat_one_qp_set_link_auto_type },
-		{ 0 }
-	};
+	int auto_mask = 0;
 
-	return rd_exec_cmd(rd, cmds, "parameter");
+	if (!rd_argc(rd))
+		return -EINVAL;
+
+	if (!strcmpx(rd_argv(rd), "off")) {
+		rd_arg_inc(rd);
+		return stat_qp_set_link_auto_sendmsg(rd, 0);
+	}
+
+	auto_mask = stat_get_auto_mode_mask(rd);
+	if (!auto_mask || !rd_argc(rd))
+		return -EINVAL;
+
+	if (!strcmpx(rd_argv(rd), "on")) {
+		rd_arg_inc(rd);
+		return stat_qp_set_link_auto_sendmsg(rd, auto_mask);
+	} else {
+		pr_err("Unknown parameter '%s'.\n", rd_argv(rd));
+		return -EINVAL;
+	}
 }
 
 static int stat_one_qp_set_link(struct rd *rd)
-- 
2.26.2

[PATCH iproute2-next 3/3] rdma: Document the new "pid" criteria for auto mode

[Leon Romanovsky]

From: Mark Zhang <markz@mellanox.com>

Document the new supported criteria of auto mode. Examples:
$ rdma statistic qp set link mlx5_2/1 auto pid on
$ rdma statistic qp set link mlx5_2/1 auto pid,type on

Signed-off-by: Mark Zhang <markz@mellanox.com>
Reviewed-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
 man/man8/rdma-statistic.8 | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/man/man8/rdma-statistic.8 b/man/man8/rdma-statistic.8
index 7de495c9..7160bcdf 100644
--- a/man/man8/rdma-statistic.8
+++ b/man/man8/rdma-statistic.8
@@ -68,11 +68,11 @@ rdma-statistic \- RDMA statistic counter configuration
 
 .ti -8
 .IR CRITERIA " := "
-.RB "{ " type " }"
+.RB "{ " type " | " pid " }"
 
 .ti -8
 .IR FILTER_NAME " := "
-.RB "{ " cntn " | " lqpn " | " pid " }"
+.RB "{ " cntn " | " lqpn " | " pid " | " qp-type " }"
 
 .SH "DESCRIPTION"
 .SS rdma statistic [object] show - Queries the specified RDMA device for RDMA and driver-specific statistics. Show the default hw counters if object is not specified
@@ -88,7 +88,7 @@ rdma-statistic \- RDMA statistic counter configuration
 - specifies a filter to show only the results matching it.
 
 .SS rdma statistic <object> set - configure counter statistic auto-mode for a specific device/port
-In auto mode all objects belong to one category are bind automatically to a single counter set. Not applicable for MR's.
+In auto mode all objects belong to one category are bind automatically to a single counter set. The "off" is global for all auto modes together. Not applicable for MR's.
 
 .SS rdma statistic <object> bind - manually bind an object (e.g., a qp) with a counter
 When bound the statistics of this object are available in this counter. Not applicable for MR's.
@@ -127,6 +127,11 @@ rdma statistic qp show link mlx5_2 pid 30489
 Shows the state of all qp counters of specified RDMA port and belonging to pid 30489
 .RE
 .PP
+rdma statistic qp show link mlx5_2 qp-type UD
+.RS 4
+Shows the state of all qp counters of specified RDMA port and with QP type UD
+.RE
+.PP
 rdma statistic qp mode
 .RS 4
 List current counter mode on all devices
@@ -139,7 +144,17 @@ List current counter mode of device mlx5_2 port 1
 .PP
 rdma statistic qp set link mlx5_2/1 auto type on
 .RS 4
-On device mlx5_2 port 1, for each new QP bind it with a counter automatically. Per counter for QPs with same qp type in each process. Currently only "type" is supported.
+On device mlx5_2 port 1, for each new user QP bind it with a counter automatically. Per counter for QPs with same qp type.
+.RE
+.PP
+rdma statistic qp set link mlx5_2/1 auto pid on
+.RS 4
+On device mlx5_2 port 1, for each new user QP bind it with a counter automatically. Per counter for QPs with same pid.
+.RE
+.PP
+rdma statistic qp set link mlx5_2/1 auto pid,type on
+.RS 4
+On device mlx5_2 port 1, for each new user QP bind it with a counter automatically. Per counter for QPs with same pid and same type.
 .RE
 .PP
 rdma statistic qp set link mlx5_2/1 auto off
-- 
2.26.2

Re: [PATCH iproute2-next 0/3] Global per-type support for QP counters

[David Ahern]

On 8/4/20 2:49 AM, Leon Romanovsky wrote:
> From: Leon Romanovsky <leonro@mellanox.com>
> 
> Changelog:
>  * Update first patch to latest rdma_netlink.h file.
>  * Drop RFC, the kernel part was accepted.
> https://lore.kernel.org/linux-rdma/20200726112011.75905-1-leon@kernel.org
> 

applied to iproute2-next

[PATCH iproute2-next 1/3] rdma: update uapi headers

[Patrisious Haddad]

Update rdma_netlink.h file upto kernel commit 36ce80759f8c
("RDMA/core: Add support to set privileged qkey parameter")

Signed-off-by: Patrisious Haddad <phaddad@nvidia.com>
Reviewed-by: Michael Guralnik <michaelgur@nvidia.com>
---
 rdma/include/uapi/rdma/rdma_netlink.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
index 92c528a0..3a506efa 100644
--- a/rdma/include/uapi/rdma/rdma_netlink.h
+++ b/rdma/include/uapi/rdma/rdma_netlink.h
@@ -554,6 +554,12 @@ enum rdma_nldev_attr {
 	RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX,	/* u32 */
 	RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC, /* u8 */
 
+	/*
+	 * To enable or disable using privileged_qkey without being
+	 * a privileged user.
+	 */
+	RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE,	/* u8 */
+
 	/*
 	 * Always the end
 	 */
-- 
2.18.1

[PATCH iproute2-next 1/3] rdma: Update uapi headers

[Mark Zhang]

From: Neta Ostrovsky <netao@nvidia.com>

Update rdma_netlink.h file upto kernel commit

Signed-off-by: Neta Ostrovsky <netao@nvidia.com>
Signed-off-by: Mark Zhang <markzhang@nvidia.com>
---
 rdma/include/uapi/rdma/rdma_netlink.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
index 37f583ee..fd347bc7 100644
--- a/rdma/include/uapi/rdma/rdma_netlink.h
+++ b/rdma/include/uapi/rdma/rdma_netlink.h
@@ -549,6 +549,9 @@ enum rdma_nldev_attr {
 
 	RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK,	/* u8 */
 
+	RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX,	/* u32 */
+	RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC, /* u8 0 - disabled, 1 - enabled */
+
 	/*
 	 * Always the end
 	 */
-- 
2.26.2

[PATCH iproute2-next 1/3] rdma: Update uapi headers

[Leon Romanovsky]

From: Neta Ostrovsky <netao@nvidia.com>

Update rdma_netlink.h file upto kernel commit c6c11ad3ab9f
("RDMA/nldev: Add QP numbers to SRQ information")

Reviewed-by: Mark Zhang <markz@mellanox.com>
Signed-off-by: Neta Ostrovsky <netao@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 rdma/include/uapi/rdma/rdma_netlink.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
index 4aef76ae..e161c245 100644
--- a/rdma/include/uapi/rdma/rdma_netlink.h
+++ b/rdma/include/uapi/rdma/rdma_netlink.h
@@ -293,6 +293,10 @@ enum rdma_nldev_command {
 
 	RDMA_NLDEV_CMD_RES_MR_GET_RAW,
 
+	RDMA_NLDEV_CMD_RES_CTX_GET, /* can dump */
+
+	RDMA_NLDEV_CMD_RES_SRQ_GET, /* can dump */
+
 	RDMA_NLDEV_NUM_OPS
 };
 
@@ -533,6 +537,15 @@ enum rdma_nldev_attr {
 
 	RDMA_NLDEV_ATTR_RES_RAW,	/* binary */
 
+	RDMA_NLDEV_ATTR_RES_CTX,		/* nested table */
+	RDMA_NLDEV_ATTR_RES_CTX_ENTRY,		/* nested table */
+
+	RDMA_NLDEV_ATTR_RES_SRQ,		/* nested table */
+	RDMA_NLDEV_ATTR_RES_SRQ_ENTRY,		/* nested table */
+	RDMA_NLDEV_ATTR_RES_SRQN,		/* u32 */
+
+	RDMA_NLDEV_ATTR_MIN_RANGE,		/* u32 */
+	RDMA_NLDEV_ATTR_MAX_RANGE,		/* u32 */
 	/*
 	 * Always the end
 	 */
-- 
2.30.2