linux-rdma.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Yishai Hadas <yishaih@nvidia.com>
To: <linux-rdma@vger.kernel.org>
Cc: <jgg@nvidia.com>, <yishaih@nvidia.com>, <maorg@nvidia.com>,
	<avihaih@nvidia.com>
Subject: [PATCH V1 rdma-core 5/8] verbs: Optimize ibv_query_gid and ibv_query_gid_type
Date: Wed, 23 Sep 2020 13:26:59 +0300	[thread overview]
Message-ID: <20200923102702.590008-6-yishaih@nvidia.com> (raw)
In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com>

From: Avihai Horon <avihaih@nvidia.com>

ibv_query_gid and ibv_query_gid_type are implemented as ioctl first and
fallback to sysfs. Currently, if the fallback path is taken, all of the
gid entry attributes are retrieved over sysfs.

For example, if ibv_query_gid is called and the fallback path is taken,
the gid type and the gid ndev ifindex will also be read over sysfs,
even though we only need the gid.

In order to eliminate these unnecessary sysfs reads, we add an attribute
mask to ibv_cmd_query_gid_entry that will allow us to mark the specific
gid entry attributes that we would like to query in fallback.

Signed-off-by: Avihai Horon <avihaih@nvidia.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
---
 libibverbs/cmd_device.c | 76 ++++++++++++++++++++++++++++++++-----------------
 libibverbs/driver.h     | 11 +++++++
 libibverbs/verbs.c      |  8 +++---
 3 files changed, 65 insertions(+), 30 deletions(-)

diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c
index f707273..4f85010 100644
--- a/libibverbs/cmd_device.c
+++ b/libibverbs/cmd_device.c
@@ -320,41 +320,49 @@ static int query_sysfs_gid_type(struct ibv_context *context, uint8_t port_num,
 
 static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num,
 				 uint32_t gid_index,
-				 struct ibv_gid_entry *entry)
+				 struct ibv_gid_entry *entry,
+				 uint32_t attr_mask)
 {
 	enum ibv_gid_type_sysfs gid_type;
 	struct ibv_port_attr port_attr = {};
-	int ret;
+	int ret = 0;
 
 	entry->gid_index = gid_index;
 	entry->port_num = port_num;
-	ret = query_sysfs_gid(context, port_num, gid_index, &entry->gid);
-	if (ret)
-		return EINVAL;
-
-	ret = query_sysfs_gid_type(context, port_num, gid_index, &gid_type);
-	if (ret)
-		return EINVAL;
 
-	if (gid_type == IBV_GID_TYPE_SYSFS_IB_ROCE_V1) {
-		ret = ibv_query_port(context, port_num, &port_attr);
+	if (attr_mask & VERBS_QUERY_GID_ATTR_GID) {
+		ret = query_sysfs_gid(context, port_num, gid_index, &entry->gid);
 		if (ret)
-			goto out;
+			return EINVAL;
+	}
 
-		if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) {
-			entry->gid_type = IBV_GID_TYPE_IB;
-		} else if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
-			entry->gid_type = IBV_GID_TYPE_ROCE_V1;
+	if (attr_mask & VERBS_QUERY_GID_ATTR_TYPE) {
+		ret = query_sysfs_gid_type(context, port_num, gid_index, &gid_type);
+		if (ret)
+			return EINVAL;
+
+		if (gid_type == IBV_GID_TYPE_SYSFS_IB_ROCE_V1) {
+			ret = ibv_query_port(context, port_num, &port_attr);
+			if (ret)
+				goto out;
+
+			if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) {
+				entry->gid_type = IBV_GID_TYPE_IB;
+			} else if (port_attr.link_layer ==
+				   IBV_LINK_LAYER_ETHERNET) {
+				entry->gid_type = IBV_GID_TYPE_ROCE_V1;
+			} else {
+				ret = EINVAL;
+				goto out;
+			}
 		} else {
-			ret = EINVAL;
-			goto out;
+			entry->gid_type = IBV_GID_TYPE_ROCE_V2;
 		}
-	} else {
-		entry->gid_type = IBV_GID_TYPE_ROCE_V2;
 	}
 
-	ret = query_sysfs_gid_ndev_ifindex(context, port_num, gid_index,
-					   &entry->ndev_ifindex);
+	if (attr_mask & VERBS_QUERY_GID_ATTR_NDEV_IFINDEX)
+		ret = query_sysfs_gid_ndev_ifindex(context, port_num, gid_index,
+						   &entry->ndev_ifindex);
 
 out:
 	return ret;
@@ -364,9 +372,10 @@ out:
  * verbs_context_ops while async_event is and doesn't use ioctl.
  */
 #define query_gid_kernel_cap async_event
-int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num,
+int __ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num,
 			    uint32_t gid_index, struct ibv_gid_entry *entry,
-			    uint32_t flags, size_t entry_size)
+			    uint32_t flags, size_t entry_size,
+			    uint32_t fallback_attr_mask)
 {
 	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_DEVICE,
 			       UVERBS_METHOD_QUERY_GID_ENTRY, 4);
@@ -386,12 +395,27 @@ int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num,
 			return EOPNOTSUPP;
 
 		ret = query_sysfs_gid_entry(context, port_num, gid_index,
-					    entry);
+					    entry, fallback_attr_mask);
 		if (ret)
 			return ret;
 
-		return is_zero_gid(&entry->gid) ? ENODATA : 0;
+		if (fallback_attr_mask & VERBS_QUERY_GID_ATTR_GID &&
+		    is_zero_gid(&entry->gid))
+			return ENODATA;
+
+		return 0;
 	default:
 		return ret;
 	}
 }
+
+int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num,
+		      uint32_t gid_index, struct ibv_gid_entry *entry,
+		      uint32_t flags, size_t entry_size)
+{
+	return __ibv_query_gid_ex(context, port_num, gid_index, entry,
+				  flags, entry_size,
+				  VERBS_QUERY_GID_ATTR_GID |
+				  VERBS_QUERY_GID_ATTR_TYPE |
+				  VERBS_QUERY_GID_ATTR_NDEV_IFINDEX);
+}
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
index 046c07d..479e16e 100644
--- a/libibverbs/driver.h
+++ b/libibverbs/driver.h
@@ -77,6 +77,12 @@ enum ibv_gid_type_sysfs {
 	IBV_GID_TYPE_SYSFS_ROCE_V2,
 };
 
+enum verbs_query_gid_attr_mask {
+	VERBS_QUERY_GID_ATTR_GID		= 1 << 0,
+	VERBS_QUERY_GID_ATTR_TYPE		= 1 << 1,
+	VERBS_QUERY_GID_ATTR_NDEV_IFINDEX	= 1 << 2,
+};
+
 enum ibv_mr_type {
 	IBV_MR_TYPE_MR,
 	IBV_MR_TYPE_NULL_MR,
@@ -633,6 +639,11 @@ int ibv_cmd_reg_dm_mr(struct ibv_pd *pd, struct verbs_dm *dm,
 		      unsigned int access, struct verbs_mr *vmr,
 		      struct ibv_command_buffer *link);
 
+int __ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num,
+			    uint32_t gid_index, struct ibv_gid_entry *entry,
+			    uint32_t flags, size_t entry_size,
+			    uint32_t fallback_attr_mask);
+
 /*
  * sysfs helper functions
  */
diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c
index 99a48d5..e50ad94 100644
--- a/libibverbs/verbs.c
+++ b/libibverbs/verbs.c
@@ -224,8 +224,8 @@ LATEST_SYMVER_FUNC(ibv_query_gid, 1_1, "IBVERBS_1.1",
 	struct ibv_gid_entry entry = {};
 	int ret;
 
-	ret = _ibv_query_gid_ex(context, port_num, index, &entry, 0,
-				sizeof(entry));
+	ret = __ibv_query_gid_ex(context, port_num, index, &entry, 0,
+				 sizeof(entry), VERBS_QUERY_GID_ATTR_GID);
 	/* Preserve API behavior for empty GID */
 	if (ret == ENODATA) {
 		memset(gid, 0, sizeof(*gid));
@@ -703,8 +703,8 @@ int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num,
 	struct ibv_gid_entry entry = {};
 	int ret;
 
-	ret = _ibv_query_gid_ex(context, port_num, index, &entry, 0,
-				sizeof(entry));
+	ret = __ibv_query_gid_ex(context, port_num, index, &entry, 0,
+				 sizeof(entry), VERBS_QUERY_GID_ATTR_TYPE);
 	/* Preserve API behavior for empty GID */
 	if (ret == ENODATA) {
 		*type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1;
-- 
1.8.3.1


  parent reply	other threads:[~2020-09-23 10:27 UTC|newest]

Thread overview: 10+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-09-23 10:26 [PATCH V1 rdma-core 0/8] verbs: Query GID table API Yishai Hadas
2020-09-23 10:26 ` [PATCH V1 rdma-core 1/8] Update kernel headers Yishai Hadas
2020-09-23 10:26 ` [PATCH V1 rdma-core 2/8] verbs: Change the name of enum ibv_gid_type Yishai Hadas
2020-09-23 10:26 ` [PATCH V1 rdma-core 3/8] verbs: Introduce a new query GID entry API Yishai Hadas
2020-09-23 10:26 ` [PATCH V1 rdma-core 4/8] verbs: Implement ibv_query_gid and ibv_query_gid_type over ioctl Yishai Hadas
2020-09-23 10:26 ` Yishai Hadas [this message]
2020-09-23 10:27 ` [PATCH V1 rdma-core 6/8] verbs: Introduce a new query GID table API Yishai Hadas
2020-09-23 10:27 ` [PATCH V1 rdma-core 7/8] pyverbs: Add query_gid_table and query_gid_ex methods Yishai Hadas
2020-09-23 10:27 ` [PATCH V1 rdma-core 8/8] tests: Add tests for ibv_query_gid_table and ibv_query_gid_ex Yishai Hadas
2020-10-11  8:01 ` [PATCH V1 rdma-core 0/8] verbs: Query GID table API Yishai Hadas

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200923102702.590008-6-yishaih@nvidia.com \
    --to=yishaih@nvidia.com \
    --cc=avihaih@nvidia.com \
    --cc=jgg@nvidia.com \
    --cc=linux-rdma@vger.kernel.org \
    --cc=maorg@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).