[PATCH for-next v2 0/2] Retrieve HW GID context from ib_gid_attr

[PATCH for-next v2 0/2] Retrieve HW GID context from ib_gid_attr

[Selvin Xavier]

Provide an option for vendor drivers to get the HW GID context
from the ib_gid_attr during modify_qp and create_ah. Required
for drivers/HW that maintains HW gid index different than the
host sgid_index.

Please review.

Thanks,
Selvin Xavier

v1 -> v2:
 Addressed review comments from Parav

Selvin Xavier (2):
  IB/core: Add option to retrieve driver gid context from ib_gid_attr
  RDMA/bnxt_re: Retrieve the driver gid context from gid_attr

 drivers/infiniband/core/cache.c          | 79 ++++++++++++++++++--------------
 drivers/infiniband/hw/bnxt_re/ib_verbs.c | 28 +++++------
 include/rdma/ib_verbs.h                  |  5 ++
 3 files changed, 65 insertions(+), 47 deletions(-)

-- 
2.5.5

[PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr

[Selvin Xavier]

Provide an option to retrieve the driver gid context from ib_gid_attr
structure. Introduce ib_gid_attr_info structure which include both
gid_attr and the GID's HW context. Replace the attr and context
members of ib_gid_table_entry with the new ib_gid_attr_info
structure. Vendor drivers can refer to its own HW gid context
using the container_of macro.

Suggested-by: Jason Gunthorpe <jgg@mellanox.com>
Reviewed-by: Parav Pandit <parav@mellanox.com>
Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
 drivers/infiniband/core/cache.c | 79 +++++++++++++++++++++++------------------
 include/rdma/ib_verbs.h         |  5 +++
 2 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index d535995..54ed25d 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -86,8 +86,7 @@ struct roce_gid_ndev_storage {
 struct ib_gid_table_entry {
 	struct kref			kref;
 	struct work_struct		del_work;
-	struct ib_gid_attr		attr;
-	void				*context;
+	struct ib_gid_attr_info		attr_info;
 	/* Store the ndev pointer to release reference later on in
 	 * call_rcu context because by that time gid_table_entry
 	 * and attr might be already freed. So keep a copy of it.
@@ -233,12 +232,13 @@ static void put_gid_ndev(struct rcu_head *head)
 
 static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
 {
-	struct ib_device *device = entry->attr.device;
-	u8 port_num = entry->attr.port_num;
+	struct ib_device *device = entry->attr_info.attr.device;
+	u8 port_num = entry->attr_info.attr.port_num;
 	struct ib_gid_table *table = rdma_gid_table(device, port_num);
 
 	dev_dbg(&device->dev, "%s port=%d index=%d gid %pI6\n", __func__,
-		port_num, entry->attr.index, entry->attr.gid.raw);
+		port_num, entry->attr_info.attr.index,
+		entry->attr_info.attr.gid.raw);
 
 	write_lock_irq(&table->rwlock);
 
@@ -248,8 +248,8 @@ static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
 	 * If new entry in table is added by the time we free here,
 	 * don't overwrite the table entry.
 	 */
-	if (entry == table->data_vec[entry->attr.index])
-		table->data_vec[entry->attr.index] = NULL;
+	if (entry == table->data_vec[entry->attr_info.attr.index])
+		table->data_vec[entry->attr_info.attr.index] = NULL;
 	/* Now this index is ready to be allocated */
 	write_unlock_irq(&table->rwlock);
 
@@ -278,8 +278,8 @@ static void free_gid_work(struct work_struct *work)
 {
 	struct ib_gid_table_entry *entry =
 		container_of(work, struct ib_gid_table_entry, del_work);
-	struct ib_device *device = entry->attr.device;
-	u8 port_num = entry->attr.port_num;
+	struct ib_device *device = entry->attr_info.attr.device;
+	u8 port_num = entry->attr_info.attr.port_num;
 	struct ib_gid_table *table = rdma_gid_table(device, port_num);
 
 	mutex_lock(&table->lock);
@@ -309,7 +309,7 @@ alloc_gid_entry(const struct ib_gid_attr *attr)
 		entry->ndev_storage->ndev = ndev;
 	}
 	kref_init(&entry->kref);
-	memcpy(&entry->attr, attr, sizeof(*attr));
+	memcpy(&entry->attr_info.attr, attr, sizeof(*attr));
 	INIT_WORK(&entry->del_work, free_gid_work);
 	entry->state = GID_TABLE_ENTRY_INVALID;
 	return entry;
@@ -320,13 +320,15 @@ static void store_gid_entry(struct ib_gid_table *table,
 {
 	entry->state = GID_TABLE_ENTRY_VALID;
 
-	dev_dbg(&entry->attr.device->dev, "%s port=%d index=%d gid %pI6\n",
-		__func__, entry->attr.port_num, entry->attr.index,
-		entry->attr.gid.raw);
+	dev_dbg(&entry->attr_info.attr.device->dev,
+		"%s port=%d index=%d gid %pI6\n",
+		__func__, entry->attr_info.attr.port_num,
+		entry->attr_info.attr.index,
+		entry->attr_info.attr.gid.raw);
 
 	lockdep_assert_held(&table->lock);
 	write_lock_irq(&table->rwlock);
-	table->data_vec[entry->attr.index] = entry;
+	table->data_vec[entry->attr_info.attr.index] = entry;
 	write_unlock_irq(&table->rwlock);
 }
 
@@ -347,7 +349,7 @@ static void put_gid_entry_locked(struct ib_gid_table_entry *entry)
 
 static int add_roce_gid(struct ib_gid_table_entry *entry)
 {
-	const struct ib_gid_attr *attr = &entry->attr;
+	const struct ib_gid_attr *attr = &entry->attr_info.attr;
 	int ret;
 
 	if (!attr->ndev) {
@@ -356,7 +358,8 @@ static int add_roce_gid(struct ib_gid_table_entry *entry)
 		return -EINVAL;
 	}
 	if (rdma_cap_roce_gid_table(attr->device, attr->port_num)) {
-		ret = attr->device->ops.add_gid(attr, &entry->context);
+		ret = attr->device->ops.add_gid(attr,
+						&entry->attr_info.context);
 		if (ret) {
 			dev_err(&attr->device->dev,
 				"%s GID add failed port=%d index=%d\n",
@@ -385,7 +388,7 @@ static void del_gid(struct ib_device *ib_dev, u8 port,
 	lockdep_assert_held(&table->lock);
 
 	dev_dbg(&ib_dev->dev, "%s port=%d index=%d gid %pI6\n", __func__, port,
-		ix, table->data_vec[ix]->attr.gid.raw);
+		ix, table->data_vec[ix]->attr_info.attr.gid.raw);
 
 	write_lock_irq(&table->rwlock);
 	entry = table->data_vec[ix];
@@ -400,12 +403,13 @@ static void del_gid(struct ib_device *ib_dev, u8 port,
 	ndev_storage = entry->ndev_storage;
 	if (ndev_storage) {
 		entry->ndev_storage = NULL;
-		rcu_assign_pointer(entry->attr.ndev, NULL);
+		rcu_assign_pointer(entry->attr_info.attr.ndev, NULL);
 		call_rcu(&ndev_storage->rcu_head, put_gid_ndev);
 	}
 
 	if (rdma_cap_roce_gid_table(ib_dev, port))
-		ib_dev->ops.del_gid(&entry->attr, &entry->context);
+		ib_dev->ops.del_gid(&entry->attr_info.attr,
+				    &entry->attr_info.context);
 
 	put_gid_entry_locked(entry);
 }
@@ -508,13 +512,13 @@ static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,
 		if (found >= 0)
 			continue;
 
-		attr = &data->attr;
+		attr = &data->attr_info.attr;
 		if (mask & GID_ATTR_FIND_MASK_GID_TYPE &&
 		    attr->gid_type != val->gid_type)
 			continue;
 
 		if (mask & GID_ATTR_FIND_MASK_GID &&
-		    memcmp(gid, &data->attr.gid, sizeof(*gid)))
+		    memcmp(gid, &data->attr_info.attr.gid, sizeof(*gid)))
 			continue;
 
 		if (mask & GID_ATTR_FIND_MASK_NETDEV &&
@@ -648,7 +652,7 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
 
 	for (ix = 0; ix < table->sz; ix++) {
 		if (is_gid_entry_valid(table->data_vec[ix]) &&
-		    table->data_vec[ix]->attr.ndev == ndev) {
+		    table->data_vec[ix]->attr_info.attr.ndev == ndev) {
 			del_gid(ib_dev, port, table, ix);
 			deleted = true;
 		}
@@ -703,7 +707,7 @@ rdma_find_gid_by_port(struct ib_device *ib_dev,
 	local_index = find_gid(table, gid, &val, false, mask, NULL);
 	if (local_index >= 0) {
 		get_gid_entry(table->data_vec[local_index]);
-		attr = &table->data_vec[local_index]->attr;
+		attr = &table->data_vec[local_index]->attr_info.attr;
 		read_unlock_irqrestore(&table->rwlock, flags);
 		return attr;
 	}
@@ -753,12 +757,12 @@ const struct ib_gid_attr *rdma_find_gid_by_filter(
 		if (!is_gid_entry_valid(entry))
 			continue;
 
-		if (memcmp(gid, &entry->attr.gid, sizeof(*gid)))
+		if (memcmp(gid, &entry->attr_info.attr.gid, sizeof(*gid)))
 			continue;
 
-		if (filter(gid, &entry->attr, context)) {
+		if (filter(gid, &entry->attr_info.attr, context)) {
 			get_gid_entry(entry);
-			res = &entry->attr;
+			res = &entry->attr_info.attr;
 			break;
 		}
 	}
@@ -964,7 +968,7 @@ int rdma_query_gid(struct ib_device *device, u8 port_num,
 	    !is_gid_entry_valid(table->data_vec[index]))
 		goto done;
 
-	memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid));
+	memcpy(gid, &table->data_vec[index]->attr_info.attr.gid, sizeof(*gid));
 	res = 0;
 
 done:
@@ -1011,7 +1015,7 @@ const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,
 			const struct ib_gid_attr *attr;
 
 			get_gid_entry(table->data_vec[index]);
-			attr = &table->data_vec[index]->attr;
+			attr = &table->data_vec[index]->attr_info.attr;
 			read_unlock_irqrestore(&table->rwlock, flags);
 			return attr;
 		}
@@ -1210,7 +1214,7 @@ rdma_get_gid_attr(struct ib_device *device, u8 port_num, int index)
 		goto done;
 
 	get_gid_entry(table->data_vec[index]);
-	attr = &table->data_vec[index]->attr;
+	attr = &table->data_vec[index]->attr_info.attr;
 done:
 	read_unlock_irqrestore(&table->rwlock, flags);
 	return attr;
@@ -1230,8 +1234,10 @@ EXPORT_SYMBOL(rdma_get_gid_attr);
  */
 void rdma_put_gid_attr(const struct ib_gid_attr *attr)
 {
+	struct ib_gid_attr_info *info =
+		container_of(attr, struct ib_gid_attr_info, attr);
 	struct ib_gid_table_entry *entry =
-		container_of(attr, struct ib_gid_table_entry, attr);
+		container_of(info, struct ib_gid_table_entry, attr_info);
 
 	put_gid_entry(entry);
 }
@@ -1249,8 +1255,10 @@ EXPORT_SYMBOL(rdma_put_gid_attr);
  */
 void rdma_hold_gid_attr(const struct ib_gid_attr *attr)
 {
+	struct ib_gid_attr_info *info =
+		container_of(attr, struct ib_gid_attr_info, attr);
 	struct ib_gid_table_entry *entry =
-		container_of(attr, struct ib_gid_table_entry, attr);
+		container_of(info, struct ib_gid_table_entry, attr_info);
 
 	get_gid_entry(entry);
 }
@@ -1270,11 +1278,14 @@ EXPORT_SYMBOL(rdma_hold_gid_attr);
  */
 struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)
 {
+	struct ib_gid_attr_info *info =
+		container_of(attr, struct ib_gid_attr_info, attr);
 	struct ib_gid_table_entry *entry =
-			container_of(attr, struct ib_gid_table_entry, attr);
-	struct ib_device *device = entry->attr.device;
+			container_of(info, struct ib_gid_table_entry,
+				     attr_info);
+	struct ib_device *device = entry->attr_info.attr.device;
 	struct net_device *ndev = ERR_PTR(-ENODEV);
-	u8 port_num = entry->attr.port_num;
+	u8 port_num = entry->attr_info.attr.port_num;
 	struct ib_gid_table *table;
 	unsigned long flags;
 	bool valid;
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 9d8d7fd..5560a14 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -174,6 +174,11 @@ struct ib_gid_attr {
 	u8			port_num;
 };
 
+struct ib_gid_attr_info {
+	struct ib_gid_attr attr;
+	void *context;
+};
+
 enum {
 	/* set the local administered indication */
 	IB_SA_WELL_KNOWN_GUID	= BIT_ULL(57) | 2,
-- 
2.5.5

[PATCH for-next v2 2/2] RDMA/bnxt_re: Retrieve the driver gid context from gid_attr

[Selvin Xavier]

Use the container_of macro to retrieve the driver gid
context.

Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
 drivers/infiniband/hw/bnxt_re/ib_verbs.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index ad5112a..b5f611f 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -640,6 +640,8 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,
 	struct bnxt_re_dev *rdev = pd->rdev;
 	const struct ib_gid_attr *sgid_attr;
 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
+	struct ib_gid_attr_info *info;
+	struct bnxt_re_gid_ctx *ctx;
 	u8 nw_type;
 	int rc;
 
@@ -651,22 +653,21 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,
 	ah->rdev = rdev;
 	ah->qplib_ah.pd = &pd->qplib_pd;
 
+	sgid_attr = grh->sgid_attr;
+
+	info = container_of(sgid_attr, struct ib_gid_attr_info, attr);
+	ctx = info->context;
+
 	/* Supply the configuration for the HW */
 	memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
 	       sizeof(union ib_gid));
-	/*
-	 * If RoCE V2 is enabled, stack will have two entries for
-	 * each GID entry. Avoiding this duplicte entry in HW. Dividing
-	 * the GID index by 2 for RoCE V2
-	 */
-	ah->qplib_ah.sgid_index = grh->sgid_index / 2;
+	ah->qplib_ah.sgid_index = ctx->idx;
 	ah->qplib_ah.host_sgid_index = grh->sgid_index;
 	ah->qplib_ah.traffic_class = grh->traffic_class;
 	ah->qplib_ah.flow_label = grh->flow_label;
 	ah->qplib_ah.hop_limit = grh->hop_limit;
 	ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
 
-	sgid_attr = grh->sgid_attr;
 	/* Get network header type for this GID */
 	nw_type = rdma_gid_attr_network_type(sgid_attr);
 	ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
@@ -1521,6 +1522,8 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 	struct bnxt_re_dev *rdev = qp->rdev;
 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 	enum ib_qp_state curr_qp_state, new_qp_state;
+	struct ib_gid_attr_info *info;
+	struct bnxt_re_gid_ctx *ctx;
 	int rc, entries;
 	unsigned int flags;
 	u8 nw_type;
@@ -1592,6 +1595,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 			rdma_ah_read_grh(&qp_attr->ah_attr);
 		const struct ib_gid_attr *sgid_attr;
 
+		sgid_attr = qp_attr->ah_attr.grh.sgid_attr;
+		info = container_of(sgid_attr, struct ib_gid_attr_info, attr);
+		ctx = info->context;
+
 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
 				     CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
 				     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
@@ -1602,11 +1609,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
 		       sizeof(qp->qplib_qp.ah.dgid.data));
 		qp->qplib_qp.ah.flow_label = grh->flow_label;
-		/* If RoCE V2 is enabled, stack will have two entries for
-		 * each GID entry. Avoiding this duplicte entry in HW. Dividing
-		 * the GID index by 2 for RoCE V2
-		 */
-		qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2;
+		qp->qplib_qp.ah.sgid_index = ctx->idx;
 		qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
 		qp->qplib_qp.ah.hop_limit = grh->hop_limit;
 		qp->qplib_qp.ah.traffic_class = grh->traffic_class;
@@ -1614,7 +1617,6 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		ether_addr_copy(qp->qplib_qp.ah.dmac,
 				qp_attr->ah_attr.roce.dmac);
 
-		sgid_attr = qp_attr->ah_attr.grh.sgid_attr;
 		rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
 					     &qp->qplib_qp.smac[0]);
 		if (rc)
-- 
2.5.5

Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr

[Jason Gunthorpe]

On Sun, Dec 15, 2019 at 10:20:00PM -0800, Selvin Xavier wrote:
> Provide an option to retrieve the driver gid context from ib_gid_attr
> structure. Introduce ib_gid_attr_info structure which include both
> gid_attr and the GID's HW context. Replace the attr and context
> members of ib_gid_table_entry with the new ib_gid_attr_info
> structure. Vendor drivers can refer to its own HW gid context
> using the container_of macro.

This seems really weird. Why are we adding a new struct instead of
adding context to the normal gid_attr, or adding some
'get_ib_attr_priv' call?

Jason

Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr

[Parav Pandit]

On 12/18/2019 7:38 PM, Jason Gunthorpe wrote:
> On Sun, Dec 15, 2019 at 10:20:00PM -0800, Selvin Xavier wrote:
>> Provide an option to retrieve the driver gid context from ib_gid_attr
>> structure. Introduce ib_gid_attr_info structure which include both
>> gid_attr and the GID's HW context. Replace the attr and context
>> members of ib_gid_table_entry with the new ib_gid_attr_info
>> structure. Vendor drivers can refer to its own HW gid context
>> using the container_of macro.
> 
> This seems really weird. Why are we adding a new struct instead of
> adding context to the normal gid_attr, or adding some
> 'get_ib_attr_priv' call?

Rest of the stack didn't need to touch context, so it is added only as
vendor driver facing container_of().

Instead I guess a new symbol as rdma_get_gid_attr_context() can be added
too.

Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr

[Selvin Xavier]

On Thu, Dec 19, 2019 at 7:52 AM Parav Pandit <parav@mellanox.com> wrote:
>
> On 12/18/2019 7:38 PM, Jason Gunthorpe wrote:
> > On Sun, Dec 15, 2019 at 10:20:00PM -0800, Selvin Xavier wrote:
> >> Provide an option to retrieve the driver gid context from ib_gid_attr
> >> structure. Introduce ib_gid_attr_info structure which include both
> >> gid_attr and the GID's HW context. Replace the attr and context
> >> members of ib_gid_table_entry with the new ib_gid_attr_info
> >> structure. Vendor drivers can refer to its own HW gid context
> >> using the container_of macro.
> >
> > This seems really weird. Why are we adding a new struct instead of
> > adding context to the normal gid_attr, or adding some
> > 'get_ib_attr_priv' call?
>
> Rest of the stack didn't need to touch context, so it is added only as
> vendor driver facing container_of().
Added the new structure since I didn't want to move the private structure
ib_gid_table_entry to a header file.
>
> Instead I guess a new symbol as rdma_get_gid_attr_context() can be added
> too.
I am okay with both adding context to gid_attr struct or adding a symbol.
Let me know your preference.
Or shall i handle this inside bnxt_re itself. Not sure whether any
other drivers intend to use this.

Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr

[Jason Gunthorpe]

On Thu, Dec 19, 2019 at 10:36:45AM +0530, Selvin Xavier wrote:
> > Instead I guess a new symbol as rdma_get_gid_attr_context() can be added
> > too.
> I am okay with both adding context to gid_attr struct or adding a symbol.
> Let me know your preference.
> Or shall i handle this inside bnxt_re itself. Not sure whether any
> other drivers intend to use this.

Having a function to return the same void * that is passed to the
driver ops functions seems reasonable and small to me.

But you could also spruce this up a bit and have it work more like a
true 'priv' and get rid of that void *..

The container_of thing is really odd

Jason

Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr

[Selvin Xavier]

On Thu, Dec 19, 2019 at 7:48 PM Jason Gunthorpe <jgg@ziepe.ca> wrote:
>
> On Thu, Dec 19, 2019 at 10:36:45AM +0530, Selvin Xavier wrote:
> > > Instead I guess a new symbol as rdma_get_gid_attr_context() can be added
> > > too.
> > I am okay with both adding context to gid_attr struct or adding a symbol.
> > Let me know your preference.
> > Or shall i handle this inside bnxt_re itself. Not sure whether any
> > other drivers intend to use this.
>
> Having a function to return the same void * that is passed to the
> driver ops functions seems reasonable and small to me.
Posting a patch with a symbol that returns void *

Thanks

>
> But you could also spruce this up a bit and have it work more like a
> true 'priv' and get rid of that void *..
>
> The container_of thing is really odd
>
> Jason