[PATCH rdma-next 00/10] Various cleanups

[PATCH rdma-next 00/10] Various cleanups

[Leon Romanovsky]

From: Leon Romanovsky <leonro@nvidia.com>

Various simple cleanups to mlx4, mlx5 and core.

Parav Pandit (10):
  IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev
  IB/mlx5: Avoid calling query device for reading pkey table length
  IB/mlx5: Support default partition key for representor port
  IB/mlx5: Improve query port for representor port
  RDMA/core: Introduce and use API to read port immutable data
  IB/mlx5: Use rdma_for_each_port for port iteration
  IB/mlx5: Return appropriate error code instead of ENOMEM
  IB/cm: Avoid a loop when device has 255 ports
  IB/mlx4: Use port iterator and validation APIs
  IB/core: Use valid port number to check link layer

 drivers/infiniband/core/cm.c         |   8 +-
 drivers/infiniband/core/device.c     |  14 +++
 drivers/infiniband/core/verbs.c      |   4 +-
 drivers/infiniband/hw/mlx4/main.c    |   2 +-
 drivers/infiniband/hw/mlx4/sysfs.c   |   4 +-
 drivers/infiniband/hw/mlx5/mad.c     |  10 +-
 drivers/infiniband/hw/mlx5/main.c    | 137 +++++++--------------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |   9 +-
 drivers/infiniband/hw/mlx5/odp.c     |   4 -
 drivers/infiniband/hw/mlx5/qp.c      |  20 ++--
 drivers/infiniband/hw/mlx5/wr.c      |   2 +-
 include/linux/mlx5/driver.h          |   8 --
 include/rdma/ib_verbs.h              |   3 +
 13 files changed, 84 insertions(+), 141 deletions(-)

--
2.29.2

[PATCH rdma-next 01/10] IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

mlx5_port_caps are RDMA specific capabilities. These are not used by the
mlx5_core_device at all. Move them to mlx5_ib_dev where it is used and reduce
the scope of it to multiple drivers.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/hw/mlx5/mad.c     |  8 ++++----
 drivers/infiniband/hw/mlx5/main.c    | 14 ++++++--------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  8 ++++++++
 drivers/infiniband/hw/mlx5/qp.c      |  6 +++---
 drivers/infiniband/hw/mlx5/wr.c      |  2 +-
 include/linux/mlx5/driver.h          |  8 --------
 6 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index 53dec6063245..e9d0a5269582 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -48,7 +48,7 @@ static bool can_do_mad_ifc(struct mlx5_ib_dev *dev, u8 port_num,
 	if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED &&
 	    in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
 		return true;
-	return dev->mdev->port_caps[port_num - 1].has_smi;
+	return dev->port_caps[port_num - 1].has_smi;
 }

 static int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey,
@@ -299,7 +299,7 @@ int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)

 	packet_error = be16_to_cpu(out_mad->status);

-	dev->mdev->port_caps[port - 1].ext_port_cap = (!err && !packet_error) ?
+	dev->port_caps[port - 1].ext_port_cap = (!err && !packet_error) ?
 		MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO : 0;

 out:
@@ -549,7 +549,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 	props->port_cap_flags	= be32_to_cpup((__be32 *)(out_mad->data + 20));
 	props->gid_tbl_len	= out_mad->data[50];
 	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
-	props->pkey_tbl_len	= mdev->port_caps[port - 1].pkey_table_len;
+	props->pkey_tbl_len	= dev->port_caps[port - 1].pkey_table_len;
 	props->bad_pkey_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 46));
 	props->qkey_viol_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 48));
 	props->active_width	= out_mad->data[31] & 0xf;
@@ -589,7 +589,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,

 	/* If reported active speed is QDR, check if is FDR-10 */
 	if (props->active_speed == 4) {
-		if (mdev->port_caps[port - 1].ext_port_cap &
+		if (dev->port_caps[port - 1].ext_port_cap &
 		    MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) {
 			init_query_mad(in_mad);
 			in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 2f38d00bc2af..979f58ed1de2 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2960,8 +2960,8 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)
 	int err;
 	int port;

-	for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) {
-		dev->mdev->port_caps[port - 1].has_smi = false;
+	for (port = 1; port <= ARRAY_SIZE(dev->port_caps); port++) {
+		dev->port_caps[port - 1].has_smi = false;
 		if (MLX5_CAP_GEN(dev->mdev, port_type) ==
 		    MLX5_CAP_PORT_TYPE_IB) {
 			if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
@@ -2973,10 +2973,10 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)
 						    port, err);
 					return err;
 				}
-				dev->mdev->port_caps[port - 1].has_smi =
+				dev->port_caps[port - 1].has_smi =
 					vport_ctx.has_smi;
 			} else {
-				dev->mdev->port_caps[port - 1].has_smi = true;
+				dev->port_caps[port - 1].has_smi = true;
 			}
 		}
 	}
@@ -3018,10 +3018,8 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 		goto out;
 	}

-	dev->mdev->port_caps[port - 1].pkey_table_len =
-					dprops->max_pkeys;
-	dev->mdev->port_caps[port - 1].gid_table_len =
-					pprops->gid_tbl_len;
+	dev->port_caps[port - 1].pkey_table_len = dprops->max_pkeys;
+	dev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
 	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
 		    port, dprops->max_pkeys, pprops->gid_tbl_len);

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 39cf7ccc8ed3..39ba8e1e9fee 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1033,6 +1033,13 @@ struct mlx5_var_table {
 	u64 num_var_hw_entries;
 };

+struct mlx5_port_caps {
+	int gid_table_len;
+	int pkey_table_len;
+	bool has_smi;
+	u8 ext_port_cap;
+};
+
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
@@ -1088,6 +1095,7 @@ struct mlx5_ib_dev {
 	struct mlx5_var_table var_table;

 	struct xarray sig_mrs;
+	struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];
 };

 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index aa5f693fe0cd..5822655fe91e 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -3178,10 +3178,10 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,

 	if (ah_flags & IB_AH_GRH) {
 		if (grh->sgid_index >=
-		    dev->mdev->port_caps[port - 1].gid_table_len) {
+		    dev->port_caps[port - 1].gid_table_len) {
 			pr_err("sgid_index (%u) too large. max is %d\n",
 			       grh->sgid_index,
-			       dev->mdev->port_caps[port - 1].gid_table_len);
+			       dev->port_caps[port - 1].gid_table_len);
 			return -EINVAL;
 		}
 	}
@@ -4313,7 +4313,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	if (attr_mask & IB_QP_PKEY_INDEX) {
 		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
 		if (attr->pkey_index >=
-		    dev->mdev->port_caps[port - 1].pkey_table_len) {
+		    dev->port_caps[port - 1].pkey_table_len) {
 			mlx5_ib_dbg(dev, "invalid pkey index %d\n",
 				    attr->pkey_index);
 			goto out;
diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c
index d6038fb6c50c..cf2852cba45c 100644
--- a/drivers/infiniband/hw/mlx5/wr.c
+++ b/drivers/infiniband/hw/mlx5/wr.c
@@ -1369,7 +1369,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 			handle_qpt_uc(wr, &seg, &size);
 			break;
 		case IB_QPT_SMI:
-			if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
+			if (unlikely(!dev->port_caps[qp->port - 1].has_smi)) {
 				mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
 				err = -EPERM;
 				*bad_wr = wr;
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index f9e7036ae5a5..7a6652f01c77 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -305,13 +305,6 @@ struct mlx5_cmd {
 	struct mlx5_cmd_stats *stats;
 };

-struct mlx5_port_caps {
-	int	gid_table_len;
-	int	pkey_table_len;
-	u8	ext_port_cap;
-	bool	has_smi;
-};
-
 struct mlx5_cmd_mailbox {
 	void	       *buf;
 	dma_addr_t	dma;
@@ -697,7 +690,6 @@ struct mlx5_core_dev {
 	u8			rev_id;
 	char			board_id[MLX5_BOARD_ID_LEN];
 	struct mlx5_cmd		cmd;
-	struct mlx5_port_caps	port_caps[MLX5_MAX_PORTS];
 	struct {
 		u32 hca_cur[MLX5_CAP_NUM][MLX5_UN_SZ_DW(hca_cap_union)];
 		u32 hca_max[MLX5_CAP_NUM][MLX5_UN_SZ_DW(hca_cap_union)];
--
2.29.2

[PATCH rdma-next 02/10] IB/mlx5: Avoid calling query device for reading pkey table length

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

Pkey table length for all the ports of the device is the same.
Currently get_ports_cap() reads and stores it for each port by querying
the device which reads more than just pkey table length.

For representor device ports which can be in range of hundreds, it
queries is for each such port and end up returning same value for all
the ports.

When in representor mode, modify QP accesses pkey port caps for a port
index that can be outside of the port_caps table.

Hence, simplify the logic to query the max pkey table length only once
during device initialization sequence.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/hw/mlx5/mad.c     |  2 +-
 drivers/infiniband/hw/mlx5/main.c    | 24 ++++++------------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  2 +-
 drivers/infiniband/hw/mlx5/qp.c      | 12 ++++--------
 4 files changed, 12 insertions(+), 28 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index e9d0a5269582..cdb47a00e516 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -549,7 +549,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 	props->port_cap_flags	= be32_to_cpup((__be32 *)(out_mad->data + 20));
 	props->gid_tbl_len	= out_mad->data[50];
 	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
-	props->pkey_tbl_len	= dev->port_caps[port - 1].pkey_table_len;
+	props->pkey_tbl_len	= dev->pkey_table_len;
 	props->bad_pkey_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 46));
 	props->qkey_viol_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 48));
 	props->active_width	= out_mad->data[31] & 0xf;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 979f58ed1de2..5765f30f1788 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -816,9 +816,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	if (err)
 		return err;

-	err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
-	if (err)
-		return err;
+	props->max_pkeys = dev->pkey_table_len;

 	err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
 	if (err)
@@ -2993,7 +2991,6 @@ static void get_ext_port_caps(struct mlx5_ib_dev *dev)

 static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 {
-	struct ib_device_attr *dprops = NULL;
 	struct ib_port_attr *pprops = NULL;
 	int err = -ENOMEM;

@@ -3001,16 +2998,6 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 	if (!pprops)
 		goto out;

-	dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
-	if (!dprops)
-		goto out;
-
-	err = mlx5_ib_query_device(&dev->ib_dev, dprops, NULL);
-	if (err) {
-		mlx5_ib_warn(dev, "query_device failed %d\n", err);
-		goto out;
-	}
-
 	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
 	if (err) {
 		mlx5_ib_warn(dev, "query_port %d failed %d\n",
@@ -3018,15 +3005,12 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 		goto out;
 	}

-	dev->port_caps[port - 1].pkey_table_len = dprops->max_pkeys;
 	dev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
 	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
-		    port, dprops->max_pkeys, pprops->gid_tbl_len);
+		    port, dev->pkey_table_len, pprops->gid_tbl_len);

 out:
 	kfree(pprops);
-	kfree(dprops);
-
 	return err;
 }

@@ -3993,6 +3977,10 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	if (err)
 		goto err_mp;

+	err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
+	if (err)
+		goto err_mp;
+
 	if (mlx5_use_mad_ifc(dev))
 		get_ext_port_caps(dev);

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 39ba8e1e9fee..c0c5e0043b3e 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1035,7 +1035,6 @@ struct mlx5_var_table {

 struct mlx5_port_caps {
 	int gid_table_len;
-	int pkey_table_len;
 	bool has_smi;
 	u8 ext_port_cap;
 };
@@ -1096,6 +1095,7 @@ struct mlx5_ib_dev {

 	struct xarray sig_mrs;
 	struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];
+	u16 pkey_table_len;
 };

 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 5822655fe91e..b65720a05a18 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -4310,14 +4310,10 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		goto out;
 	}

-	if (attr_mask & IB_QP_PKEY_INDEX) {
-		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
-		if (attr->pkey_index >=
-		    dev->port_caps[port - 1].pkey_table_len) {
-			mlx5_ib_dbg(dev, "invalid pkey index %d\n",
-				    attr->pkey_index);
-			goto out;
-		}
+	if ((attr_mask & IB_QP_PKEY_INDEX) &&
+	    attr->pkey_index >= dev->pkey_table_len) {
+		mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr->pkey_index);
+		goto out;
 	}

 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
--
2.29.2

[PATCH rdma-next 03/10] IB/mlx5: Support default partition key for representor port

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

Representor port has only one default pkey.
Hence have simpler query pkey callback or it.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/hw/mlx5/main.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 5765f30f1788..6f2c03230c49 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -1398,6 +1398,16 @@ static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port,
 	return ret;
 }

+static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
+				  u16 *pkey)
+{
+	/* Default special Pkey for representor device port as per the
+	 * IB specification 1.3 section 10.9.1.2.
+	 */
+	*pkey = 0xffff;
+	return 0;
+}
+
 static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 			     union ib_gid *gid)
 {
@@ -4217,6 +4227,7 @@ static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
 static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
 	.get_port_immutable = mlx5_port_rep_immutable,
 	.query_port = mlx5_ib_rep_query_port,
+	.query_pkey = mlx5_ib_rep_query_pkey,
 };

 static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
--
2.29.2

[PATCH rdma-next 04/10] IB/mlx5: Improve query port for representor port

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

Improve query port functionality for representor port as below.

1. RoCE Qkey violation counters are not applicable for representor
port.
2. Avoid setting gid_tbl_len twice for representor port.
3. Avoid setting ip_gids and IB_PORT_CM_SUP property for representor
port as GID table is empty and CM support is not present in
representor mode.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/hw/mlx5/main.c | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 6f2c03230c49..5a7f8fa2f452 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -462,7 +462,6 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 	struct net_device *ndev, *upper;
 	enum ib_mtu ndev_ib_mtu;
 	bool put_mdev = true;
-	u16 qkey_viol_cntr;
 	u32 eth_prot_oper;
 	u8 mdev_port_num;
 	bool ext;
@@ -500,20 +499,22 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 	translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
 				 &props->active_width, ext);

-	props->port_cap_flags |= IB_PORT_CM_SUP;
-	props->ip_gids = true;
+	if (!dev->is_rep && mlx5_is_roce_enabled(mdev)) {
+		u16 qkey_viol_cntr;

-	props->gid_tbl_len      = MLX5_CAP_ROCE(dev->mdev,
-						roce_address_table_size);
+		props->port_cap_flags |= IB_PORT_CM_SUP;
+		props->ip_gids = true;
+		props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
+						   roce_address_table_size);
+		mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
+		props->qkey_viol_cntr = qkey_viol_cntr;
+	}
 	props->max_mtu          = IB_MTU_4096;
 	props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
 	props->pkey_tbl_len     = 1;
 	props->state            = IB_PORT_DOWN;
 	props->phys_state       = IB_PORT_PHYS_STATE_DISABLED;

-	mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
-	props->qkey_viol_cntr = qkey_viol_cntr;
-
 	/* If this is a stub query for an unaffiliated port stop here */
 	if (!put_mdev)
 		goto out;
@@ -1383,19 +1384,7 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port,
 				  struct ib_port_attr *props)
 {
-	int ret;
-
-	/* Only link layer == ethernet is valid for representors
-	 * and we always use port 1
-	 */
-	ret = mlx5_query_port_roce(ibdev, port, props);
-	if (ret || !props)
-		return ret;
-
-	/* We don't support GIDS */
-	props->gid_tbl_len = 0;
-
-	return ret;
+	return mlx5_query_port_roce(ibdev, port, props);
 }

 static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
--
2.29.2

[PATCH rdma-next 05/10] RDMA/core: Introduce and use API to read port immutable data

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

Currently mlx5 driver caches port GID table length for 2 ports.
It is also cached by IB core as port immutable data.

When mlx5 representor ports are present, which are usually more than 2,
invalid access to port_caps array can happen while validating the GID
table length which is only for 2 ports.

To avoid this, take help of the IB cores port immutable data by exposing
an API to read the port immutable fields.

Remove mlx5 driver's internal cache, thereby reduce code and data.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/core/device.c     | 14 +++++++
 drivers/infiniband/hw/mlx5/main.c    | 55 +---------------------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  1 -
 drivers/infiniband/hw/mlx5/qp.c      |  8 ++--
 include/rdma/ib_verbs.h              |  3 ++
 5 files changed, 23 insertions(+), 58 deletions(-)

diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index c895d7bfa512..051c018fb73c 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -848,6 +848,20 @@ static int setup_port_data(struct ib_device *device)
 	return 0;
 }

+/**
+ * ib_port_immutable_read() - Read rdma port's immutable data
+ * @dev - IB device
+ * @port - port number whose immutable data to read. It starts with index 1 and
+ *         valid upto including rdma_end_port().
+ */
+const struct ib_port_immutable*
+ib_port_immutable_read(struct ib_device *dev, unsigned int port)
+{
+	WARN_ON(!rdma_is_port_valid(dev, port));
+	return &dev->port_data[port].immutable;
+}
+EXPORT_SYMBOL(ib_port_immutable_read);
+
 void ib_get_device_fw_str(struct ib_device *dev, char *str)
 {
 	if (dev->ops.get_dev_fw_str)
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 5a7f8fa2f452..742e565309f8 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2988,41 +2988,6 @@ static void get_ext_port_caps(struct mlx5_ib_dev *dev)
 		mlx5_query_ext_port_caps(dev, port);
 }

-static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
-{
-	struct ib_port_attr *pprops = NULL;
-	int err = -ENOMEM;
-
-	pprops = kzalloc(sizeof(*pprops), GFP_KERNEL);
-	if (!pprops)
-		goto out;
-
-	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
-	if (err) {
-		mlx5_ib_warn(dev, "query_port %d failed %d\n",
-			     port, err);
-		goto out;
-	}
-
-	dev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
-	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
-		    port, dev->pkey_table_len, pprops->gid_tbl_len);
-
-out:
-	kfree(pprops);
-	return err;
-}
-
-static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
-{
-	/* For representors use port 1, is this is the only native
-	 * port
-	 */
-	if (dev->is_rep)
-		return __get_port_caps(dev, 1);
-	return __get_port_caps(dev, port);
-}
-
 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
 {
 	switch (umr_fence_cap) {
@@ -3496,10 +3461,6 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
 	if (err)
 		goto unbind;

-	err = get_port_caps(ibdev, mlx5_core_native_port_num(mpi->mdev));
-	if (err)
-		goto unbind;
-
 	err = mlx5_add_netdev_notifier(ibdev, port_num);
 	if (err) {
 		mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
@@ -3577,11 +3538,9 @@ static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
 				break;
 			}
 		}
-		if (!bound) {
-			get_port_caps(dev, i + 1);
+		if (!bound)
 			mlx5_ib_dbg(dev, "no free port found for port %d\n",
 				    i + 1);
-		}
 	}

 	list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
@@ -3964,18 +3923,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	if (err)
 		goto err_mp;

-	if (!mlx5_core_mp_enabled(mdev)) {
-		for (i = 1; i <= dev->num_ports; i++) {
-			err = get_port_caps(dev, i);
-			if (err)
-				break;
-		}
-	} else {
-		err = get_port_caps(dev, mlx5_core_native_port_num(mdev));
-	}
-	if (err)
-		goto err_mp;
-
 	err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
 	if (err)
 		goto err_mp;
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index c0c5e0043b3e..69c45adb2518 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1034,7 +1034,6 @@ struct mlx5_var_table {
 };

 struct mlx5_port_caps {
-	int gid_table_len;
 	bool has_smi;
 	u8 ext_port_cap;
 };
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index b65720a05a18..abdb73fdbec2 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -3177,11 +3177,13 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			 alt ? attr->alt_pkey_index : attr->pkey_index);

 	if (ah_flags & IB_AH_GRH) {
-		if (grh->sgid_index >=
-		    dev->port_caps[port - 1].gid_table_len) {
+		const struct ib_port_immutable *immutable;
+
+		immutable = ib_port_immutable_read(&dev->ib_dev, port);
+		if (grh->sgid_index >= immutable->gid_tbl_len) {
 			pr_err("sgid_index (%u) too large. max is %d\n",
 			       grh->sgid_index,
-			       dev->port_caps[port - 1].gid_table_len);
+			       immutable->gid_tbl_len);
 			return -EINVAL;
 		}
 	}
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 62e574c50555..ca28fca5736b 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -4674,4 +4674,7 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)

 	return (u32)(v & IB_GRH_FLOWLABEL_MASK);
 }
+
+const struct ib_port_immutable*
+ib_port_immutable_read(struct ib_device *dev, unsigned int port);
 #endif /* IB_VERBS_H */
--
2.29.2

[PATCH rdma-next 06/10] IB/mlx5: Use rdma_for_each_port for port iteration

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

Instead of open coding the loop for port iteration, use
rdma_for_each_port macro provided by core.

To use such macro, early initialization of phys_port_cnt is needed.
Hence, initialize such constant early enough in the init stage.

Whichever functions are called with port using rdma_for_each_port(),
convert their port type from u8 to unsigned int to match the core API.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/hw/mlx5/mad.c     |  2 +-
 drivers/infiniband/hw/mlx5/main.c    | 15 ++++++++-------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index cdb47a00e516..652c6ccf1881 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -279,7 +279,7 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
 }

-int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)
+int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port)
 {
 	struct ib_smp *in_mad  = NULL;
 	struct ib_smp *out_mad = NULL;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 742e565309f8..ad7bb37e501d 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2982,9 +2982,9 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)

 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
 {
-	int port;
+	unsigned int port;

-	for (port = 1; port <= dev->num_ports; port++)
+	rdma_for_each_port (&dev->ib_dev, port)
 		mlx5_query_ext_port_caps(dev, port);
 }

@@ -3905,6 +3905,12 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	int err;
 	int i;

+	dev->ib_dev.node_type = RDMA_NODE_IB_CA;
+	dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
+	dev->ib_dev.phys_port_cnt = dev->num_ports;
+	dev->ib_dev.dev.parent = mdev->device;
+	dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
+
 	for (i = 0; i < dev->num_ports; i++) {
 		spin_lock_init(&dev->port[i].mp.mpi_lock);
 		rwlock_init(&dev->port[i].roce.netdev_lock);
@@ -3930,12 +3936,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	if (mlx5_use_mad_ifc(dev))
 		get_ext_port_caps(dev);

-	dev->ib_dev.node_type		= RDMA_NODE_IB_CA;
-	dev->ib_dev.local_dma_lkey	= 0 /* not supported for now */;
-	dev->ib_dev.phys_port_cnt	= dev->num_ports;
 	dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_count(mdev);
-	dev->ib_dev.dev.parent		= mdev->device;
-	dev->ib_dev.lag_flags		= RDMA_LAG_FLAGS_HASH_ALL_SLAVES;

 	mutex_init(&dev->cap_mask_mutex);
 	INIT_LIST_HEAD(&dev->qp_list);
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 69c45adb2518..88cc26e008fc 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1291,7 +1291,7 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 			size_t *out_mad_size, u16 *out_mad_pkey_index);
 int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
-int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
+int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port);
 int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
 					 __be64 *sys_image_guid);
 int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
--
2.29.2

[PATCH rdma-next 07/10] IB/mlx5: Return appropriate error code instead of ENOMEM

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

When mlx5_ib_stage_init_init() fails, return the error code related to
failure instead of -ENOMEM.

Fixes: 16c1975f1032 ("IB/mlx5: Create profile infrastructure to add and remove stages")
Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/hw/mlx5/main.c | 3 +--
 drivers/infiniband/hw/mlx5/odp.c  | 4 ----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index ad7bb37e501d..9e8b4d591138 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3952,8 +3952,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)

 err_mp:
 	mlx5_ib_cleanup_multiport_master(dev);
-
-	return -ENOMEM;
+	return err;
 }

 static int mlx5_ib_enable_driver(struct ib_device *dev)
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index f4b82daf1e22..a1be8fb2800e 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -484,10 +484,6 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
 	}

 	xa_lock(&imr->implicit_children);
-	/*
-	 * Once the store to either xarray completes any error unwind has to
-	 * use synchronize_srcu(). Avoid this with xa_reserve()
-	 */
 	ret = __xa_cmpxchg(&imr->implicit_children, idx, NULL, mr,
 			   GFP_KERNEL);
 	if (unlikely(ret)) {
--
2.29.2

[PATCH rdma-next 08/10] IB/cm: Avoid a loop when device has 255 ports

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

When RDMA device has 255 ports, loop iterator i overflows.
Due to which cm_add_one() port iterator loops infinitely.
Use core provided port iterator to avoid the infinite loop.

Fixes: a977049dacde ("[PATCH] IB: Add the kernel CM implementation")
Signed-off-by: Mark Bloch <mbloch@nvidia.com>
Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/core/cm.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 98165589c8ab..be996dba040c 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -4333,7 +4333,7 @@ static int cm_add_one(struct ib_device *ib_device)
 	unsigned long flags;
 	int ret;
 	int count = 0;
-	u8 i;
+	unsigned int i;

 	cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
 			 GFP_KERNEL);
@@ -4345,7 +4345,7 @@ static int cm_add_one(struct ib_device *ib_device)
 	cm_dev->going_down = 0;

 	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
-	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
+	rdma_for_each_port (ib_device, i) {
 		if (!rdma_cap_ib_cm(ib_device, i))
 			continue;

@@ -4431,7 +4431,7 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
 		.clr_port_cap_mask = IB_PORT_CM_SUP
 	};
 	unsigned long flags;
-	int i;
+	unsigned int i;

 	write_lock_irqsave(&cm.device_lock, flags);
 	list_del(&cm_dev->list);
@@ -4441,7 +4441,7 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
 	cm_dev->going_down = 1;
 	spin_unlock_irq(&cm.lock);

-	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
+	rdma_for_each_port (ib_device, i) {
 		if (!rdma_cap_ib_cm(ib_device, i))
 			continue;

--
2.29.2

[PATCH rdma-next 09/10] IB/mlx4: Use port iterator and validation APIs

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

Use IB core provided routine to check if the port is valid or not and
to iterate over IB ports.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/hw/mlx4/main.c  | 2 +-
 drivers/infiniband/hw/mlx4/sysfs.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index e3cd402c079a..f26a0d920842 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -1699,7 +1699,7 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
 	struct mlx4_dev *dev = (to_mdev(qp->device))->dev;
 	int is_bonded = mlx4_is_bonded(dev);

-	if (flow_attr->port < 1 || flow_attr->port > qp->device->phys_port_cnt)
+	if (!rdma_is_port_valid(qp->device, flow_attr->port))
 		return ERR_PTR(-EINVAL);

 	if (flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP)
diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c
index 1b5891130aab..24ee79aa2122 100644
--- a/drivers/infiniband/hw/mlx4/sysfs.c
+++ b/drivers/infiniband/hw/mlx4/sysfs.c
@@ -798,7 +798,7 @@ static void unregister_pkey_tree(struct mlx4_ib_dev *device)

 int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *dev)
 {
-	int i;
+	unsigned int i;
 	int ret = 0;

 	if (!mlx4_is_master(dev->dev))
@@ -817,7 +817,7 @@ int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *dev)
 		goto err_ports;
 	}

-	for (i = 1; i <= dev->ib_dev.phys_port_cnt; ++i) {
+	rdma_for_each_port(&dev->ib_dev, i) {
 		ret = add_port_entries(dev, i);
 		if (ret)
 			goto err_add_entries;
--
2.29.2

[PATCH rdma-next 10/10] IB/core: Use valid port number to check link layer

[Leon Romanovsky]

From: Parav Pandit <parav@nvidia.com>

IB HCA port starts from 1. Use IB core provided port iterator API to
avoid any assumption with start port number.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/infiniband/core/verbs.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 6fc6a5b41f86..d25695ad50de 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -2281,7 +2281,7 @@ static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
 	struct ib_qp_init_attr init_attr = {};
 	struct ib_qp_attr attr = {};
 	int num_eth_ports = 0;
-	int port;
+	unsigned int port;
 
 	/* If QP state >= init, it is assigned to a port and we can check this
 	 * port only.
@@ -2296,7 +2296,7 @@ static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
 	}
 
 	/* Can't get a quick answer, iterate over all ports */
-	for (port = 0; port < qp->device->phys_port_cnt; port++)
+	rdma_for_each_port(qp->device, port)
 		if (rdma_port_get_link_layer(qp->device, port) !=
 		    IB_LINK_LAYER_INFINIBAND)
 			num_eth_ports++;
-- 
2.29.2

Re: [PATCH rdma-next 02/10] IB/mlx5: Avoid calling query device for reading pkey table length

[Leon Romanovsky]

On Wed, Jan 27, 2021 at 05:00:02PM +0200, Leon Romanovsky wrote:
> From: Parav Pandit <parav@nvidia.com>
>
> Pkey table length for all the ports of the device is the same.
> Currently get_ports_cap() reads and stores it for each port by querying
> the device which reads more than just pkey table length.
>
> For representor device ports which can be in range of hundreds, it
> queries is for each such port and end up returning same value for all
> the ports.
>
> When in representor mode, modify QP accesses pkey port caps for a port
> index that can be outside of the port_caps table.
>
> Hence, simplify the logic to query the max pkey table length only once
> during device initialization sequence.
>
> Signed-off-by: Parav Pandit <parav@nvidia.com>
> Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
> ---
>  drivers/infiniband/hw/mlx5/mad.c     |  2 +-
>  drivers/infiniband/hw/mlx5/main.c    | 24 ++++++------------------
>  drivers/infiniband/hw/mlx5/mlx5_ib.h |  2 +-
>  drivers/infiniband/hw/mlx5/qp.c      | 12 ++++--------
>  4 files changed, 12 insertions(+), 28 deletions(-)
>
> diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
> index e9d0a5269582..cdb47a00e516 100644
> --- a/drivers/infiniband/hw/mlx5/mad.c
> +++ b/drivers/infiniband/hw/mlx5/mad.c
> @@ -549,7 +549,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
>  	props->port_cap_flags	= be32_to_cpup((__be32 *)(out_mad->data + 20));
>  	props->gid_tbl_len	= out_mad->data[50];
>  	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
> -	props->pkey_tbl_len	= dev->port_caps[port - 1].pkey_table_len;
> +	props->pkey_tbl_len	= dev->pkey_table_len;
>  	props->bad_pkey_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 46));
>  	props->qkey_viol_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 48));
>  	props->active_width	= out_mad->data[31] & 0xf;
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index 979f58ed1de2..5765f30f1788 100644
> --- a/drivers/infiniband/hw/mlx5/main.c
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -816,9 +816,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
>  	if (err)
>  		return err;
>
> -	err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
> -	if (err)
> -		return err;
> +	props->max_pkeys = dev->pkey_table_len;
>
>  	err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
>  	if (err)
> @@ -2993,7 +2991,6 @@ static void get_ext_port_caps(struct mlx5_ib_dev *dev)
>
>  static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
>  {
> -	struct ib_device_attr *dprops = NULL;
>  	struct ib_port_attr *pprops = NULL;
>  	int err = -ENOMEM;
>
> @@ -3001,16 +2998,6 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
>  	if (!pprops)
>  		goto out;
>
> -	dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
> -	if (!dprops)
> -		goto out;
> -
> -	err = mlx5_ib_query_device(&dev->ib_dev, dprops, NULL);
> -	if (err) {
> -		mlx5_ib_warn(dev, "query_device failed %d\n", err);
> -		goto out;
> -	}
> -
>  	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
>  	if (err) {
>  		mlx5_ib_warn(dev, "query_port %d failed %d\n",
> @@ -3018,15 +3005,12 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
>  		goto out;
>  	}
>
> -	dev->port_caps[port - 1].pkey_table_len = dprops->max_pkeys;
>  	dev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
>  	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
> -		    port, dprops->max_pkeys, pprops->gid_tbl_len);
> +		    port, dev->pkey_table_len, pprops->gid_tbl_len);
>
>  out:
>  	kfree(pprops);
> -	kfree(dprops);
> -
>  	return err;
>  }
>
> @@ -3993,6 +3977,10 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
>  	if (err)
>  		goto err_mp;
>
> +	err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
> +	if (err)
> +		goto err_mp;
> +
>  	if (mlx5_use_mad_ifc(dev))
>  		get_ext_port_caps(dev);
>
> diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
> index 39ba8e1e9fee..c0c5e0043b3e 100644
> --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
> +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
> @@ -1035,7 +1035,6 @@ struct mlx5_var_table {
>
>  struct mlx5_port_caps {
>  	int gid_table_len;
> -	int pkey_table_len;
>  	bool has_smi;
>  	u8 ext_port_cap;
>  };
> @@ -1096,6 +1095,7 @@ struct mlx5_ib_dev {
>
>  	struct xarray sig_mrs;
>  	struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];
> +	u16 pkey_table_len;
>  };
>
>  static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
> diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
> index 5822655fe91e..b65720a05a18 100644
> --- a/drivers/infiniband/hw/mlx5/qp.c
> +++ b/drivers/infiniband/hw/mlx5/qp.c
> @@ -4310,14 +4310,10 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
>  		goto out;
>  	}
>
> -	if (attr_mask & IB_QP_PKEY_INDEX) {
> -		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
> -		if (attr->pkey_index >=
> -		    dev->port_caps[port - 1].pkey_table_len) {
> -			mlx5_ib_dbg(dev, "invalid pkey index %d\n",
> -				    attr->pkey_index);
> -			goto out;
> -		}
> +	if ((attr_mask & IB_QP_PKEY_INDEX) &&
> +	    attr->pkey_index >= dev->pkey_table_len) {
> +		mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr->pkey_index);
> +		goto out;
>  	}

This hunk made "int port;" in mlx5_ib_modify_qp() obsolete.

Thanks

>
>  	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
> --
> 2.29.2
>

Re: [PATCH rdma-next 07/10] IB/mlx5: Return appropriate error code instead of ENOMEM

[Jason Gunthorpe]

On Wed, Jan 27, 2021 at 05:00:07PM +0200, Leon Romanovsky wrote:
> From: Parav Pandit <parav@nvidia.com>
> 
> When mlx5_ib_stage_init_init() fails, return the error code related to
> failure instead of -ENOMEM.
> 
> Fixes: 16c1975f1032 ("IB/mlx5: Create profile infrastructure to add and remove stages")
> Signed-off-by: Parav Pandit <parav@nvidia.com>
> Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
>  drivers/infiniband/hw/mlx5/main.c | 3 +--
>  drivers/infiniband/hw/mlx5/odp.c  | 4 ----
>  2 files changed, 1 insertion(+), 6 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index ad7bb37e501d..9e8b4d591138 100644
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -3952,8 +3952,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
> 
>  err_mp:
>  	mlx5_ib_cleanup_multiport_master(dev);
> -
> -	return -ENOMEM;
> +	return err;
>  }
> 
>  static int mlx5_ib_enable_driver(struct ib_device *dev)
> diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
> index f4b82daf1e22..a1be8fb2800e 100644
> +++ b/drivers/infiniband/hw/mlx5/odp.c
> @@ -484,10 +484,6 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
>  	}
> 
>  	xa_lock(&imr->implicit_children);
> -	/*
> -	 * Once the store to either xarray completes any error unwind has to
> -	 * use synchronize_srcu(). Avoid this with xa_reserve()
> -	 */

It is not wrong to remove this comment, but why is it in this patch?

Jason

RE: [PATCH rdma-next 07/10] IB/mlx5: Return appropriate error code instead of ENOMEM

[Parav Pandit]

> From: Jason Gunthorpe <jgg@nvidia.com>
> Sent: Tuesday, February 2, 2021 8:51 PM
> >
> >  	xa_lock(&imr->implicit_children);
> > -	/*
> > -	 * Once the store to either xarray completes any error unwind has to
> > -	 * use synchronize_srcu(). Avoid this with xa_reserve()
> > -	 */
> 
> It is not wrong to remove this comment, but why is it in this patch?
> 
> Jason

Hi Leon,
Can you please check your tree, how this hunk got merged in this patch?
In my internal gerrit submission, I do not have this hunk.
May be somehow it got here as part of Yishai's ODP srcu patches?

Re: [PATCH rdma-next 05/10] RDMA/core: Introduce and use API to read port immutable data

[Jason Gunthorpe]

On Wed, Jan 27, 2021 at 05:00:05PM +0200, Leon Romanovsky wrote:
> + * ib_port_immutable_read() - Read rdma port's immutable data
> + * @dev - IB device
> + * @port - port number whose immutable data to read. It starts with index 1 and
> + *         valid upto including rdma_end_port().
> + */
> +const struct ib_port_immutable*
> +ib_port_immutable_read(struct ib_device *dev, unsigned int port)
> +{
> +	WARN_ON(!rdma_is_port_valid(dev, port));
> +	return &dev->port_data[port].immutable;
> +}
> +EXPORT_SYMBOL(ib_port_immutable_read);

Why add this function and only call it in one place?

Jason

RE: [PATCH rdma-next 05/10] RDMA/core: Introduce and use API to read port immutable data

[Parav Pandit]

> From: Jason Gunthorpe <jgg@nvidia.com>
> Sent: Tuesday, February 2, 2021 10:20 PM
> 
> On Wed, Jan 27, 2021 at 05:00:05PM +0200, Leon Romanovsky wrote:
> > + * ib_port_immutable_read() - Read rdma port's immutable data
> > + * @dev - IB device
> > + * @port - port number whose immutable data to read. It starts with
> index 1 and
> > + *         valid upto including rdma_end_port().
> > + */
> > +const struct ib_port_immutable*
> > +ib_port_immutable_read(struct ib_device *dev, unsigned int port) {
> > +	WARN_ON(!rdma_is_port_valid(dev, port));
> > +	return &dev->port_data[port].immutable; }
> > +EXPORT_SYMBOL(ib_port_immutable_read);
> 
> Why add this function and only call it in one place?
> 
A helper API from core helps
(a) to cut down mlx5 ib per port data structures and code around it
(b) it also avoids the need to maintain such driver internal data for large port count (which is not done today)

May be in future more drivers can use the same APIs.

Re: [PATCH rdma-next 05/10] RDMA/core: Introduce and use API to read port immutable data

[Jason Gunthorpe]

On Tue, Feb 02, 2021 at 06:05:48PM +0000, Parav Pandit wrote:
> 
> 
> > From: Jason Gunthorpe <jgg@nvidia.com>
> > Sent: Tuesday, February 2, 2021 10:20 PM
> > 
> > On Wed, Jan 27, 2021 at 05:00:05PM +0200, Leon Romanovsky wrote:
> > > + * ib_port_immutable_read() - Read rdma port's immutable data
> > > + * @dev - IB device
> > > + * @port - port number whose immutable data to read. It starts with
> > index 1 and
> > > + *         valid upto including rdma_end_port().
> > > + */
> > > +const struct ib_port_immutable*
> > > +ib_port_immutable_read(struct ib_device *dev, unsigned int port) {
> > > +	WARN_ON(!rdma_is_port_valid(dev, port));
> > > +	return &dev->port_data[port].immutable; }
> > > +EXPORT_SYMBOL(ib_port_immutable_read);
> > 
> > Why add this function and only call it in one place?
> > 
> A helper API from core helps
> (a) to cut down mlx5 ib per port data structures and code around it
> (b) it also avoids the need to maintain such driver internal data for large port count (which is not done today)
>
I mean why not just access the pointer directly, is isn't in a private
header or anything

Jason

Re: [PATCH rdma-next 07/10] IB/mlx5: Return appropriate error code instead of ENOMEM

[Leon Romanovsky]

On Tue, Feb 02, 2021 at 04:01:24PM +0000, Parav Pandit wrote:
>
>
> > From: Jason Gunthorpe <jgg@nvidia.com>
> > Sent: Tuesday, February 2, 2021 8:51 PM
> > >
> > >  	xa_lock(&imr->implicit_children);
> > > -	/*
> > > -	 * Once the store to either xarray completes any error unwind has to
> > > -	 * use synchronize_srcu(). Avoid this with xa_reserve()
> > > -	 */
> >
> > It is not wrong to remove this comment, but why is it in this patch?
> >
> > Jason
>
> Hi Leon,
> Can you please check your tree, how this hunk got merged in this patch?
> In my internal gerrit submission, I do not have this hunk.
> May be somehow it got here as part of Yishai's ODP srcu patches?

This explains why I didn't find this hunk in Yishai's patch.
https://lore.kernel.org/linux-rdma/20210128064812.1921519-1-leon@kernel.org

Thanks.

RE: [PATCH rdma-next 05/10] RDMA/core: Introduce and use API to read port immutable data

[Parav Pandit]

> From: Jason Gunthorpe <jgg@nvidia.com>
> Sent: Tuesday, February 2, 2021 11:44 PM
> 
> On Tue, Feb 02, 2021 at 06:05:48PM +0000, Parav Pandit wrote:
> >
> >
> > > From: Jason Gunthorpe <jgg@nvidia.com>
> > > Sent: Tuesday, February 2, 2021 10:20 PM
> > >
> > > On Wed, Jan 27, 2021 at 05:00:05PM +0200, Leon Romanovsky wrote:
> > > > + * ib_port_immutable_read() - Read rdma port's immutable data
> > > > + * @dev - IB device
> > > > + * @port - port number whose immutable data to read. It starts
> > > > + with
> > > index 1 and
> > > > + *         valid upto including rdma_end_port().
> > > > + */
> > > > +const struct ib_port_immutable*
> > > > +ib_port_immutable_read(struct ib_device *dev, unsigned int port) {
> > > > +	WARN_ON(!rdma_is_port_valid(dev, port));
> > > > +	return &dev->port_data[port].immutable; }
> > > > +EXPORT_SYMBOL(ib_port_immutable_read);
> > >
> > > Why add this function and only call it in one place?
> > >
> > A helper API from core helps
> > (a) to cut down mlx5 ib per port data structures and code around it
> > (b) it also avoids the need to maintain such driver internal data for
> > large port count (which is not done today)
> >
> I mean why not just access the pointer directly, is isn't in a private header or
> anything
Oh I see. Yes its accessible directly. An API just looked better to indicate that its RO for drivers by doing const*.

Re: [PATCH rdma-next 00/10] Various cleanups

[Jason Gunthorpe]

On Wed, Jan 27, 2021 at 05:00:00PM +0200, Leon Romanovsky wrote:
> From: Leon Romanovsky <leonro@nvidia.com>
> 
> Various simple cleanups to mlx4, mlx5 and core.
> 
> Parav Pandit (10):
>   IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev
>   IB/mlx5: Avoid calling query device for reading pkey table length
>   IB/mlx5: Improve query port for representor port
>   RDMA/core: Introduce and use API to read port immutable data
>   IB/mlx5: Use rdma_for_each_port for port iteration

These didn't want to apply, can you resend them with that other thing
fixed?

>   IB/mlx5: Support default partition key for representor port
>   IB/mlx5: Return appropriate error code instead of ENOMEM
>   IB/cm: Avoid a loop when device has 255 ports
>   IB/mlx4: Use port iterator and validation APIs
>   IB/core: Use valid port number to check link layer

I fixed the misplaced hunk and took these to for-next though

Thanks,
Jason

Re: [PATCH rdma-next 00/10] Various cleanups

[Leon Romanovsky]

On Tue, Feb 02, 2021 at 07:35:04PM -0400, Jason Gunthorpe wrote:
> On Wed, Jan 27, 2021 at 05:00:00PM +0200, Leon Romanovsky wrote:
> > From: Leon Romanovsky <leonro@nvidia.com>
> >
> > Various simple cleanups to mlx4, mlx5 and core.
> >
> > Parav Pandit (10):
> >   IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev
> >   IB/mlx5: Avoid calling query device for reading pkey table length
> >   IB/mlx5: Improve query port for representor port
> >   RDMA/core: Introduce and use API to read port immutable data
> >   IB/mlx5: Use rdma_for_each_port for port iteration
>
> These didn't want to apply, can you resend them with that other thing
> fixed?
>
> >   IB/mlx5: Support default partition key for representor port
> >   IB/mlx5: Return appropriate error code instead of ENOMEM
> >   IB/cm: Avoid a loop when device has 255 ports
> >   IB/mlx4: Use port iterator and validation APIs
> >   IB/core: Use valid port number to check link layer
>
> I fixed the misplaced hunk and took these to for-next though

Thanks, I'll resend.

>
> Thanks,
> Jason