From: Leon Romanovsky <leon@kernel.org>
To: Doug Ledford <dledford@redhat.com>, Jason Gunthorpe <jgg@nvidia.com>
Cc: Parav Pandit <parav@nvidia.com>, linux-rdma@vger.kernel.org
Subject: [PATCH rdma-next 05/10] RDMA/core: Introduce and use API to read port immutable data
Date: Wed, 27 Jan 2021 17:00:05 +0200 [thread overview]
Message-ID: <20210127150010.1876121-6-leon@kernel.org> (raw)
In-Reply-To: <20210127150010.1876121-1-leon@kernel.org>
From: Parav Pandit <parav@nvidia.com>
Currently mlx5 driver caches port GID table length for 2 ports.
It is also cached by IB core as port immutable data.
When mlx5 representor ports are present, which are usually more than 2,
invalid access to port_caps array can happen while validating the GID
table length which is only for 2 ports.
To avoid this, take help of the IB cores port immutable data by exposing
an API to read the port immutable fields.
Remove mlx5 driver's internal cache, thereby reduce code and data.
Signed-off-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
drivers/infiniband/core/device.c | 14 +++++++
drivers/infiniband/hw/mlx5/main.c | 55 +---------------------------
drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 -
drivers/infiniband/hw/mlx5/qp.c | 8 ++--
include/rdma/ib_verbs.h | 3 ++
5 files changed, 23 insertions(+), 58 deletions(-)
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index c895d7bfa512..051c018fb73c 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -848,6 +848,20 @@ static int setup_port_data(struct ib_device *device)
return 0;
}
+/**
+ * ib_port_immutable_read() - Read rdma port's immutable data
+ * @dev - IB device
+ * @port - port number whose immutable data to read. It starts with index 1 and
+ * valid upto including rdma_end_port().
+ */
+const struct ib_port_immutable*
+ib_port_immutable_read(struct ib_device *dev, unsigned int port)
+{
+ WARN_ON(!rdma_is_port_valid(dev, port));
+ return &dev->port_data[port].immutable;
+}
+EXPORT_SYMBOL(ib_port_immutable_read);
+
void ib_get_device_fw_str(struct ib_device *dev, char *str)
{
if (dev->ops.get_dev_fw_str)
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 5a7f8fa2f452..742e565309f8 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2988,41 +2988,6 @@ static void get_ext_port_caps(struct mlx5_ib_dev *dev)
mlx5_query_ext_port_caps(dev, port);
}
-static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
-{
- struct ib_port_attr *pprops = NULL;
- int err = -ENOMEM;
-
- pprops = kzalloc(sizeof(*pprops), GFP_KERNEL);
- if (!pprops)
- goto out;
-
- err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
- if (err) {
- mlx5_ib_warn(dev, "query_port %d failed %d\n",
- port, err);
- goto out;
- }
-
- dev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
- mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
- port, dev->pkey_table_len, pprops->gid_tbl_len);
-
-out:
- kfree(pprops);
- return err;
-}
-
-static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
-{
- /* For representors use port 1, is this is the only native
- * port
- */
- if (dev->is_rep)
- return __get_port_caps(dev, 1);
- return __get_port_caps(dev, port);
-}
-
static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
{
switch (umr_fence_cap) {
@@ -3496,10 +3461,6 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
if (err)
goto unbind;
- err = get_port_caps(ibdev, mlx5_core_native_port_num(mpi->mdev));
- if (err)
- goto unbind;
-
err = mlx5_add_netdev_notifier(ibdev, port_num);
if (err) {
mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
@@ -3577,11 +3538,9 @@ static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
break;
}
}
- if (!bound) {
- get_port_caps(dev, i + 1);
+ if (!bound)
mlx5_ib_dbg(dev, "no free port found for port %d\n",
i + 1);
- }
}
list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
@@ -3964,18 +3923,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
if (err)
goto err_mp;
- if (!mlx5_core_mp_enabled(mdev)) {
- for (i = 1; i <= dev->num_ports; i++) {
- err = get_port_caps(dev, i);
- if (err)
- break;
- }
- } else {
- err = get_port_caps(dev, mlx5_core_native_port_num(mdev));
- }
- if (err)
- goto err_mp;
-
err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
if (err)
goto err_mp;
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index c0c5e0043b3e..69c45adb2518 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1034,7 +1034,6 @@ struct mlx5_var_table {
};
struct mlx5_port_caps {
- int gid_table_len;
bool has_smi;
u8 ext_port_cap;
};
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index b65720a05a18..abdb73fdbec2 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -3177,11 +3177,13 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
alt ? attr->alt_pkey_index : attr->pkey_index);
if (ah_flags & IB_AH_GRH) {
- if (grh->sgid_index >=
- dev->port_caps[port - 1].gid_table_len) {
+ const struct ib_port_immutable *immutable;
+
+ immutable = ib_port_immutable_read(&dev->ib_dev, port);
+ if (grh->sgid_index >= immutable->gid_tbl_len) {
pr_err("sgid_index (%u) too large. max is %d\n",
grh->sgid_index,
- dev->port_caps[port - 1].gid_table_len);
+ immutable->gid_tbl_len);
return -EINVAL;
}
}
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 62e574c50555..ca28fca5736b 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -4674,4 +4674,7 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)
return (u32)(v & IB_GRH_FLOWLABEL_MASK);
}
+
+const struct ib_port_immutable*
+ib_port_immutable_read(struct ib_device *dev, unsigned int port);
#endif /* IB_VERBS_H */
--
2.29.2
next prev parent reply other threads:[~2021-01-27 15:35 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-27 15:00 [PATCH rdma-next 00/10] Various cleanups Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 01/10] IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 02/10] IB/mlx5: Avoid calling query device for reading pkey table length Leon Romanovsky
2021-02-01 10:19 ` Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 03/10] IB/mlx5: Support default partition key for representor port Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 04/10] IB/mlx5: Improve query port " Leon Romanovsky
2021-01-27 15:00 ` Leon Romanovsky [this message]
2021-02-02 16:50 ` [PATCH rdma-next 05/10] RDMA/core: Introduce and use API to read port immutable data Jason Gunthorpe
2021-02-02 18:05 ` Parav Pandit
2021-02-02 18:14 ` Jason Gunthorpe
2021-02-02 19:00 ` Parav Pandit
2021-01-27 15:00 ` [PATCH rdma-next 06/10] IB/mlx5: Use rdma_for_each_port for port iteration Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 07/10] IB/mlx5: Return appropriate error code instead of ENOMEM Leon Romanovsky
2021-02-02 15:21 ` Jason Gunthorpe
2021-02-02 16:01 ` Parav Pandit
2021-02-02 18:57 ` Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 08/10] IB/cm: Avoid a loop when device has 255 ports Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 09/10] IB/mlx4: Use port iterator and validation APIs Leon Romanovsky
2021-01-27 15:00 ` [PATCH rdma-next 10/10] IB/core: Use valid port number to check link layer Leon Romanovsky
2021-02-02 23:35 ` [PATCH rdma-next 00/10] Various cleanups Jason Gunthorpe
2021-02-03 5:47 ` Leon Romanovsky
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210127150010.1876121-6-leon@kernel.org \
--to=leon@kernel.org \
--cc=dledford@redhat.com \
--cc=jgg@nvidia.com \
--cc=linux-rdma@vger.kernel.org \
--cc=parav@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.