From: Aurelien Aptel <aaptel@nvidia.com>
To: linux-nvme@lists.infradead.org, netdev@vger.kernel.org,
sagi@grimberg.me, hch@lst.de, kbusch@kernel.org, axboe@fb.com,
chaitanyak@nvidia.com, davem@davemloft.net, kuba@kernel.org
Cc: Aurelien Aptel <aaptel@nvidia.com>,
aurelien.aptel@gmail.com, smalin@nvidia.com, malin1024@gmail.com,
ogerlitz@nvidia.com, yorayz@nvidia.com, borisp@nvidia.com
Subject: [PATCH v10 23/25] net/mlx5e: NVMEoTCP, async ddp invalidation
Date: Thu, 26 Jan 2023 18:21:34 +0200 [thread overview]
Message-ID: <20230126162136.13003-24-aaptel@nvidia.com> (raw)
In-Reply-To: <20230126162136.13003-1-aaptel@nvidia.com>
From: Ben Ben-Ishay <benishay@nvidia.com>
After the ULP consumed the buffers of the offloaded request, it calls the
ddp_teardown op to release the NIC mapping for them and allow the NIC to
reuse the HW contexts associated with offloading this IO. We do a
fast/async un-mapping via UMR WQE. In this case, the ULP does holds off
with completing the request towards the upper/application layers until the
HW unmapping is done.
When the corresponding CQE is received, a notification is done via the
the teardown_done ddp callback advertised by the ULP in the ddp context.
Signed-off-by: Ben Ben-Ishay <benishay@nvidia.com>
Signed-off-by: Boris Pismenny <borisp@nvidia.com>
Signed-off-by: Or Gerlitz <ogerlitz@nvidia.com>
Signed-off-by: Yoray Zack <yorayz@nvidia.com>
Signed-off-by: Aurelien Aptel <aaptel@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/en/txrx.h | 4 ++
.../mellanox/mlx5/core/en_accel/nvmeotcp.c | 66 ++++++++++++++++---
.../mellanox/mlx5/core/en_accel/nvmeotcp.h | 1 +
.../net/ethernet/mellanox/mlx5/core/en_rx.c | 6 ++
4 files changed, 67 insertions(+), 10 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
index eb5b9822190b..31ca6e2f912a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
@@ -52,6 +52,7 @@ enum mlx5e_icosq_wqe_type {
#endif
#ifdef CONFIG_MLX5_EN_NVMEOTCP
MLX5E_ICOSQ_WQE_UMR_NVMEOTCP,
+ MLX5E_ICOSQ_WQE_UMR_NVMEOTCP_INVALIDATE,
MLX5E_ICOSQ_WQE_SET_PSV_NVMEOTCP,
#endif
};
@@ -212,6 +213,9 @@ struct mlx5e_icosq_wqe_info {
struct {
struct mlx5e_nvmeotcp_queue *queue;
} nvmeotcp_q;
+ struct {
+ struct mlx5e_nvmeotcp_queue_entry *entry;
+ } nvmeotcp_qe;
#endif
};
};
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
index e7c2cf83fd20..68f2e94a015a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
@@ -142,10 +142,11 @@ build_nvmeotcp_klm_umr(struct mlx5e_nvmeotcp_queue *queue, struct mlx5e_umr_wqe
u16 ccid, int klm_entries, u32 klm_offset, u32 len,
enum wqe_type klm_type)
{
- u32 id = (klm_type == KLM_UMR) ? queue->ccid_table[ccid].klm_mkey :
- (mlx5e_tir_get_tirn(&queue->tir) << MLX5_WQE_CTRL_TIR_TIS_INDEX_SHIFT);
- u8 opc_mod = (klm_type == KLM_UMR) ? MLX5_CTRL_SEGMENT_OPC_MOD_UMR_UMR :
- MLX5_OPC_MOD_TRANSPORT_TIR_STATIC_PARAMS;
+ u32 id = (klm_type == BSF_KLM_UMR) ?
+ (mlx5e_tir_get_tirn(&queue->tir) << MLX5_WQE_CTRL_TIR_TIS_INDEX_SHIFT) :
+ queue->ccid_table[ccid].klm_mkey;
+ u8 opc_mod = (klm_type == BSF_KLM_UMR) ? MLX5_OPC_MOD_TRANSPORT_TIR_STATIC_PARAMS :
+ MLX5_CTRL_SEGMENT_OPC_MOD_UMR_UMR;
u32 ds_cnt = MLX5E_KLM_UMR_DS_CNT(ALIGN(klm_entries, MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT));
struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
@@ -158,6 +159,13 @@ build_nvmeotcp_klm_umr(struct mlx5e_nvmeotcp_queue *queue, struct mlx5e_umr_wqe
cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt);
cseg->general_id = cpu_to_be32(id);
+ if (!klm_entries) { /* this is invalidate */
+ ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
+ ucseg->flags = MLX5_UMR_INLINE;
+ mkc->status = MLX5_MKEY_STATUS_FREE;
+ return;
+ }
+
if (klm_type == KLM_UMR && !klm_offset) {
ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_XLT_OCT_SIZE |
MLX5_MKEY_MASK_LEN | MLX5_MKEY_MASK_FREE);
@@ -259,8 +267,8 @@ build_nvmeotcp_static_params(struct mlx5e_nvmeotcp_queue *queue,
static void
mlx5e_nvmeotcp_fill_wi(struct mlx5e_nvmeotcp_queue *nvmeotcp_queue,
- struct mlx5e_icosq *sq, u32 wqebbs, u16 pi,
- enum wqe_type type)
+ struct mlx5e_icosq *sq, u32 wqebbs,
+ u16 pi, u16 ccid, enum wqe_type type)
{
struct mlx5e_icosq_wqe_info *wi = &sq->db.wqe_info[pi];
@@ -272,6 +280,10 @@ mlx5e_nvmeotcp_fill_wi(struct mlx5e_nvmeotcp_queue *nvmeotcp_queue,
wi->wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_NVMEOTCP;
wi->nvmeotcp_q.queue = nvmeotcp_queue;
break;
+ case KLM_INV_UMR:
+ wi->wqe_type = MLX5E_ICOSQ_WQE_UMR_NVMEOTCP_INVALIDATE;
+ wi->nvmeotcp_qe.entry = &nvmeotcp_queue->ccid_table[ccid];
+ break;
default:
/* cases where no further action is required upon completion, such as ddp setup */
wi->wqe_type = MLX5E_ICOSQ_WQE_UMR_NVMEOTCP;
@@ -290,7 +302,7 @@ mlx5e_nvmeotcp_rx_post_static_params_wqe(struct mlx5e_nvmeotcp_queue *queue, u32
wqebbs = MLX5E_TRANSPORT_SET_STATIC_PARAMS_WQEBBS;
pi = mlx5e_icosq_get_next_pi(sq, wqebbs);
wqe = MLX5E_TRANSPORT_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
- mlx5e_nvmeotcp_fill_wi(NULL, sq, wqebbs, pi, BSF_UMR);
+ mlx5e_nvmeotcp_fill_wi(NULL, sq, wqebbs, pi, 0, BSF_UMR);
build_nvmeotcp_static_params(queue, wqe, resync_seq, queue->crc_rx);
sq->pc += wqebbs;
mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl);
@@ -307,7 +319,7 @@ mlx5e_nvmeotcp_rx_post_progress_params_wqe(struct mlx5e_nvmeotcp_queue *queue, u
wqebbs = MLX5E_NVMEOTCP_PROGRESS_PARAMS_WQEBBS;
pi = mlx5e_icosq_get_next_pi(sq, wqebbs);
wqe = MLX5E_NVMEOTCP_FETCH_PROGRESS_PARAMS_WQE(sq, pi);
- mlx5e_nvmeotcp_fill_wi(queue, sq, wqebbs, pi, SET_PSV_UMR);
+ mlx5e_nvmeotcp_fill_wi(queue, sq, wqebbs, pi, 0, SET_PSV_UMR);
build_nvmeotcp_progress_params(queue, wqe, seq);
sq->pc += wqebbs;
mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl);
@@ -330,7 +342,7 @@ post_klm_wqe(struct mlx5e_nvmeotcp_queue *queue,
wqebbs = DIV_ROUND_UP(wqe_sz, MLX5_SEND_WQE_BB);
pi = mlx5e_icosq_get_next_pi(sq, wqebbs);
wqe = MLX5E_NVMEOTCP_FETCH_KLM_WQE(sq, pi);
- mlx5e_nvmeotcp_fill_wi(queue, sq, wqebbs, pi, wqe_type);
+ mlx5e_nvmeotcp_fill_wi(queue, sq, wqebbs, pi, ccid, wqe_type);
build_nvmeotcp_klm_umr(queue, wqe, ccid, cur_klm_entries, klm_offset,
klm_length, wqe_type);
sq->pc += wqebbs;
@@ -345,7 +357,10 @@ mlx5e_nvmeotcp_post_klm_wqe(struct mlx5e_nvmeotcp_queue *queue, enum wqe_type wq
struct mlx5e_icosq *sq = &queue->sq;
u32 klm_offset = 0, wqes, i;
- wqes = DIV_ROUND_UP(klm_length, queue->max_klms_per_wqe);
+ if (wqe_type == KLM_INV_UMR)
+ wqes = 1;
+ else
+ wqes = DIV_ROUND_UP(klm_length, queue->max_klms_per_wqe);
spin_lock_bh(&queue->sq_lock);
@@ -842,12 +857,43 @@ void mlx5e_nvmeotcp_ctx_complete(struct mlx5e_icosq_wqe_info *wi)
complete(&queue->static_params_done);
}
+void mlx5e_nvmeotcp_ddp_inv_done(struct mlx5e_icosq_wqe_info *wi)
+{
+ struct mlx5e_nvmeotcp_queue_entry *q_entry = wi->nvmeotcp_qe.entry;
+ struct mlx5e_nvmeotcp_queue *queue = q_entry->queue;
+ struct mlx5_core_dev *mdev = queue->priv->mdev;
+ struct ulp_ddp_io *ddp = q_entry->ddp;
+ const struct ulp_ddp_ulp_ops *ulp_ops;
+
+ dma_unmap_sg(mdev->device, ddp->sg_table.sgl,
+ q_entry->sgl_length, DMA_FROM_DEVICE);
+
+ q_entry->sgl_length = 0;
+
+ ulp_ops = inet_csk(queue->sk)->icsk_ulp_ddp_ops;
+ if (ulp_ops && ulp_ops->ddp_teardown_done)
+ ulp_ops->ddp_teardown_done(q_entry->ddp_ctx);
+}
+
static void
mlx5e_nvmeotcp_ddp_teardown(struct net_device *netdev,
struct sock *sk,
struct ulp_ddp_io *ddp,
void *ddp_ctx)
{
+ struct mlx5e_nvmeotcp_queue_entry *q_entry;
+ struct mlx5e_nvmeotcp_queue *queue;
+
+ queue = container_of(ulp_ddp_get_ctx(sk), struct mlx5e_nvmeotcp_queue, ulp_ddp_ctx);
+ q_entry = &queue->ccid_table[ddp->command_id];
+ WARN_ONCE(q_entry->sgl_length == 0,
+ "Invalidation of empty sgl (CID 0x%x, queue 0x%x)\n",
+ ddp->command_id, queue->id);
+
+ q_entry->ddp_ctx = ddp_ctx;
+ q_entry->queue = queue;
+
+ mlx5e_nvmeotcp_post_klm_wqe(queue, KLM_INV_UMR, ddp->command_id, 0);
}
static void
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h
index 555f3ed7e2e2..a5cfd9e31be7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.h
@@ -109,6 +109,7 @@ void mlx5e_nvmeotcp_cleanup(struct mlx5e_priv *priv);
struct mlx5e_nvmeotcp_queue *
mlx5e_nvmeotcp_get_queue(struct mlx5e_nvmeotcp *nvmeotcp, int id);
void mlx5e_nvmeotcp_put_queue(struct mlx5e_nvmeotcp_queue *queue);
+void mlx5e_nvmeotcp_ddp_inv_done(struct mlx5e_icosq_wqe_info *wi);
void mlx5e_nvmeotcp_ctx_complete(struct mlx5e_icosq_wqe_info *wi);
static inline void mlx5e_nvmeotcp_init_rx(struct mlx5e_priv *priv) {}
void mlx5e_nvmeotcp_cleanup_rx(struct mlx5e_priv *priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 3ae2b35155a8..9b9c3603a00b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -891,6 +891,9 @@ void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq)
break;
#endif
#ifdef CONFIG_MLX5_EN_NVMEOTCP
+ case MLX5E_ICOSQ_WQE_UMR_NVMEOTCP_INVALIDATE:
+ mlx5e_nvmeotcp_ddp_inv_done(wi);
+ break;
case MLX5E_ICOSQ_WQE_SET_PSV_NVMEOTCP:
mlx5e_nvmeotcp_ctx_complete(wi);
break;
@@ -996,6 +999,9 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq, int budget)
#ifdef CONFIG_MLX5_EN_NVMEOTCP
case MLX5E_ICOSQ_WQE_UMR_NVMEOTCP:
break;
+ case MLX5E_ICOSQ_WQE_UMR_NVMEOTCP_INVALIDATE:
+ mlx5e_nvmeotcp_ddp_inv_done(wi);
+ break;
case MLX5E_ICOSQ_WQE_SET_PSV_NVMEOTCP:
mlx5e_nvmeotcp_ctx_complete(wi);
break;
--
2.31.1
next prev parent reply other threads:[~2023-01-26 16:26 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-01-26 16:21 [PATCH v10 00/25] nvme-tcp receive offloads Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 01/25] net: Introduce direct data placement tcp offload Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 02/25] net/ethtool: add new stringset ETH_SS_ULP_DDP_{CAPS,STATS} Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 03/25] net/ethtool: add ULP_DDP_{GET,SET} operations for caps and stats Aurelien Aptel
2023-02-01 4:53 ` Jakub Kicinski
2023-02-01 19:18 ` Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 04/25] Documentation: document netlink ULP_DDP_GET/SET messages Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 05/25] iov_iter: skip copy if src == dst for direct data placement Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 06/25] net/tls,core: export get_netdev_for_sock Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 07/25] nvme-tcp: Add DDP offload control path Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 08/25] nvme-tcp: Add DDP data-path Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 09/25] nvme-tcp: RX DDGST offload Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 10/25] nvme-tcp: Deal with netdevice DOWN events Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 11/25] nvme-tcp: Add modparam to control the ULP offload enablement Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 12/25] Documentation: add ULP DDP offload documentation Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 13/25] net/mlx5e: Rename from tls to transport static params Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 14/25] net/mlx5e: Refactor ico sq polling to get budget Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 15/25] net/mlx5e: Have mdev pointer directly on the icosq structure Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 16/25] net/mlx5e: Refactor doorbell function to allow avoiding a completion Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 17/25] net/mlx5: Add NVMEoTCP caps, HW bits, 128B CQE and enumerations Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 18/25] net/mlx5e: NVMEoTCP, offload initialization Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 19/25] net/mlx5e: TCP flow steering for nvme-tcp acceleration Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 20/25] net/mlx5e: NVMEoTCP, use KLM UMRs for buffer registration Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 21/25] net/mlx5e: NVMEoTCP, queue init/teardown Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 22/25] net/mlx5e: NVMEoTCP, ddp setup and resync Aurelien Aptel
2023-01-26 16:21 ` Aurelien Aptel [this message]
2023-01-26 16:21 ` [PATCH v10 24/25] net/mlx5e: NVMEoTCP, data-path for DDP+DDGST offload Aurelien Aptel
2023-01-26 16:21 ` [PATCH v10 25/25] net/mlx5e: NVMEoTCP, statistics Aurelien Aptel
2023-01-30 6:45 ` [PATCH v10 00/25] nvme-tcp receive offloads Christoph Hellwig
2023-02-06 5:12 ` Chaitanya Kulkarni
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230126162136.13003-24-aaptel@nvidia.com \
--to=aaptel@nvidia.com \
--cc=aurelien.aptel@gmail.com \
--cc=axboe@fb.com \
--cc=borisp@nvidia.com \
--cc=chaitanyak@nvidia.com \
--cc=davem@davemloft.net \
--cc=hch@lst.de \
--cc=kbusch@kernel.org \
--cc=kuba@kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=malin1024@gmail.com \
--cc=netdev@vger.kernel.org \
--cc=ogerlitz@nvidia.com \
--cc=sagi@grimberg.me \
--cc=smalin@nvidia.com \
--cc=yorayz@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.