From: Sagi Grimberg <sagig@mellanox.com>
To: linux-rdma@vger.kernel.org, target-devel@vger.kernel.org
Cc: "Nicholas A. Bellinger" <nab@linux-iscsi.org>,
Or Gerlitz <ogerlitz@mellanox.com>,
Steve Wise <swise@opengridcomputing.com>,
Jenny Derzhavetz <jennyf@mellanox.com>
Subject: [PATCH for-next 08/10] iser-target: Support the remote invalidation exception
Date: Mon, 16 Nov 2015 18:37:39 +0200 [thread overview]
Message-ID: <1447691861-3796-9-git-send-email-sagig@mellanox.com> (raw)
In-Reply-To: <1447691861-3796-1-git-send-email-sagig@mellanox.com>
From: Jenny Derzhavetz <jennyf@mellanox.com>
We'll use remote invalidate, according to negotiation result
during connection establishment. If the initiator declared that
it supports the remote invalidate exception then the target will
use IB_WR_SEND_WITH_INV with the correct rkey for the response.
Signed-off-by: Jenny Derzhavetz <jennyf@mellanox.com>
Signed-off-by: Sagi Grimberg <sagig@mellanox.com>
---
drivers/infiniband/ulp/isert/ib_isert.c | 39 +++++++++++++++++++++++++++------
drivers/infiniband/ulp/isert/ib_isert.h | 2 ++
2 files changed, 34 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 667238f6253f..035fd12ec7ae 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -679,6 +679,25 @@ out_login_buf:
return ret;
}
+static void
+isert_set_nego_params(struct isert_conn *isert_conn,
+ struct rdma_conn_param *param)
+{
+ struct isert_device *device = isert_conn->device;
+
+ /* Set max inflight RDMA READ requests */
+ isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
+ device->dev_attr.max_qp_init_rd_atom);
+ isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
+
+ if (param->private_data) {
+ u8 flags = *(u8 *)param->private_data;
+
+ isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP);
+ isert_info("Using remote invalidation\n");
+ }
+}
+
static int
isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
@@ -717,11 +736,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
}
isert_conn->device = device;
- /* Set max inflight RDMA READ requests */
- isert_conn->initiator_depth = min_t(u8,
- event->param.conn.initiator_depth,
- device->dev_attr.max_qp_init_rd_atom);
- isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
+ isert_set_nego_params(isert_conn, &event->param.conn);
ret = isert_conn_setup_qp(isert_conn, cma_id);
if (ret)
@@ -1100,7 +1115,14 @@ isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;
- send_wr->opcode = IB_WR_SEND;
+
+ if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
+ send_wr->opcode = IB_WR_SEND_WITH_INV;
+ send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
+ } else {
+ send_wr->opcode = IB_WR_SEND;
+ }
+
send_wr->sg_list = &tx_desc->tx_sg[0];
send_wr->num_sge = isert_cmd->tx_desc.num_sge;
send_wr->send_flags = IB_SEND_SIGNALED;
@@ -1489,6 +1511,7 @@ isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
isert_cmd->read_va = read_va;
isert_cmd->write_stag = write_stag;
isert_cmd->write_va = write_va;
+ isert_cmd->inv_rkey = read_stag ? read_stag : write_stag;
ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
rx_desc, (unsigned char *)hdr);
@@ -3107,7 +3130,9 @@ isert_rdma_accept(struct isert_conn *isert_conn)
cp.rnr_retry_count = 7;
memset(&rsp_hdr, 0, sizeof(rsp_hdr));
- rsp_hdr.flags = (ISERT_ZBVA_NOT_USED | ISERT_SEND_W_INV_NOT_USED);
+ rsp_hdr.flags = ISERT_ZBVA_NOT_USED;
+ if (!isert_conn->snd_w_inv)
+ rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED;
cp.private_data = (void *)&rsp_hdr;
cp.private_data_len = sizeof(rsp_hdr);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index 00edd5b68ca8..b23086d3d08b 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -162,6 +162,7 @@ struct isert_cmd {
uint32_t write_stag;
uint64_t read_va;
uint64_t write_va;
+ uint32_t inv_rkey;
u64 pdu_buf_dma;
u32 pdu_buf_len;
struct isert_conn *conn;
@@ -209,6 +210,7 @@ struct isert_conn {
struct work_struct release_work;
struct ib_recv_wr beacon;
bool logout_posted;
+ bool snd_w_inv;
};
#define ISERT_MAX_CQ 64
--
1.8.4.3
next prev parent reply other threads:[~2015-11-16 16:37 UTC|newest]
Thread overview: 74+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-11-16 16:37 [PATCH for-next 00/10] iSER support for remote invalidate Sagi Grimberg
2015-11-16 16:37 ` [PATCH for-next 01/10] IB/iser: Fix module init not cleaning up on error flow Sagi Grimberg
[not found] ` <1447691861-3796-2-git-send-email-sagig-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 6:53 ` Or Gerlitz
[not found] ` <564ACEF7.8030809-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:31 ` Sagi Grimberg
2015-11-17 8:56 ` Christoph Hellwig
2015-11-16 16:37 ` [PATCH for-next 02/10] IB/iser: Default to fastreg instead of fmr Sagi Grimberg
[not found] ` <1447691861-3796-3-git-send-email-sagig-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 7:41 ` Or Gerlitz
2015-11-17 9:35 ` Sagi Grimberg
2015-11-17 10:09 ` Or Gerlitz
2015-11-17 10:46 ` Sagi Grimberg
2015-11-17 8:57 ` Christoph Hellwig
2015-11-17 9:35 ` Sagi Grimberg
[not found] ` <1447691861-3796-1-git-send-email-sagig-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-16 16:37 ` [PATCH for-next 03/10] IB/iser: Don't register memory for all immediatedata writes Sagi Grimberg
2015-11-17 7:47 ` Or Gerlitz
[not found] ` <564ADB7D.20806-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:41 ` Sagi Grimberg
[not found] ` <564AF653.6060401-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>
2015-11-17 10:11 ` Or Gerlitz
[not found] ` <564AFD4A.4030300-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 10:15 ` Sagi Grimberg
2015-11-23 18:13 ` Jason Gunthorpe
2015-11-24 10:04 ` Sagi Grimberg
2015-11-24 18:15 ` Jason Gunthorpe
2015-11-16 16:37 ` [PATCH for-next 07/10] iser-target: Declare correct flags when accepting a connection Sagi Grimberg
[not found] ` <1447691861-3796-8-git-send-email-sagig-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 7:59 ` Or Gerlitz
[not found] ` <564ADE63.3030901-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:48 ` Sagi Grimberg
[not found] ` <564AF808.7010404-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>
2015-11-17 10:13 ` Or Gerlitz
2015-11-17 9:03 ` Christoph Hellwig
2015-11-17 9:38 ` Sagi Grimberg
2015-11-16 16:37 ` [PATCH for-next 09/10] IB/iser: Increment the rkey when registering and not when invalidating Sagi Grimberg
2015-11-17 8:13 ` Or Gerlitz
2015-11-17 8:10 ` [PATCH for-next 00/10] iSER support for remote invalidate Or Gerlitz
[not found] ` <564AE0E8.7030705-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:56 ` Sagi Grimberg
2015-11-16 16:37 ` [PATCH for-next 04/10] IB/iser: set intuitive values for mr_valid Sagi Grimberg
2015-11-17 7:43 ` Or Gerlitz
[not found] ` <564ADAB5.3080208-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:43 ` Sagi Grimberg
2015-11-16 16:37 ` [PATCH for-next 05/10] iser: Have initiator and target to share protocol structures and definitions Sagi Grimberg
2015-11-17 8:59 ` Christoph Hellwig
[not found] ` <20151117085933.GC19578-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>
2015-11-17 9:58 ` Sagi Grimberg
[not found] ` <564AFA42.5060808-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>
2015-11-17 10:07 ` Christoph Hellwig
[not found] ` <1447691861-3796-6-git-send-email-sagig-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-19 7:20 ` Or Gerlitz
[not found] ` <564D7835.4010407-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-19 8:40 ` Sagi Grimberg
2015-11-16 16:37 ` [PATCH for-next 06/10] iser-target: Remove unused file iser_proto.h Sagi Grimberg
2015-11-17 7:57 ` Or Gerlitz
2015-11-17 9:45 ` Sagi Grimberg
2015-11-17 9:00 ` Christoph Hellwig
[not found] ` <20151117090046.GD19578-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>
2015-11-17 9:46 ` Sagi Grimberg
2015-11-16 16:37 ` Sagi Grimberg [this message]
[not found] ` <1447691861-3796-9-git-send-email-sagig-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 8:09 ` [PATCH for-next 08/10] iser-target: Support the remote invalidation exception Or Gerlitz
[not found] ` <564AE0C6.2030203-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:50 ` Sagi Grimberg
[not found] ` <564AF857.5090701-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>
2015-11-17 10:05 ` Or Gerlitz
2015-11-17 10:14 ` Christoph Hellwig
2015-11-17 10:16 ` Sagi Grimberg
2015-11-19 7:25 ` Or Gerlitz
2015-11-16 16:37 ` [PATCH for-next 10/10] IB/iser: " Sagi Grimberg
[not found] ` <1447691861-3796-11-git-send-email-sagig-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 8:03 ` Or Gerlitz
[not found] ` <564ADF68.1050408-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:50 ` Sagi Grimberg
2015-11-17 8:05 ` Or Gerlitz
2015-11-17 9:53 ` Sagi Grimberg
2015-11-17 10:04 ` Or Gerlitz
2015-11-17 10:15 ` Christoph Hellwig
[not found] ` <564AFB93.9010602-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 10:26 ` Sagi Grimberg
[not found] ` <564B00C3.1030506-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>
2015-11-17 16:53 ` Or Gerlitz
2015-11-18 11:38 ` Sagi Grimberg
2015-11-18 13:33 ` Or Gerlitz
2015-11-18 13:52 ` Christoph Hellwig
[not found] ` <20151118135237.GA3214-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>
2015-11-18 13:58 ` Or Gerlitz
[not found] ` <CAJ3xEMiBwowDFPEVqjKr-DMeDiWih8qbwshP2QbaH_56aNsRDg-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2015-11-18 14:10 ` Christoph Hellwig
[not found] ` <20151118141023.GA10198-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>
2015-11-19 7:12 ` Or Gerlitz
[not found] ` <564D7653.8070101-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-19 9:15 ` Sagi Grimberg
2015-11-19 10:01 ` Christoph Hellwig
[not found] ` <20151119100106.GA24120-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>
2015-11-19 11:17 ` Or Gerlitz
[not found] ` <CAJ3xEMhxOQ-HXKiXuBOKNet0VeDbNQingphFvzUXBiLcDjCmNQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2015-11-18 14:16 ` Sagi Grimberg
2015-11-19 7:16 ` Or Gerlitz
2015-11-17 8:08 ` Or Gerlitz
[not found] ` <564AE06C.9020504-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2015-11-17 9:55 ` Sagi Grimberg
2015-11-17 9:35 ` [PATCH for-next 00/10] iSER support for remote invalidate Christoph Hellwig
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1447691861-3796-9-git-send-email-sagig@mellanox.com \
--to=sagig@mellanox.com \
--cc=jennyf@mellanox.com \
--cc=linux-rdma@vger.kernel.org \
--cc=nab@linux-iscsi.org \
--cc=ogerlitz@mellanox.com \
--cc=swise@opengridcomputing.com \
--cc=target-devel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.