[PATCH v3 3/3] libceph: just wait for more data to be available on the socket

From: xiubli@redhat.com
To: ceph-devel@vger.kernel.org
Cc: idryomov@gmail.com, jlayton@kernel.org, vshankar@redhat.com,
	mchangir@redhat.com, Xiubo Li <xiubli@redhat.com>
Subject: [PATCH v3 3/3] libceph: just wait for more data to be available on the socket
Date: Fri, 15 Dec 2023 08:20:34 +0800	[thread overview]
Message-ID: <20231215002034.205780-4-xiubli@redhat.com> (raw)
In-Reply-To: <20231215002034.205780-1-xiubli@redhat.com>

From: Xiubo Li <xiubli@redhat.com>

The messages from ceph maybe split into multiple socket packages
and we just need to wait for all the data to be availiable on the
sokcet.

This will add 'sr_total_resid' to record the total length for all
data items for sparse-read message and 'sr_resid_elen' to record
the current extent total length.

URL: https://tracker.ceph.com/issues/63586
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 include/linux/ceph/messenger.h |  2 ++
 net/ceph/messenger.c           |  1 +
 net/ceph/messenger_v1.c        | 21 ++++++++++++++++-----
 net/ceph/osd_client.c          |  1 +
 4 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 2eaaabbe98cb..05e9b39a58f8 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -231,10 +231,12 @@ struct ceph_msg_data {
 
 struct ceph_msg_data_cursor {
 	size_t			total_resid;	/* across all data items */
+	size_t			sr_total_resid;	/* across all data items for sparse-read */
 
 	struct ceph_msg_data	*data;		/* current data item */
 	size_t			resid;		/* bytes not yet consumed */
 	int			sr_resid;	/* residual sparse_read len */
+	int			sr_resid_elen;	/* total sparse_read elen */
 	bool			need_crc;	/* crc update needed */
 	union {
 #ifdef CONFIG_BLOCK
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 3c8b78d9c4d1..eafd592af382 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -1073,6 +1073,7 @@ void ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor,
 	cursor->total_resid = length;
 	cursor->data = msg->data;
 	cursor->sr_resid = 0;
+	cursor->sr_resid_elen = 0;
 
 	__ceph_msg_data_cursor_init(cursor);
 }
diff --git a/net/ceph/messenger_v1.c b/net/ceph/messenger_v1.c
index 4cb60bacf5f5..7425fa26e4c3 100644
--- a/net/ceph/messenger_v1.c
+++ b/net/ceph/messenger_v1.c
@@ -160,7 +160,9 @@ static size_t sizeof_footer(struct ceph_connection *con)
 static void prepare_message_data(struct ceph_msg *msg, u32 data_len)
 {
 	/* Initialize data cursor if it's not a sparse read */
-	if (!msg->sparse_read)
+	if (msg->sparse_read)
+		msg->cursor.sr_total_resid = data_len;
+	else
 		ceph_msg_data_cursor_init(&msg->cursor, msg, data_len);
 }
 
@@ -1032,18 +1034,25 @@ static int read_partial_sparse_msg_data(struct ceph_connection *con)
 	bool do_datacrc = !ceph_test_opt(from_msgr(con->msgr), NOCRC);
 	u32 crc = 0;
 	int ret = 1;
+	int len;
 
 	if (do_datacrc)
 		crc = con->in_data_crc;
 
-	do {
-		if (con->v1.in_sr_kvec.iov_base)
+	while (cursor->sr_total_resid && ret > 0) {
+		len = 0;
+		if (con->v1.in_sr_kvec.iov_base) {
 			ret = read_partial_message_chunk(con,
 							 &con->v1.in_sr_kvec,
 							 con->v1.in_sr_len,
 							 &crc);
-		else if (cursor->sr_resid > 0)
+			if (ret == 1)
+				len = con->v1.in_sr_len;
+		} else if (cursor->sr_resid > 0) {
 			ret = read_partial_sparse_msg_extent(con, &crc);
+			if (ret == 1)
+				len = cursor->sr_resid_elen;
+		}
 
 		if (ret <= 0) {
 			if (do_datacrc)
@@ -1051,11 +1060,13 @@ static int read_partial_sparse_msg_data(struct ceph_connection *con)
 			return ret;
 		}
 
+		cursor->sr_total_resid -= len;
+
 		memset(&con->v1.in_sr_kvec, 0, sizeof(con->v1.in_sr_kvec));
 		ret = con->ops->sparse_read(con, cursor,
 				(char **)&con->v1.in_sr_kvec.iov_base);
 		con->v1.in_sr_len = ret;
-	} while (ret > 0);
+	}
 
 	if (do_datacrc)
 		con->in_data_crc = crc;
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 848ef19055a0..b53b017afc0a 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -5946,6 +5946,7 @@ static int osd_sparse_read(struct ceph_connection *con,
 
 		/* send back the new length and nullify the ptr */
 		cursor->sr_resid = elen;
+		cursor->sr_resid_elen = elen;
 		ret = elen;
 		*pbuf = NULL;
 
-- 
2.43.0

