[PATCH 0/4] libceph: abstract setting message data info

[PATCH 0/4] libceph: abstract setting message data info

[Alex Elder]

This series makes the fields related to the data portion of
a ceph message not get manipulated by code outside the ceph
messenger.  It implements some interface functions that can
be used to assign data-related fields.  Doing this will allow
the way message data is managed to be changed independent
of the users of the messenger module.

					-Alex

[PATCH 1/4] libceph: distinguish page array and pagelist count
[PATCH 2/4] libceph: set page alignment in start_request()
[PATCH 3/4] libceph: isolate message page field manipulation
[PATCH 4/4] libceph: isolate other message data fields

[PATCH 1/4] libceph: distinguish page array and pagelist count

[Alex Elder]

Use distinct fields for tracking the number of pages in a message's
page array and in a message's page list.  Currently only one or the
other is used at a time, but that will be changing soon.

Signed-off-by: Alex Elder <elder@inktank.com>
---
 fs/ceph/mds_client.c           |    4 ++--
 include/linux/ceph/messenger.h |    3 ++-
 net/ceph/messenger.c           |   14 ++++++++------
 net/ceph/osd_client.c          |    4 ++--
 4 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index d958420..44435c2 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -1719,7 +1719,7 @@ static struct ceph_msg
*create_request_message(struct ceph_mds_client *mdsc,
 	msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);

 	msg->pages = req->r_pages;
-	msg->nr_pages = req->r_num_pages;
+	msg->page_count = req->r_num_pages;
 	msg->hdr.data_len = cpu_to_le32(req->r_data_len);
 	msg->hdr.data_off = cpu_to_le16(0);

@@ -2600,10 +2600,10 @@ static void send_mds_reconnect(struct
ceph_mds_client *mdsc,
 	}

 	reply->pagelist = pagelist;
+	reply->pagelist_count = calc_pages_for(0, pagelist->length);
 	if (recon_state.flock)
 		reply->hdr.version = cpu_to_le16(2);
 	reply->hdr.data_len = cpu_to_le32(pagelist->length);
-	reply->nr_pages = calc_pages_for(0, pagelist->length);
 	ceph_con_send(&session->s_con, reply);

 	mutex_unlock(&session->s_mutex);
diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 8297288..1b08349 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -75,9 +75,10 @@ struct ceph_msg {
 	struct kvec front;              /* unaligned blobs of message */
 	struct ceph_buffer *middle;
 	struct page **pages;            /* data payload.  NOT OWNER. */
-	unsigned nr_pages;              /* size of page array */
+	unsigned page_count;		/* size of page array */
 	unsigned page_alignment;        /* io offset in first page */
 	struct ceph_pagelist *pagelist; /* instead of pages */
+	unsigned int pagelist_count;	/* number of pages in pagelist */

 	struct ceph_connection *con;
 	struct list_head list_head;
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index c06f940..9d8abb0 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -813,7 +813,7 @@ static void prepare_write_message(struct
ceph_connection *con)
 	     m, con->out_seq, le16_to_cpu(m->hdr.type),
 	     le32_to_cpu(m->hdr.front_len), le32_to_cpu(m->hdr.middle_len),
 	     le32_to_cpu(m->hdr.data_len),
-	     m->nr_pages);
+	     m->page_count);
 	BUG_ON(le32_to_cpu(m->hdr.front_len) != m->front.iov_len);

 	/* tag + hdr + front + middle */
@@ -1072,7 +1072,7 @@ static int write_partial_msg_pages(struct
ceph_connection *con)
 	const size_t trail_off = data_len - trail_len;

 	dout("write_partial_msg_pages %p msg %p page %d/%d offset %d\n",
-	     con, msg, con->out_msg_pos.page, msg->nr_pages,
+	     con, msg, con->out_msg_pos.page, msg->page_count,
 	     con->out_msg_pos.page_pos);

 	/*
@@ -2715,9 +2715,10 @@ struct ceph_msg *ceph_msg_new(int type, int
front_len, gfp_t flags,
 	m->middle = NULL;

 	/* data */
-	m->nr_pages = 0;
+	m->page_count = 0;
 	m->page_alignment = 0;
 	m->pages = NULL;
+	m->pagelist_count = 0;
 	m->pagelist = NULL;
 #ifdef	CONFIG_BLOCK
 	m->bio = NULL;
@@ -2890,13 +2891,14 @@ void ceph_msg_last_put(struct kref *kref)
 		ceph_buffer_put(m->middle);
 		m->middle = NULL;
 	}
-	m->nr_pages = 0;
+	m->page_count = 0;
 	m->pages = NULL;

 	if (m->pagelist) {
 		ceph_pagelist_release(m->pagelist);
 		kfree(m->pagelist);
 		m->pagelist = NULL;
+		m->pagelist_count = 0;
 	}

 	m->trail = NULL;
@@ -2910,8 +2912,8 @@ EXPORT_SYMBOL(ceph_msg_last_put);

 void ceph_msg_dump(struct ceph_msg *msg)
 {
-	pr_debug("msg_dump %p (front_max %d nr_pages %d)\n", msg,
-		 msg->front_max, msg->nr_pages);
+	pr_debug("msg_dump %p (front_max %d page_count %d)\n", msg,
+		 msg->front_max, msg->page_count);
 	print_hex_dump(KERN_DEBUG, "header: ",
 		       DUMP_PREFIX_OFFSET, 16, 1,
 		       &msg->hdr, sizeof(msg->hdr), true);
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 980911e..0a4d4a0 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1631,7 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
 	int rc = 0;

 	req->r_request->pages = req->r_pages;
-	req->r_request->nr_pages = req->r_num_pages;
+	req->r_request->page_count = req->r_num_pages;
 #ifdef CONFIG_BLOCK
 	req->r_request->bio = req->r_bio;
 #endif
@@ -1982,7 +1982,7 @@ static struct ceph_msg *get_reply(struct
ceph_connection *con,
 			goto out;
 		}
 		m->pages = req->r_pages;
-		m->nr_pages = req->r_num_pages;
+		m->page_count = req->r_num_pages;
 		m->page_alignment = req->r_page_alignment;
 #ifdef CONFIG_BLOCK
 		m->bio = req->r_bio;
-- 
1.7.9.5

[PATCH 2/4] libceph: set page alignment in start_request()

[Alex Elder]

The page alignment field for a request is currently set in
ceph_osdc_build_request().  It's not needed at that point
nor do either of its callers need that value assigned at
any point before they call ceph_osdc_start_request().

So move that assignment into ceph_osdc_start_request().

Signed-off-by: Alex Elder <elder@inktank.com>
---
 net/ceph/osd_client.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 0a4d4a0..1bb2b59 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -369,7 +369,6 @@ void ceph_osdc_build_request(struct ceph_osd_request
*req,
 		data_len += len;
 	}
 	req->r_request->hdr.data_len = cpu_to_le32(data_len);
-	req->r_request->page_alignment = req->r_page_alignment;

 	BUG_ON(p > msg->front.iov_base + msg->front.iov_len);
 	msg_size = p - msg->front.iov_base;
@@ -1632,6 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,

 	req->r_request->pages = req->r_pages;
 	req->r_request->page_count = req->r_num_pages;
+	req->r_request->page_alignment = req->r_page_alignment;
 #ifdef CONFIG_BLOCK
 	req->r_request->bio = req->r_bio;
 #endif
-- 
1.7.9.5

[PATCH 3/4] libceph: isolate message page field manipulation

[Alex Elder]

Define a function ceph_msg_data_set_pages(), which more clearly
abstracts the assignment page-related fields for data in a ceph
message structure.  These fields should never be set more than once
(add BUG_ON() calls to guarantee that).  Use this new function in
the osd client and mds client.

Rearrange the field order in a ceph_msg structure to group those
that are used to define the possible data payloads.

Signed-off-by: Alex Elder <elder@inktank.com>
---
 fs/ceph/mds_client.c           |    4 ++--
 include/linux/ceph/messenger.h |   22 +++++++++++++---------
 net/ceph/messenger.c           |   12 ++++++++++++
 net/ceph/osd_client.c          |   11 +++++------
 4 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index 44435c2..d8842a1 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -1718,8 +1718,8 @@ static struct ceph_msg
*create_request_message(struct ceph_mds_client *mdsc,
 	msg->front.iov_len = p - msg->front.iov_base;
 	msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);

-	msg->pages = req->r_pages;
-	msg->page_count = req->r_num_pages;
+	ceph_msg_data_set_pages(msg, req->r_pages, req->r_num_pages, 0);
+
 	msg->hdr.data_len = cpu_to_le32(req->r_data_len);
 	msg->hdr.data_off = cpu_to_le16(0);

diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 1b08349..eeb28a0 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -74,22 +74,23 @@ struct ceph_msg {
 	struct ceph_msg_footer footer;	/* footer */
 	struct kvec front;              /* unaligned blobs of message */
 	struct ceph_buffer *middle;
-	struct page **pages;            /* data payload.  NOT OWNER. */
-	unsigned page_count;		/* size of page array */
-	unsigned page_alignment;        /* io offset in first page */
+
+	struct page **pages;		/* data payload.  NOT OWNER. */
+	unsigned int page_alignment;	/* io offset in first page */
+	unsigned int page_count;	/* # pages in array or list */
 	struct ceph_pagelist *pagelist; /* instead of pages */
 	unsigned int pagelist_count;	/* number of pages in pagelist */
-
-	struct ceph_connection *con;
-	struct list_head list_head;
-
-	struct kref kref;
 #ifdef CONFIG_BLOCK
+	unsigned int bio_seg;		/* current bio segment */
 	struct bio  *bio;		/* instead of pages/pagelist */
 	struct bio  *bio_iter;		/* bio iterator */
-	unsigned int bio_seg;		/* current bio segment */
 #endif /* CONFIG_BLOCK */
 	struct ceph_pagelist *trail;	/* the trailing part of the data */
+
+	struct ceph_connection *con;
+	struct list_head list_head;
+
+	struct kref kref;
 	bool front_is_vmalloc;
 	bool more_to_follow;
 	bool needs_out_seq;
@@ -219,6 +220,9 @@ extern void ceph_msg_revoke_incoming(struct ceph_msg
*msg);

 extern void ceph_con_keepalive(struct ceph_connection *con);

+extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page
**pages,
+				unsigned int page_count, size_t alignment);
+
 extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
 				     bool can_fail);
 extern void ceph_msg_kfree(struct ceph_msg *m);
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 9d8abb0..7328e1b 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -2674,6 +2674,18 @@ void ceph_con_keepalive(struct ceph_connection *con)
 }
 EXPORT_SYMBOL(ceph_con_keepalive);

+void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page **pages,
+		unsigned int page_count, size_t alignment)
+{
+	BUG_ON(msg->pages);
+	BUG_ON(msg->page_count);
+	BUG_ON(alignment >= PAGE_SIZE);
+
+	msg->pages = pages;
+	msg->page_count = page_count;
+	msg->page_alignment = alignment;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_pages);

 /*
  * construct a new message with given type, size
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 1bb2b59..0f8351d 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1629,9 +1629,8 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
 {
 	int rc = 0;

-	req->r_request->pages = req->r_pages;
-	req->r_request->page_count = req->r_num_pages;
-	req->r_request->page_alignment = req->r_page_alignment;
+	ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
+				req->r_page_alignment);
 #ifdef CONFIG_BLOCK
 	req->r_request->bio = req->r_bio;
 #endif
@@ -1981,9 +1980,9 @@ static struct ceph_msg *get_reply(struct
ceph_connection *con,
 			m = NULL;
 			goto out;
 		}
-		m->pages = req->r_pages;
-		m->page_count = req->r_num_pages;
-		m->page_alignment = req->r_page_alignment;
+
+		ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
+					req->r_page_alignment);
 #ifdef CONFIG_BLOCK
 		m->bio = req->r_bio;
 #endif
-- 
1.7.9.5

[PATCH 4/4] libceph: isolate other message data fields

[Alex Elder]

Define ceph_msg_data_set_pagelist(), ceph_msg_data_set_bio(), and
ceph_msg_data_set_trail() to clearly abstract the assignment the
remaining data-related fields in a ceph message structure.  These
fields should never be used more than once; add BUG_ON() calls to
guarantee this.  Use the new functions in the osd client and mds
client.

Signed-off-by: Alex Elder <elder@inktank.com>
---
 fs/ceph/mds_client.c           |    4 ++--
 include/linux/ceph/messenger.h |    6 ++++++
 net/ceph/messenger.c           |   28 ++++++++++++++++++++++++++++
 net/ceph/osd_client.c          |    6 +++---
 4 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index d8842a1..c0eb44d 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -2599,8 +2599,8 @@ static void send_mds_reconnect(struct
ceph_mds_client *mdsc,
 			goto fail;
 	}

-	reply->pagelist = pagelist;
-	reply->pagelist_count = calc_pages_for(0, pagelist->length);
+	ceph_msg_data_set_pagelist(reply, pagelist,
+				calc_pages_for(0, pagelist->length));
 	if (recon_state.flock)
 		reply->hdr.version = cpu_to_le16(2);
 	reply->hdr.data_len = cpu_to_le32(pagelist->length);
diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index eeb28a0..3cb3e8c 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -222,6 +222,12 @@ extern void ceph_con_keepalive(struct
ceph_connection *con);

 extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page
**pages,
 				unsigned int page_count, size_t alignment);
+extern void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+				struct ceph_pagelist *pagelist,
+				unsigned int page_count);
+extern void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio);
+extern void ceph_msg_data_set_trail(struct ceph_msg *msg,
+				struct ceph_pagelist *trail);

 extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
 				     bool can_fail);
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 7328e1b..82e3212 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -2687,6 +2687,34 @@ void ceph_msg_data_set_pages(struct ceph_msg
*msg, struct page **pages,
 }
 EXPORT_SYMBOL(ceph_msg_data_set_pages);

+void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+				struct ceph_pagelist *pagelist,
+				unsigned int pagelist_count)
+{
+	BUG_ON(msg->pagelist);
+	BUG_ON(msg->pagelist_count);
+
+	msg->pagelist = pagelist;
+	msg->pagelist_count = pagelist_count;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_pagelist);
+
+void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio)
+{
+	BUG_ON(msg->bio);
+
+	msg->bio = bio;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_bio);
+
+void ceph_msg_data_set_trail(struct ceph_msg *msg, struct ceph_pagelist
*trail)
+{
+	BUG_ON(msg->trail);
+
+	msg->trail = trail;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_trail);
+
 /*
  * construct a new message with given type, size
  * the new msg has a ref count of 1.
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 0f8351d..f984500 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1632,9 +1632,9 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
 	ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
 				req->r_page_alignment);
 #ifdef CONFIG_BLOCK
-	req->r_request->bio = req->r_bio;
+	ceph_msg_data_set_bio(req->r_request, req->r_bio);
 #endif
-	req->r_request->trail = &req->r_trail;
+	ceph_msg_data_set_trail(req->r_request, &req->r_trail);

 	register_request(osdc, req);

@@ -1984,7 +1984,7 @@ static struct ceph_msg *get_reply(struct
ceph_connection *con,
 		ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
 					req->r_page_alignment);
 #ifdef CONFIG_BLOCK
-		m->bio = req->r_bio;
+		ceph_msg_data_set_bio(m, req->r_bio);
 #endif
 	}
 	*skip = 0;
-- 
1.7.9.5

Re: [PATCH 0/4] libceph: abstract setting message data info

[Josh Durgin]

On 02/25/2013 03:40 PM, Alex Elder wrote:
> This series makes the fields related to the data portion of
> a ceph message not get manipulated by code outside the ceph
> messenger.  It implements some interface functions that can
> be used to assign data-related fields.  Doing this will allow
> the way message data is managed to be changed independent
> of the users of the messenger module.
>
> 					-Alex
>
> [PATCH 1/4] libceph: distinguish page array and pagelist count
> [PATCH 2/4] libceph: set page alignment in start_request()
> [PATCH 3/4] libceph: isolate message page field manipulation
> [PATCH 4/4] libceph: isolate other message data fields

These all look good.
Reviewed-by: Josh Durgin <josh.durgin@inktank.com>