[PATCH] block: Minimize the number of requests in Direct I/O when bio get pages.

[Zhengxu Zhang <zhengxu.zhang@unisoc.com>]

We find that when direct io is decomposed into requests,
there are often have some requests size that are less than the maximum
size of a request, which increases the number of requests for a direct io
 and slows down the speed.

We try to let bio determine how many pages canbe obtained by dividing it
by the maximum request through bio_export_page
when obtaining a set of pages. Then try to obtain
as many pages as possible.

Of course, if the last page obtained by
bio is less than the maximum number that bio can obtain,
it will also be added to that bio (if the bio have enough vector)
or the next bio.

Device info: ufs RAM:6GB kernel:6.6 improve seq read 5%
Device info: emmc RAM:4GB kernel:5.15 improve seq read 10%

1.androidbench test result(ufs RAM:6GB kernel6.6):

before:								 avg
seq read	1812	1735	1879	1887	1817	1849	1829
seq write	1271	1294	1287	1305	1296	1296	1291
ram read	301	298	297	297	295	294	297
ram write	343	342	344	351	349	345	345
sql insert	2773	2779	2856	2801	2932	2913	2842
sql update	2819	2888	3009	2953	2961	2974	2934
sql delete	4117	4004	4121	4158	4088	4071	4093

after:								 avg
seq read	1923	1919	1946	1921	1953	1937	1933	+5.65%
seq write	1277	1301	1301	1296	1291	1299	1294	+0.21%
ram read	302	301	302	306	301	297	301	+1.52%
ram write	350	359	350	339	351	356	350	+1.49%
sql insert	2788	2935	3011	3022	3069	3037	2977	+4.74%
sql update	2991	3123	3125	3070	3087	3132	3088	+5.25%
sql delete	4157	4174	4173	4177	4297	4246	4204	+2.71%

seq read avg: 1933MB/s better than 1829MB/s (up about 5.65%)

2.trace:
before:   lots of (2048+x),it will product 2+1 requests,
           the "1" will lead speed down.

	block_bio_queue: 179,0 R 84601376 + 2064 [Thread-2]
	block_bio_queue: 179,0 R 84603440 + 2048 [Thread-2]
	block_bio_queue: 179,0 R 84605488 + 2072 [Thread-2]
	block_bio_queue: 179,0 R 84607560 + 2064 [Thread-2]
	block_bio_queue: 179,0 R 84609624 + 2064 [Thread-2]

after:   we make size of bio can divide by request_max_size.
       and then do not product the "1", minimize one direct-io requests  ,
       and minimize the number of bio as soon as possible at the same time.

	block_bio_queue: 179,0 R 69829456 + 2048 [Thread-3]
	block_bio_queue: 179,0 R 69831504 + 3072 [Thread-3]
	block_bio_queue: 179,0 R 69834576 + 4096 [Thread-3]
	block_bio_queue: 179,0 R 69838672 + 5120 [Thread-3]

	block_bio_queue: 8,0 WFS 153795416 + 5120 [Thread-11]
	block_bio_queue: 8,0 WFS 153800536 + 7168 [Thread-11]
	block_bio_queue: 8,0 WFS 153807704 + 9216 [Thread-11]
	block_bio_queue: 8,0 WFS 153816920 + 10240 [Thread-11]
	block_bio_queue: 8,0 WFS 153827160 + 8192 [Thread-11]

Signed-off-by: Zhengxu Zhang <Zhengxu.Zhang@unisoc.com>
---
 block/bio.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 71 insertions(+), 3 deletions(-)

diff --git a/block/bio.c b/block/bio.c
index b9642a41f286..f821a5fb72bd 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -903,6 +903,70 @@ static inline bool bio_full(struct bio *bio, unsigned len)
 	return false;
 }
 
+/**
+ * bio_export_pages - calculate how many pages does this bio needs,
+ * in order to minimize the number of requests the bio needs to split.
+ * @bio:        bio to check
+ * @page_total: total number of pages need to be added. If no page_total,
+ *              it can be used by UINT_MAX.
+ *
+ * Return the page numbers can be added to this bio.
+ * If return 0 means do not suggest add page in this bio,
+ * lead to add more requests in direct-io.
+ */
+static unsigned int bio_export_pages(struct bio *bio, unsigned int page_total)
+{
+	unsigned int request_max_size;
+	unsigned int request_max_pages;
+	unsigned short bio_export_pages;
+	unsigned int last_request_size;
+	unsigned int nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
+
+	/*
+	 * The number of pages that need to add is less than the pages that bio can be get,
+	 * we can get whole pages directly.
+	 */
+	if (page_total < nr_pages)
+		return page_total;
+
+	if (bio->bi_bdev) {
+		request_max_size = queue_max_bytes(bdev_get_queue(bio->bi_bdev));
+		/* request_max_size >> PAGE_SHIFT should equals 2^n */
+		request_max_pages = request_max_size >> PAGE_SHIFT;
+		last_request_size = bio->bi_iter.bi_size & (request_max_size - 1);
+
+		/*
+		 * when bio->bi_iter.bi_size % request_max_size = 0, we need confirm if
+		 * we still need to add a page to bio, because now the bio size is best for request.
+		 *
+		 * If bio can get the max number of pages is less than the number of
+		 * one complete request, we should still get pages if possible.
+		 *
+		 * If the bio can not get the size of one complete request that we badly think
+		 * the page physical addresses are not contiguous.
+		 * we suggest this bio do not add pages, the pages maybe break
+		 *  "bio->bi_iter.bi_size % request_max_size = 0", and then make a litte request.
+		 * the next bio continues maybe better.
+		 */
+		if (last_request_size == 0 && bio->bi_max_vecs > request_max_pages &&
+			nr_pages < request_max_pages)
+			return 0;
+
+		/*
+		 * base on the bio->vcnt, we confirm the max number of pages that can keep
+		 *  bio->bi_iter.bi_size % request_max_size = 0 when we can get,
+		 * note: the sum of the pages size maybe equals more than one request size.
+		 */
+		bio_export_pages = (request_max_size - last_request_size) >> PAGE_SHIFT;
+		if (!bio_export_pages)
+			bio_export_pages = request_max_pages;
+		if (nr_pages > bio_export_pages)
+			nr_pages -= nr_pages & (request_max_pages - 1);
+	}
+
+	return nr_pages;
+}
+
 static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page,
 		unsigned int len, unsigned int off, bool *same_page)
 {
@@ -1228,16 +1292,16 @@ static int bio_iov_add_zone_append_page(struct bio *bio, struct page *page,
  * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
  * @bio: bio to add pages to
  * @iter: iov iterator describing the region to be mapped
+ * @nr_pages: the number of pages that bio want to get.
  *
  * Extracts pages from *iter and appends them to @bio's bvec array.  The pages
  * will have to be cleaned up in the way indicated by the BIO_PAGE_PINNED flag.
  * For a multi-segment *iter, this function only adds pages from the next
  * non-empty segment of the iov iterator.
  */
-static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
+static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned short nr_pages)
 {
 	iov_iter_extraction_t extraction_flags = 0;
-	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
 	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
 	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
 	struct page **pages = (struct page **)bv;
@@ -1329,6 +1393,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 {
 	int ret = 0;
+	unsigned short nr_pages;
 
 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
 		return -EIO;
@@ -1342,7 +1407,10 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 	if (iov_iter_extract_will_pin(iter))
 		bio_set_flag(bio, BIO_PAGE_PINNED);
 	do {
-		ret = __bio_iov_iter_get_pages(bio, iter);
+		nr_pages = bio_export_pages(bio, iov_iter_count(iter));
+		if (!nr_pages)
+			break;
+		ret = __bio_iov_iter_get_pages(bio, iter, nr_pages);
 	} while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
 
 	return bio->bi_vcnt ? 0 : ret;
-- 
2.25.1