From: "Matthew Wilcox (Oracle)" <willy@infradead.org>
To: linux-fsdevel@vger.kernel.org, linux-xfs@vger.kernel.org
Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>, linux-mm@kvack.org
Subject: [PATCH 03/14] iomap: Support THPs in BIO completion path
Date: Wed, 14 Oct 2020 04:03:46 +0100 [thread overview]
Message-ID: <20201014030357.21898-4-willy@infradead.org> (raw)
In-Reply-To: <20201014030357.21898-1-willy@infradead.org>
bio_for_each_segment_all() iterates once per regular sized page.
Use bio_for_each_bvec_all() to iterate once per bvec and handle
merged THPs ourselves, instead of teaching the block layer about THPs.
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
fs/iomap/buffered-io.c | 62 ++++++++++++++++++++++++++++++------------
1 file changed, 44 insertions(+), 18 deletions(-)
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index 3e1eb40a73fd..935468d79d9d 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -167,32 +167,45 @@ iomap_set_range_uptodate(struct page *page, unsigned off, unsigned len)
SetPageUptodate(page);
}
-static void
-iomap_read_page_end_io(struct bio_vec *bvec, int error)
+static void iomap_finish_page_read(struct page *page, size_t offset,
+ size_t length, int error)
{
- struct page *page = bvec->bv_page;
struct iomap_page *iop = to_iomap_page(page);
if (unlikely(error)) {
ClearPageUptodate(page);
SetPageError(page);
} else {
- iomap_set_range_uptodate(page, bvec->bv_offset, bvec->bv_len);
+ iomap_set_range_uptodate(page, offset, length);
}
- if (!iop || atomic_sub_and_test(bvec->bv_len, &iop->read_bytes_pending))
+ if (!iop || atomic_sub_and_test(length, &iop->read_bytes_pending))
unlock_page(page);
}
-static void
-iomap_read_end_io(struct bio *bio)
+static void iomap_finish_bvec_read(struct page *page, size_t offset,
+ size_t length, int error)
+{
+ while (length > 0) {
+ size_t count = min(thp_size(page) - offset, length);
+
+ iomap_finish_page_read(page, offset, count, error);
+
+ page += (offset + count) / PAGE_SIZE;
+ offset = 0;
+ length -= count;
+ }
+}
+
+static void iomap_read_end_io(struct bio *bio)
{
- int error = blk_status_to_errno(bio->bi_status);
+ int i, error = blk_status_to_errno(bio->bi_status);
struct bio_vec *bvec;
- struct bvec_iter_all iter_all;
- bio_for_each_segment_all(bvec, bio, iter_all)
- iomap_read_page_end_io(bvec, error);
+ bio_for_each_bvec_all(bvec, bio, i)
+ iomap_finish_bvec_read(bvec->bv_page, bvec->bv_offset,
+ bvec->bv_len, error);
+
bio_put(bio);
}
@@ -1035,9 +1048,8 @@ vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
}
EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
-static void
-iomap_finish_page_writeback(struct inode *inode, struct page *page,
- int error, unsigned int len)
+static void iomap_finish_page_write(struct inode *inode, struct page *page,
+ unsigned int len, int error)
{
struct iomap_page *iop = to_iomap_page(page);
@@ -1053,6 +1065,20 @@ iomap_finish_page_writeback(struct inode *inode, struct page *page,
end_page_writeback(page);
}
+static void iomap_finish_bvec_write(struct inode *inode, struct page *page,
+ size_t offset, size_t length, int error)
+{
+ while (length > 0) {
+ size_t count = min(thp_size(page) - offset, length);
+
+ iomap_finish_page_write(inode, page, count, error);
+
+ page += (offset + count) / PAGE_SIZE;
+ offset = 0;
+ length -= count;
+ }
+}
+
/*
* We're now finished for good with this ioend structure. Update the page
* state, release holds on bios, and finally free up memory. Do not use the
@@ -1070,7 +1096,7 @@ iomap_finish_ioend(struct iomap_ioend *ioend, int error)
for (bio = &ioend->io_inline_bio; bio; bio = next) {
struct bio_vec *bv;
- struct bvec_iter_all iter_all;
+ int i;
/*
* For the last bio, bi_private points to the ioend, so we
@@ -1082,9 +1108,9 @@ iomap_finish_ioend(struct iomap_ioend *ioend, int error)
next = bio->bi_private;
/* walk each page on bio, ending page IO on them */
- bio_for_each_segment_all(bv, bio, iter_all)
- iomap_finish_page_writeback(inode, bv->bv_page, error,
- bv->bv_len);
+ bio_for_each_bvec_all(bv, bio, i)
+ iomap_finish_bvec_write(inode, bv->bv_page,
+ bv->bv_offset, bv->bv_len, error);
bio_put(bio);
}
/* The ioend has been freed by bio_put() */
--
2.28.0
next prev parent reply other threads:[~2020-10-14 9:22 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-10-14 3:03 [PATCH 00/14] Transparent Huge Page support for XFS Matthew Wilcox (Oracle)
2020-10-14 3:03 ` [PATCH 01/14] fs: Support THPs in vfs_dedupe_file_range Matthew Wilcox (Oracle)
2020-10-14 16:12 ` Darrick J. Wong
2020-10-14 17:16 ` Matthew Wilcox
2020-10-14 3:03 ` [PATCH 02/14] fs: Make page_mkwrite_check_truncate thp-aware Matthew Wilcox (Oracle)
2020-10-14 16:17 ` Darrick J. Wong
2020-10-14 17:23 ` Matthew Wilcox
2020-10-14 3:03 ` Matthew Wilcox (Oracle) [this message]
2020-10-15 9:50 ` [PATCH 03/14] iomap: Support THPs in BIO completion path Christoph Hellwig
2020-10-14 3:03 ` [PATCH 04/14] iomap: Support THPs in iomap_adjust_read_range Matthew Wilcox (Oracle)
2020-10-15 9:50 ` Christoph Hellwig
2020-10-14 3:03 ` [PATCH 05/14] iomap: Support THPs in invalidatepage Matthew Wilcox (Oracle)
2020-10-14 16:33 ` Darrick J. Wong
2020-10-14 17:26 ` Matthew Wilcox
2020-10-14 20:00 ` Brian Foster
2020-10-14 3:03 ` [PATCH 06/14] iomap: Support THPs in iomap_is_partially_uptodate Matthew Wilcox (Oracle)
2020-10-14 3:03 ` [PATCH 07/14] iomap: Support THPs in readpage Matthew Wilcox (Oracle)
2020-10-14 16:39 ` Darrick J. Wong
2020-10-14 17:35 ` Matthew Wilcox
2020-10-14 3:03 ` [PATCH 08/14] iomap: Support THPs in readahead Matthew Wilcox (Oracle)
2020-10-15 9:52 ` Christoph Hellwig
2020-10-14 3:03 ` [PATCH 09/14] iomap: Change iomap_write_begin calling convention Matthew Wilcox (Oracle)
2020-10-14 16:47 ` Darrick J. Wong
2020-10-14 17:41 ` Matthew Wilcox
2020-10-14 18:08 ` Matthew Wilcox
2020-10-14 3:03 ` [PATCH 10/14] iomap: Handle THPs when writing to pages Matthew Wilcox (Oracle)
2020-10-14 3:03 ` [PATCH 11/14] iomap: Support THP writeback Matthew Wilcox (Oracle)
2020-10-14 3:03 ` [PATCH 12/14] iomap: Inline data shouldn't see THPs Matthew Wilcox (Oracle)
2020-10-14 3:03 ` [PATCH 13/14] iomap: Handle tail pages in iomap_page_mkwrite Matthew Wilcox (Oracle)
2020-10-14 3:03 ` [PATCH 14/14] xfs: Support THPs Matthew Wilcox (Oracle)
2020-10-14 16:51 ` Darrick J. Wong
2020-10-14 17:30 ` Matthew Wilcox
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201014030357.21898-4-willy@infradead.org \
--to=willy@infradead.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-xfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.