From: Pavel Begunkov <asml.silence@gmail.com>
To: Jens Axboe <axboe@kernel.dk>,
linux-block@vger.kernel.org, linux-kernel@vger.kernel.org
Cc: Christoph Hellwig <hch@infradead.org>, asml.silence@gmail.com
Subject: [PATCH v2 1/3] block: cache bdev in struct file for raw bdev IO
Date: Wed, 13 Oct 2021 09:57:11 +0100 [thread overview]
Message-ID: <8415f9fe12e544b9da89593dfbca8de2b52efe03.1634115360.git.asml.silence@gmail.com> (raw)
In-Reply-To: <cover.1634115360.git.asml.silence@gmail.com>
bdev = &BDEV_I(file->f_mapping->host)->bdev
Getting struct block_device from a file requires 2 memory dereferences
as illustrated above, that takes a toll on performance, so cache it in
yet unused file->private_data. That gives a noticeable peak performance
improvement.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
block/fops.c | 29 +++++++++++++----------------
1 file changed, 13 insertions(+), 16 deletions(-)
diff --git a/block/fops.c b/block/fops.c
index 7bb9581a146c..c71e91cd6bcb 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -17,7 +17,7 @@
#include <linux/fs.h>
#include "blk.h"
-static struct inode *bdev_file_inode(struct file *file)
+static inline struct inode *bdev_file_inode(struct file *file)
{
return file->f_mapping->host;
}
@@ -54,8 +54,7 @@ static void blkdev_bio_end_io_simple(struct bio *bio)
static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,
struct iov_iter *iter, unsigned int nr_pages)
{
- struct file *file = iocb->ki_filp;
- struct block_device *bdev = I_BDEV(bdev_file_inode(file));
+ struct block_device *bdev = iocb->ki_filp->private_data;
struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs;
loff_t pos = iocb->ki_pos;
bool should_dirty = false;
@@ -143,7 +142,7 @@ static struct bio_set blkdev_dio_pool;
static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
{
- struct block_device *bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
+ struct block_device *bdev = kiocb->ki_filp->private_data;
struct request_queue *q = bdev_get_queue(bdev);
return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait);
@@ -191,9 +190,7 @@ static void blkdev_bio_end_io(struct bio *bio)
static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
unsigned int nr_pages)
{
- struct file *file = iocb->ki_filp;
- struct inode *inode = bdev_file_inode(file);
- struct block_device *bdev = I_BDEV(inode);
+ struct block_device *bdev = iocb->ki_filp->private_data;
struct blk_plug plug;
struct blkdev_dio *dio;
struct bio *bio;
@@ -405,8 +402,7 @@ static loff_t blkdev_llseek(struct file *file, loff_t offset, int whence)
static int blkdev_fsync(struct file *filp, loff_t start, loff_t end,
int datasync)
{
- struct inode *bd_inode = bdev_file_inode(filp);
- struct block_device *bdev = I_BDEV(bd_inode);
+ struct block_device *bdev = filp->private_data;
int error;
error = file_write_and_wait_range(filp, start, end);
@@ -448,6 +444,8 @@ static int blkdev_open(struct inode *inode, struct file *filp)
bdev = blkdev_get_by_dev(inode->i_rdev, filp->f_mode, filp);
if (IS_ERR(bdev))
return PTR_ERR(bdev);
+
+ filp->private_data = bdev;
filp->f_mapping = bdev->bd_inode->i_mapping;
filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
return 0;
@@ -455,7 +453,7 @@ static int blkdev_open(struct inode *inode, struct file *filp)
static int blkdev_close(struct inode *inode, struct file *filp)
{
- struct block_device *bdev = I_BDEV(bdev_file_inode(filp));
+ struct block_device *bdev = filp->private_data;
blkdev_put(bdev, filp->f_mode);
return 0;
@@ -470,14 +468,14 @@ static int blkdev_close(struct inode *inode, struct file *filp)
*/
static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
- struct file *file = iocb->ki_filp;
- struct inode *bd_inode = bdev_file_inode(file);
+ struct block_device *bdev = iocb->ki_filp->private_data;
+ struct inode *bd_inode = bdev->bd_inode;
loff_t size = i_size_read(bd_inode);
struct blk_plug plug;
size_t shorted = 0;
ssize_t ret;
- if (bdev_read_only(I_BDEV(bd_inode)))
+ if (bdev_read_only(bdev))
return -EPERM;
if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode->i_rdev))
@@ -509,9 +507,8 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
- struct file *file = iocb->ki_filp;
- struct inode *bd_inode = bdev_file_inode(file);
- loff_t size = i_size_read(bd_inode);
+ struct block_device *bdev = iocb->ki_filp->private_data;
+ loff_t size = i_size_read(bdev->bd_inode);
loff_t pos = iocb->ki_pos;
size_t shorted = 0;
ssize_t ret;
--
2.33.0
next prev parent reply other threads:[~2021-10-13 8:58 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-13 8:57 [PATCH v2 0/3] on top of for-5.16/block Pavel Begunkov
2021-10-13 8:57 ` Pavel Begunkov [this message]
2021-10-13 15:19 ` [PATCH v2 1/3] block: cache bdev in struct file for raw bdev IO Christoph Hellwig
2021-10-13 8:57 ` [PATCH v2 2/3] block: don't hide inode from block_device users Pavel Begunkov
2021-10-13 14:03 ` kernel test robot
2021-10-13 15:03 ` kernel test robot
2021-10-13 15:12 ` Pavel Begunkov
2021-10-15 0:40 ` [kbuild-all] " Chen, Rong A
2021-10-13 15:25 ` Christoph Hellwig
2021-10-13 18:44 ` Pavel Begunkov
2021-10-14 4:51 ` Christoph Hellwig
2021-10-14 9:35 ` Pavel Begunkov
2021-10-13 8:57 ` [PATCH v2 3/3] blk-mq: optimise *end_request non-stat path Pavel Begunkov
2021-10-13 15:30 ` Christoph Hellwig
2021-10-13 9:15 ` [PATCH v2 0/3] on top of for-5.16/block Pavel Begunkov
2021-10-13 15:53 ` (subset) " Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=8415f9fe12e544b9da89593dfbca8de2b52efe03.1634115360.git.asml.silence@gmail.com \
--to=asml.silence@gmail.com \
--cc=axboe@kernel.dk \
--cc=hch@infradead.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).