From: Al Viro <viro@zeniv.linux.org.uk>
To: linux-fsdevel@vger.kernel.org
Cc: Linus Torvalds <torvalds@linux-foundation.org>,
Jens Axboe <axboe@kernel.dk>, Christoph Hellwig <hch@lst.de>,
Matthew Wilcox <willy@infradead.org>,
David Howells <dhowells@redhat.com>,
Dominique Martinet <asmadeus@codewreck.org>,
Christian Brauner <brauner@kernel.org>
Subject: [PATCH 16/44] ITER_PIPE: allocate buffers as we go in copy-to-pipe primitives
Date: Wed, 22 Jun 2022 05:15:24 +0100 [thread overview]
Message-ID: <20220622041552.737754-16-viro@zeniv.linux.org.uk> (raw)
In-Reply-To: <20220622041552.737754-1-viro@zeniv.linux.org.uk>
New helper: append_pipe(). Extends the last buffer if possible,
allocates a new one otherwise. Returns page and offset in it
on success, NULL on failure. iov_iter is advanced past the
data we've got.
Use that instead of push_pipe() in copy-to-pipe primitives;
they get simpler that way. Handling of short copy (in "mc" one)
is done simply by iov_iter_revert() - iov_iter is in consistent
state after that one, so we can use that.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
---
lib/iov_iter.c | 159 +++++++++++++++++++++++++++++--------------------
1 file changed, 93 insertions(+), 66 deletions(-)
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 924854c2a7ce..2a445261096e 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -259,6 +259,44 @@ static void push_page(struct pipe_inode_info *pipe, struct page *page,
get_page(page);
}
+static inline bool allocated(struct pipe_buffer *buf)
+{
+ return buf->ops == &default_pipe_buf_ops;
+}
+
+static struct page *append_pipe(struct iov_iter *i, size_t size, size_t *off)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ size_t offset = i->iov_offset;
+ struct pipe_buffer *buf;
+ struct page *page;
+
+ if (offset && offset < PAGE_SIZE) {
+ // some space in the last buffer; can we add to it?
+ buf = pipe_buf(pipe, pipe->head - 1);
+ if (allocated(buf)) {
+ size = min_t(size_t, size, PAGE_SIZE - offset);
+ buf->len += size;
+ i->iov_offset += size;
+ i->count -= size;
+ *off = offset;
+ return buf->page;
+ }
+ }
+ // OK, we need a new buffer
+ *off = 0;
+ size = min_t(size_t, size, PAGE_SIZE);
+ if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
+ return NULL;
+ page = push_anon(pipe, size);
+ if (!page)
+ return NULL;
+ i->head = pipe->head - 1;
+ i->iov_offset = size;
+ i->count -= size;
+ return page;
+}
+
static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
@@ -396,11 +434,6 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction,
}
EXPORT_SYMBOL(iov_iter_init);
-static inline bool allocated(struct pipe_buffer *buf)
-{
- return buf->ops == &default_pipe_buf_ops;
-}
-
static inline void data_start(const struct iov_iter *i,
unsigned int *iter_headp, size_t *offp)
{
@@ -459,28 +492,26 @@ static size_t push_pipe(struct iov_iter *i, size_t size,
static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
struct iov_iter *i)
{
- struct pipe_inode_info *pipe = i->pipe;
- unsigned int p_mask = pipe->ring_size - 1;
- unsigned int i_head;
size_t n, off;
- if (!sanity(i))
+ if (unlikely(bytes > i->count))
+ bytes = i->count;
+ if (unlikely(!bytes))
return 0;
- bytes = n = push_pipe(i, bytes, &i_head, &off);
- if (unlikely(!n))
+ if (!sanity(i))
return 0;
- do {
+
+ n = bytes;
+ while (n) {
+ struct page *page = append_pipe(i, n, &off);
size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
- memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
- i->head = i_head;
- i->iov_offset = off + chunk;
- n -= chunk;
+ if (!page)
+ break;
+ memcpy_to_page(page, off, addr, chunk);
addr += chunk;
- off = 0;
- i_head++;
- } while (n);
- i->count -= bytes;
+ n -= chunk;
+ }
return bytes;
}
@@ -494,31 +525,32 @@ static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
struct iov_iter *i, __wsum *sump)
{
- struct pipe_inode_info *pipe = i->pipe;
- unsigned int p_mask = pipe->ring_size - 1;
__wsum sum = *sump;
size_t off = 0;
- unsigned int i_head;
size_t r;
+ if (unlikely(bytes > i->count))
+ bytes = i->count;
+ if (unlikely(!bytes))
+ return 0;
+
if (!sanity(i))
return 0;
- bytes = push_pipe(i, bytes, &i_head, &r);
while (bytes) {
+ struct page *page = append_pipe(i, bytes, &r);
size_t chunk = min_t(size_t, bytes, PAGE_SIZE - r);
- char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page);
+ char *p;
+
+ if (!page)
+ break;
+ p = kmap_local_page(page);
sum = csum_and_memcpy(p + r, addr + off, chunk, sum, off);
kunmap_local(p);
- i->head = i_head;
- i->iov_offset = r + chunk;
- bytes -= chunk;
off += chunk;
- r = 0;
- i_head++;
+ bytes -= chunk;
}
*sump = sum;
- i->count -= off;
return off;
}
@@ -550,39 +582,35 @@ static int copyout_mc(void __user *to, const void *from, size_t n)
static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
struct iov_iter *i)
{
- struct pipe_inode_info *pipe = i->pipe;
- unsigned int p_mask = pipe->ring_size - 1;
- unsigned int i_head;
- unsigned int valid = pipe->head;
- size_t n, off, xfer = 0;
+ size_t off, xfer = 0;
+
+ if (unlikely(bytes > i->count))
+ bytes = i->count;
+ if (unlikely(!bytes))
+ return 0;
if (!sanity(i))
return 0;
- n = push_pipe(i, bytes, &i_head, &off);
- while (n) {
- size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
- char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page);
+ while (bytes) {
+ struct page *page = append_pipe(i, bytes, &off);
+ size_t chunk = min_t(size_t, bytes, PAGE_SIZE - off);
unsigned long rem;
+ char *p;
+
+ if (!page)
+ break;
+ p = kmap_local_page(page);
rem = copy_mc_to_kernel(p + off, addr + xfer, chunk);
chunk -= rem;
kunmap_local(p);
- if (chunk) {
- i->head = i_head;
- i->iov_offset = off + chunk;
- xfer += chunk;
- valid = i_head + 1;
- }
+ xfer += chunk;
+ bytes -= chunk;
if (rem) {
- pipe->bufs[i_head & p_mask].len -= rem;
- pipe_discard_from(pipe, valid);
+ iov_iter_revert(i, rem);
break;
}
- n -= chunk;
- off = 0;
- i_head++;
}
- i->count -= xfer;
return xfer;
}
@@ -769,30 +797,29 @@ EXPORT_SYMBOL(copy_page_from_iter);
static size_t pipe_zero(size_t bytes, struct iov_iter *i)
{
- struct pipe_inode_info *pipe = i->pipe;
- unsigned int p_mask = pipe->ring_size - 1;
- unsigned int i_head;
size_t n, off;
- if (!sanity(i))
+ if (unlikely(bytes > i->count))
+ bytes = i->count;
+ if (unlikely(!bytes))
return 0;
- bytes = n = push_pipe(i, bytes, &i_head, &off);
- if (unlikely(!n))
+ if (!sanity(i))
return 0;
- do {
+ n = bytes;
+ while (n) {
+ struct page *page = append_pipe(i, n, &off);
size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
- char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page);
+ char *p;
+
+ if (!page)
+ break;
+ p = kmap_local_page(page);
memset(p + off, 0, chunk);
kunmap_local(p);
- i->head = i_head;
- i->iov_offset = off + chunk;
n -= chunk;
- off = 0;
- i_head++;
- } while (n);
- i->count -= bytes;
+ }
return bytes;
}
--
2.30.2
next prev parent reply other threads:[~2022-06-22 4:16 UTC|newest]
Thread overview: 118+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-22 4:10 [RFC][CFT][PATCHSET] iov_iter stuff Al Viro
2022-06-22 4:15 ` [PATCH 01/44] 9p: handling Rerror without copy_from_iter_full() Al Viro
2022-06-22 4:15 ` [PATCH 02/44] No need of likely/unlikely on calls of check_copy_size() Al Viro
2022-06-22 4:15 ` [PATCH 03/44] teach iomap_dio_rw() to suppress dsync Al Viro
2022-06-22 4:15 ` [PATCH 04/44] btrfs: use IOMAP_DIO_NOSYNC Al Viro
2022-06-22 4:15 ` [PATCH 05/44] struct file: use anonymous union member for rcuhead and llist Al Viro
2022-06-22 4:15 ` [PATCH 06/44] iocb: delay evaluation of IS_SYNC(...) until we want to check IOCB_DSYNC Al Viro
2022-06-22 4:15 ` [PATCH 07/44] keep iocb_flags() result cached in struct file Al Viro
2022-06-22 4:15 ` [PATCH 08/44] copy_page_{to,from}_iter(): switch iovec variants to generic Al Viro
2022-06-27 18:31 ` Jeff Layton
2022-06-28 12:32 ` Christian Brauner
2022-06-28 18:36 ` Al Viro
2022-06-22 4:15 ` [PATCH 09/44] new iov_iter flavour - ITER_UBUF Al Viro
2022-06-27 18:47 ` Jeff Layton
2022-06-28 18:41 ` Al Viro
2022-06-28 12:38 ` Christian Brauner
2022-06-28 18:44 ` Al Viro
2022-07-28 9:55 ` [PATCH 9/44] " Alexander Gordeev
2022-07-29 17:21 ` Al Viro
2022-07-29 21:12 ` Alexander Gordeev
2022-07-30 0:03 ` Al Viro
2022-06-22 4:15 ` [PATCH 10/44] switch new_sync_{read,write}() to ITER_UBUF Al Viro
2022-06-22 4:15 ` [PATCH 11/44] iov_iter_bvec_advance(): don't bother with bvec_iter Al Viro
2022-06-27 18:48 ` Jeff Layton
2022-06-28 12:40 ` Christian Brauner
2022-06-22 4:15 ` [PATCH 12/44] fix short copy handling in copy_mc_pipe_to_iter() Al Viro
2022-06-27 19:15 ` Jeff Layton
2022-06-28 12:42 ` Christian Brauner
2022-06-22 4:15 ` [PATCH 13/44] splice: stop abusing iov_iter_advance() to flush a pipe Al Viro
2022-06-27 19:17 ` Jeff Layton
2022-06-28 12:43 ` Christian Brauner
2022-06-22 4:15 ` [PATCH 14/44] ITER_PIPE: helper for getting pipe buffer by index Al Viro
2022-06-28 10:38 ` Jeff Layton
2022-06-28 12:45 ` Christian Brauner
2022-06-22 4:15 ` [PATCH 15/44] ITER_PIPE: helpers for adding pipe buffers Al Viro
2022-06-28 11:32 ` Jeff Layton
2022-06-22 4:15 ` Al Viro [this message]
2022-06-22 4:15 ` [PATCH 17/44] ITER_PIPE: fold push_pipe() into __pipe_get_pages() Al Viro
2022-06-22 4:15 ` [PATCH 18/44] ITER_PIPE: lose iter_head argument of __pipe_get_pages() Al Viro
2022-06-22 4:15 ` [PATCH 19/44] ITER_PIPE: clean pipe_advance() up Al Viro
2022-06-22 4:15 ` [PATCH 20/44] ITER_PIPE: clean iov_iter_revert() Al Viro
2022-06-22 4:15 ` [PATCH 21/44] ITER_PIPE: cache the type of last buffer Al Viro
2022-06-22 4:15 ` [PATCH 22/44] ITER_PIPE: fold data_start() and pipe_space_for_user() together Al Viro
2022-06-22 4:15 ` [PATCH 23/44] iov_iter_get_pages{,_alloc}(): cap the maxsize with MAX_RW_COUNT Al Viro
2022-06-28 11:41 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 24/44] iov_iter_get_pages_alloc(): lift freeing pages array on failure exits into wrapper Al Viro
2022-06-28 11:45 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 25/44] iov_iter_get_pages(): sanity-check arguments Al Viro
2022-06-28 11:47 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 26/44] unify pipe_get_pages() and pipe_get_pages_alloc() Al Viro
2022-06-28 11:49 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 27/44] unify xarray_get_pages() and xarray_get_pages_alloc() Al Viro
2022-06-28 11:50 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 28/44] unify the rest of iov_iter_get_pages()/iov_iter_get_pages_alloc() guts Al Viro
2022-06-28 11:54 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 29/44] ITER_XARRAY: don't open-code DIV_ROUND_UP() Al Viro
2022-06-28 11:54 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 30/44] iov_iter: lift dealing with maxpages out of first_{iovec,bvec}_segment() Al Viro
2022-06-28 11:56 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 31/44] iov_iter: first_{iovec,bvec}_segment() - simplify a bit Al Viro
2022-06-28 11:58 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 32/44] iov_iter: massage calling conventions for first_{iovec,bvec}_segment() Al Viro
2022-06-28 12:06 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 33/44] found_iovec_segment(): just return address Al Viro
2022-06-28 12:09 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 34/44] fold __pipe_get_pages() into pipe_get_pages() Al Viro
2022-06-28 12:11 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 35/44] iov_iter: saner helper for page array allocation Al Viro
2022-06-28 12:12 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 36/44] iov_iter: advancing variants of iov_iter_get_pages{,_alloc}() Al Viro
2022-06-28 12:13 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 37/44] block: convert to " Al Viro
2022-06-28 12:16 ` Jeff Layton
2022-06-30 22:11 ` [block.git conflicts] " Al Viro
2022-06-30 22:39 ` Al Viro
2022-07-01 2:07 ` Keith Busch
2022-07-01 17:40 ` Al Viro
2022-07-01 17:53 ` Keith Busch
2022-07-01 18:07 ` Al Viro
2022-07-01 18:12 ` Al Viro
2022-07-01 18:38 ` Keith Busch
2022-07-01 19:08 ` Al Viro
2022-07-01 19:28 ` Keith Busch
2022-07-01 19:43 ` Al Viro
2022-07-01 19:56 ` Keith Busch
2022-07-02 5:35 ` Al Viro
2022-07-02 21:02 ` Keith Busch
2022-07-01 19:05 ` Keith Busch
2022-07-01 21:30 ` Jens Axboe
2022-06-30 23:07 ` Jens Axboe
2022-07-10 18:04 ` Sedat Dilek
2022-06-22 4:15 ` [PATCH 38/44] iter_to_pipe(): switch to advancing variant of iov_iter_get_pages() Al Viro
2022-06-28 12:18 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 39/44] af_alg_make_sg(): " Al Viro
2022-06-28 12:18 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 40/44] 9p: convert to advancing variant of iov_iter_get_pages_alloc() Al Viro
2022-07-01 9:01 ` Dominique Martinet
2022-07-01 13:47 ` Christian Schoenebeck
2022-07-06 22:06 ` Christian Schoenebeck
2022-06-22 4:15 ` [PATCH 41/44] ceph: switch the last caller " Al Viro
2022-06-28 12:20 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 42/44] get rid of non-advancing variants Al Viro
2022-06-28 12:21 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 43/44] pipe_get_pages(): switch to append_pipe() Al Viro
2022-06-28 12:23 ` Jeff Layton
2022-06-22 4:15 ` [PATCH 44/44] expand those iov_iter_advance() Al Viro
2022-06-28 12:23 ` Jeff Layton
2022-07-01 6:21 ` [PATCH 01/44] 9p: handling Rerror without copy_from_iter_full() Dominique Martinet
2022-07-01 6:25 ` Dominique Martinet
2022-07-01 16:02 ` Christian Schoenebeck
2022-07-01 21:00 ` Dominique Martinet
2022-07-03 13:30 ` Christian Schoenebeck
2022-08-01 12:42 ` [PATCH 09/44] new iov_iter flavour - ITER_UBUF David Howells
2022-08-01 21:14 ` Al Viro
2022-08-01 22:54 ` David Howells
2022-06-23 15:21 ` [RFC][CFT][PATCHSET] iov_iter stuff David Howells
2022-06-23 20:32 ` Al Viro
2022-06-28 12:25 ` Jeff Layton
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220622041552.737754-16-viro@zeniv.linux.org.uk \
--to=viro@zeniv.linux.org.uk \
--cc=asmadeus@codewreck.org \
--cc=axboe@kernel.dk \
--cc=brauner@kernel.org \
--cc=dhowells@redhat.com \
--cc=hch@lst.de \
--cc=linux-fsdevel@vger.kernel.org \
--cc=torvalds@linux-foundation.org \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).