Re: [RFC][CFT][PATCHSET v1] uaccess unification

[Al Viro <viro@ZenIV.linux.org.uk>]

On Thu, Mar 30, 2017 at 05:22:41PM +0100, Russell King - ARM Linux wrote:
> On Wed, Mar 29, 2017 at 06:57:06AM +0100, Al Viro wrote:
> > Comments, review, testing, replacement patches, etc. are very welcome.
> 
> I've given this a spin, and it appears to work (in that the box boots).
> 
> Kernel size wise:
> 
>    text    data      bss      dec     hex filename
> 8020229 3014220 10243276 21277725 144ac1d vmlinux.orig
> 8034741 3014388 10243276 21292405 144e575 vmlinux.uaccess
> 7976719 3014324 10243276 21234319 144028f vmlinux.noinline
> 
> Performance using hdparm -T (cached reads) to evaluate against a SSD
> gives me the following results:
> 
> * original:
>  Timing cached reads:   580 MB in  2.00 seconds = 289.64 MB/sec
>  Timing cached reads:   580 MB in  2.00 seconds = 290.06 MB/sec
>  Timing cached reads:   580 MB in  2.00 seconds = 289.65 MB/sec
>  Timing cached reads:   582 MB in  2.00 seconds = 290.82 MB/sec
>  Timing cached reads:   578 MB in  2.00 seconds = 289.07 MB/sec
> 
>  Average = 289.85MB/s
> 
> * uaccess:
>  Timing cached reads:   578 MB in  2.00 seconds = 288.36 MB/sec
>  Timing cached reads:   534 MB in  2.00 seconds = 266.68 MB/sec
>  Timing cached reads:   534 MB in  2.00 seconds = 267.07 MB/sec
>  Timing cached reads:   552 MB in  2.00 seconds = 275.45 MB/sec
>  Timing cached reads:   532 MB in  2.00 seconds = 266.08 MB/sec
> 
>  Average = 272.73 MB/sec
> 
> * noinline:
>  Timing cached reads:   548 MB in  2.00 seconds = 274.16 MB/sec
>  Timing cached reads:   574 MB in  2.00 seconds = 287.19 MB/sec
>  Timing cached reads:   574 MB in  2.00 seconds = 286.47 MB/sec
>  Timing cached reads:   572 MB in  2.00 seconds = 286.20 MB/sec
>  Timing cached reads:   578 MB in  2.00 seconds = 288.86 MB/sec
> 
>  Average = 284.58 MB/sec
> 
> I've run the test twice, and there's definitely a reproducable drop in
> performance for some reason when switching between current and Al's
> uaccess patches, which is partly recovered by switching to the out of
> line versions.
> 
> The only difference that I can identify that could explain this are
> the extra might_fault() checks in Al's version but which are missing
> from the ARM version.

How would the following affect things?

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index e68604ae3ced..d24d338f0682 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -184,7 +184,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
 
 	kaddr = kmap(page);
 	from = kaddr + offset;
-	left = __copy_to_user(buf, from, copy);
+	left = __copy_to_user_inatomic(buf, from, copy);
 	copy -= left;
 	skip += copy;
 	from += copy;
@@ -193,7 +193,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
 		iov++;
 		buf = iov->iov_base;
 		copy = min(bytes, iov->iov_len);
-		left = __copy_to_user(buf, from, copy);
+		left = __copy_to_user_inatomic(buf, from, copy);
 		copy -= left;
 		skip = copy;
 		from += copy;
@@ -267,7 +267,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
 
 	kaddr = kmap(page);
 	to = kaddr + offset;
-	left = __copy_from_user(to, buf, copy);
+	left = __copy_from_user_inatomic(to, buf, copy);
 	copy -= left;
 	skip += copy;
 	to += copy;
@@ -276,7 +276,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
 		iov++;
 		buf = iov->iov_base;
 		copy = min(bytes, iov->iov_len);
-		left = __copy_from_user(to, buf, copy);
+		left = __copy_from_user_inatomic(to, buf, copy);
 		copy -= left;
 		skip = copy;
 		to += copy;
@@ -541,7 +541,7 @@ size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 	if (unlikely(i->type & ITER_PIPE))
 		return copy_pipe_to_iter(addr, bytes, i);
 	iterate_and_advance(i, bytes, v,
-		__copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
+		__copy_to_user_inatomic(v.iov_base, (from += v.iov_len) - v.iov_len,
 			       v.iov_len),
 		memcpy_to_page(v.bv_page, v.bv_offset,
 			       (from += v.bv_len) - v.bv_len, v.bv_len),
@@ -560,7 +560,7 @@ size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
 		return 0;
 	}
 	iterate_and_advance(i, bytes, v,
-		__copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
+		__copy_from_user_inatomic((to += v.iov_len) - v.iov_len, v.iov_base,
 				 v.iov_len),
 		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 				 v.bv_offset, v.bv_len),
@@ -582,7 +582,7 @@ bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
 		return false;
 
 	iterate_all_kinds(i, bytes, v, ({
-		if (__copy_from_user((to += v.iov_len) - v.iov_len,
+		if (__copy_from_user_inatomic((to += v.iov_len) - v.iov_len,
 				      v.iov_base, v.iov_len))
 			return false;
 		0;}),