From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:37838) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1bjumh-0004UW-R1 for qemu-devel@nongnu.org; Tue, 13 Sep 2016 16:58:29 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1bjumc-000323-Ny for qemu-devel@nongnu.org; Tue, 13 Sep 2016 16:58:26 -0400 Received: from mail-pa0-f66.google.com ([209.85.220.66]:36029) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1bjumc-00031u-Fq for qemu-devel@nongnu.org; Tue, 13 Sep 2016 16:58:22 -0400 Received: by mail-pa0-f66.google.com with SMTP id p2so54307pap.3 for ; Tue, 13 Sep 2016 13:58:22 -0700 (PDT) Sender: Richard Henderson From: Richard Henderson Date: Tue, 13 Sep 2016 13:57:19 -0700 Message-Id: <1473800239-13841-1-git-send-email-rth@twiddle.net> Subject: [Qemu-devel] [PATCH v5] cutils: Rewrite x86 buffer zero checking List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: qemu-devel@nongnu.org Cc: Paolo Bonzini Handle alignment of buffers, so that the vector paths can be used more often. Signed-off-by: Richard Henderson Signed-off-by: Paolo Bonzini --- util/bufferiszero.c | 236 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 162 insertions(+), 74 deletions(-) Tested with Fedora 24 (gcc 6.1), CentOS 7 (gcc 4.8), and Centos 6 (gcc 4.4). Added some commentary to record some of the conversation I had with Paolo regarding the fixups required for gcc 4.8. Added a test for bit_AVX2, as required by gcc 4.4. r~ diff --git a/util/bufferiszero.c b/util/bufferiszero.c index 4d8a8c8..4c7d09d 100644 --- a/util/bufferiszero.c +++ b/util/bufferiszero.c @@ -26,38 +26,6 @@ #include "qemu/cutils.h" #include "qemu/bswap.h" - -/* vector definitions */ - -extern void link_error(void); - -#define ACCEL_BUFFER_ZERO(NAME, SIZE, VECTYPE, NONZERO) \ -static bool NAME(const void *buf, size_t len) \ -{ \ - const void *end = buf + len; \ - do { \ - const VECTYPE *p = buf; \ - VECTYPE t; \ - __builtin_prefetch(buf + SIZE); \ - barrier(); \ - if (SIZE == sizeof(VECTYPE) * 4) { \ - t = (p[0] | p[1]) | (p[2] | p[3]); \ - } else if (SIZE == sizeof(VECTYPE) * 8) { \ - t = p[0] | p[1]; \ - t |= p[2] | p[3]; \ - t |= p[4] | p[5]; \ - t |= p[6] | p[7]; \ - } else { \ - link_error(); \ - } \ - if (unlikely(NONZERO(t))) { \ - return false; \ - } \ - buf += SIZE; \ - } while (buf < end); \ - return true; \ -} - static bool buffer_zero_int(const void *buf, size_t len) { @@ -96,40 +64,172 @@ buffer_zero_int(const void *buf, size_t len) } } -#if defined(CONFIG_AVX2_OPT) || (defined(CONFIG_CPUID_H) && defined(__SSE2__)) -#include - +#if (defined(CONFIG_AVX2_OPT) && defined(CONFIG_CPUID_H)) || defined(__SSE2__) +#ifdef CONFIG_AVX2_OPT #pragma GCC push_options #pragma GCC target("sse2") +#endif #include -#define SSE2_NONZERO(X) \ - (_mm_movemask_epi8(_mm_cmpeq_epi8((X), _mm_setzero_si128())) != 0xFFFF) -ACCEL_BUFFER_ZERO(buffer_zero_sse2, 64, __m128i, SSE2_NONZERO) -#pragma GCC pop_options + +/* Note that each of these vectorized functions require len >= 64. */ + +static bool +buffer_zero_sse2(const void *buf, size_t len) +{ + __m128i t = _mm_loadu_si128(buf); + __m128i *p = (__m128i *)(((uintptr_t)buf + 5 * 16) & -16); + __m128i *e = (__m128i *)(((uintptr_t)buf + len) & -16); + __m128i zero = _mm_setzero_si128(); + + /* Loop over 16-byte aligned blocks of 64. */ + while (likely(p <= e)) { + __builtin_prefetch(p); + t = _mm_cmpeq_epi8(t, zero); + if (unlikely(_mm_movemask_epi8(t) != 0xFFFF)) { + return false; + } + t = p[-4] | p[-3] | p[-2] | p[-1]; + p += 4; + } + + /* Finish the aligned tail. */ + t |= e[-3]; + t |= e[-2]; + t |= e[-1]; + + /* Finish the unaligned tail. */ + t |= _mm_loadu_si128(buf + len - 16); + + return _mm_movemask_epi8(_mm_cmpeq_epi8(t, zero)) == 0xFFFF; +} #ifdef CONFIG_AVX2_OPT +/* Note that due to restrictions/bugs wrt __builtin functions in gcc <= 4.8, + * the includes have to be within the corresponding push_options region, and + * therefore the regions themselves have to be ordered with increasing ISA. + */ +#pragma GCC pop_options #pragma GCC push_options #pragma GCC target("sse4") #include -#define SSE4_NONZERO(X) !_mm_testz_si128((X), (X)) -ACCEL_BUFFER_ZERO(buffer_zero_sse4, 64, __m128i, SSE4_NONZERO) -#pragma GCC pop_options +static bool +buffer_zero_sse4(const void *buf, size_t len) +{ + __m128i t = _mm_loadu_si128(buf); + __m128i *p = (__m128i *)(((uintptr_t)buf + 5 * 16) & -16); + __m128i *e = (__m128i *)(((uintptr_t)buf + len) & -16); + + /* Loop over 16-byte aligned blocks of 64. */ + while (likely(p <= e)) { + __builtin_prefetch(p); + if (unlikely(!_mm_testz_si128(t, t))) { + return false; + } + t = p[-4] | p[-3] | p[-2] | p[-1]; + p += 4; + } + + /* Finish the aligned tail. */ + t |= e[-3]; + t |= e[-2]; + t |= e[-1]; + + /* Finish the unaligned tail. */ + t |= _mm_loadu_si128(buf + len - 16); + + return _mm_testz_si128(t, t); +} + +#pragma GCC pop_options #pragma GCC push_options #pragma GCC target("avx2") #include -#define AVX2_NONZERO(X) !_mm256_testz_si256((X), (X)) -ACCEL_BUFFER_ZERO(buffer_zero_avx2, 128, __m256i, AVX2_NONZERO) + +static bool +buffer_zero_avx2(const void *buf, size_t len) +{ + /* Begin with an unaligned head of 32 bytes. */ + __m256i t = _mm256_loadu_si256(buf); + __m256i *p = (__m256i *)(((uintptr_t)buf + 5 * 32) & -32); + __m256i *e = (__m256i *)(((uintptr_t)buf + len) & -32); + + if (likely(p <= e)) { + /* Loop over 32-byte aligned blocks of 128. */ + do { + __builtin_prefetch(p); + if (unlikely(!_mm256_testz_si256(t, t))) { + return false; + } + t = p[-4] | p[-3] | p[-2] | p[-1]; + p += 4; + } while (p <= e); + } else { + t |= _mm256_loadu_si256(buf + 32); + if (len <= 128) { + goto last2; + } + } + + /* Finish the last block of 128 unaligned. */ + t |= _mm256_loadu_si256(buf + len - 4 * 32); + t |= _mm256_loadu_si256(buf + len - 3 * 32); + last2: + t |= _mm256_loadu_si256(buf + len - 2 * 32); + t |= _mm256_loadu_si256(buf + len - 1 * 32); + + return _mm256_testz_si256(t, t); +} #pragma GCC pop_options +#endif /* CONFIG_AVX2_OPT */ + +/* Note that for test_buffer_is_zero_next_accel, the most preferred + * ISA must have the least significant bit. + */ +#define CACHE_AVX2 1 +#define CACHE_SSE4 2 +#define CACHE_SSE2 4 + +/* Make sure that these variables are appropriately initialized when + * SSE2 is enabled on the compiler command-line, but the compiler is + * too old to support . + */ +#ifdef CONFIG_CPUID_H +# define INIT_CACHE +# define INIT_ACCEL +#else +# ifndef __SSE2__ +# error "ISA selection confusion" +# endif +# define INIT_CACHE = CACHE_SSE2 +# define INIT_ACCEL = buffer_zero_sse2 #endif -#define CACHE_AVX2 2 -#define CACHE_AVX1 4 -#define CACHE_SSE4 8 -#define CACHE_SSE2 16 +static unsigned cpuid_cache INIT_CACHE; +static bool (*buffer_accel)(const void *, size_t) INIT_ACCEL; -static unsigned cpuid_cache; +#undef INIT_CACHE +#undef INIT_ACCEL +static void init_accel(unsigned cache) +{ + bool (*fn)(const void *, size_t) = buffer_zero_int; + if (cache & CACHE_SSE2) { + fn = buffer_zero_sse2; + } +#ifdef CONFIG_AVX2_OPT + if (cache & CACHE_SSE4) { + fn = buffer_zero_sse4; + } + if (cache & CACHE_AVX2) { + fn = buffer_zero_avx2; + } +#endif + buffer_accel = fn; +} + +#ifdef CONFIG_CPUID_H +#include static void __attribute__((constructor)) init_cpuid_cache(void) { int max = __get_cpuid_max(0, NULL); @@ -145,24 +245,23 @@ static void __attribute__((constructor)) init_cpuid_cache(void) cache |= CACHE_SSE4; } +#ifdef bit_AVX2 /* We must check that AVX is not just available, but usable. */ - if ((c & bit_OSXSAVE) && (c & bit_AVX)) { - __asm("xgetbv" : "=a"(a), "=d"(d) : "c"(0)); - if ((a & 6) == 6) { - cache |= CACHE_AVX1; - if (max >= 7) { - __cpuid_count(7, 0, a, b, c, d); - if (b & bit_AVX2) { - cache |= CACHE_AVX2; - } - } + if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { + int bv; + __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0)); + __cpuid_count(7, 0, a, b, c, d); + if ((bv & 6) == 6 && (b & bit_AVX2)) { + cache |= CACHE_AVX2; } } +#endif } cpuid_cache = cache; + init_accel(cache); } +#endif /* CONFIG_CPUID_H */ -#define HAVE_NEXT_ACCEL bool test_buffer_is_zero_next_accel(void) { /* If no bits set, we just tested buffer_zero_int, and there @@ -172,31 +271,20 @@ bool test_buffer_is_zero_next_accel(void) } /* Disable the accelerator we used before and select a new one. */ cpuid_cache &= cpuid_cache - 1; + init_accel(cpuid_cache); return true; } static bool select_accel_fn(const void *buf, size_t len) { - uintptr_t ibuf = (uintptr_t)buf; -#ifdef CONFIG_AVX2_OPT - if (len % 128 == 0 && ibuf % 32 == 0 && (cpuid_cache & CACHE_AVX2)) { - return buffer_zero_avx2(buf, len); - } - if (len % 64 == 0 && ibuf % 16 == 0 && (cpuid_cache & CACHE_SSE4)) { - return buffer_zero_sse4(buf, len); - } -#endif - if (len % 64 == 0 && ibuf % 16 == 0 && (cpuid_cache & CACHE_SSE2)) { - return buffer_zero_sse2(buf, len); + if (likely(len >= 64)) { + return buffer_accel(buf, len); } return buffer_zero_int(buf, len); } #else #define select_accel_fn buffer_zero_int -#endif - -#ifndef HAVE_NEXT_ACCEL bool test_buffer_is_zero_next_accel(void) { return false; -- 2.7.4