From: "Christian König" <christian.koenig@amd.com>
To: "Thomas Hellström" <thomas.hellstrom@linux.intel.com>,
intel-gfx@lists.freedesktop.org, dri-devel@lists.freedesktop.org
Subject: Re: [PATCH v7 07/15] drm: Add a prefetching memcpy_from_wc
Date: Mon, 31 May 2021 14:37:26 +0200 [thread overview]
Message-ID: <d77b1ae8-1062-ce5d-6bce-48d55717b304@amd.com> (raw)
In-Reply-To: <20210531121940.267032-8-thomas.hellstrom@linux.intel.com>
Am 31.05.21 um 14:19 schrieb Thomas Hellström:
> Reading out of write-combining mapped memory is typically very slow
> since the CPU doesn't prefetch. However some archs have special
> instructions to do this.
>
> So add a best-effort memcpy_from_wc taking dma-buf-map pointer
> arguments that attempts to use a fast prefetching memcpy and
> otherwise falls back to ordinary memcopies, taking the iomem tagging
> into account.
>
> The code is largely copied from i915_memcpy_from_wc.
>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Cc: Christian König <christian.koenig@amd.com>
> Suggested-by: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Acked-by: Christian König <christian.koenig@amd.com>
> ---
> v7:
> - Perform a memcpy even if warning with in_interrupt(). Suggested by
> Christian König.
> - Fix compilation failure on !X86 (Reported by kernel test robot
> lkp@intel.com)
> ---
> Documentation/gpu/drm-mm.rst | 2 +-
> drivers/gpu/drm/drm_cache.c | 147 +++++++++++++++++++++++++++++++++++
> drivers/gpu/drm/drm_drv.c | 2 +
> include/drm/drm_cache.h | 7 ++
> 4 files changed, 157 insertions(+), 1 deletion(-)
>
> diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst
> index 21be6deadc12..c66058c5bce7 100644
> --- a/Documentation/gpu/drm-mm.rst
> +++ b/Documentation/gpu/drm-mm.rst
> @@ -469,7 +469,7 @@ DRM MM Range Allocator Function References
> .. kernel-doc:: drivers/gpu/drm/drm_mm.c
> :export:
>
> -DRM Cache Handling
> +DRM Cache Handling and Fast WC memcpy()
> ==================
>
> .. kernel-doc:: drivers/gpu/drm/drm_cache.c
> diff --git a/drivers/gpu/drm/drm_cache.c b/drivers/gpu/drm/drm_cache.c
> index 79a50ef1250f..b887d7dec8b8 100644
> --- a/drivers/gpu/drm/drm_cache.c
> +++ b/drivers/gpu/drm/drm_cache.c
> @@ -28,6 +28,7 @@
> * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
> */
>
> +#include <linux/dma-buf-map.h>
> #include <linux/export.h>
> #include <linux/highmem.h>
> #include <linux/mem_encrypt.h>
> @@ -35,6 +36,9 @@
>
> #include <drm/drm_cache.h>
>
> +/* A small bounce buffer that fits on the stack. */
> +#define MEMCPY_BOUNCE_SIZE 128
> +
> #if defined(CONFIG_X86)
> #include <asm/smp.h>
>
> @@ -209,3 +213,146 @@ bool drm_need_swiotlb(int dma_bits)
> return max_iomem > ((u64)1 << dma_bits);
> }
> EXPORT_SYMBOL(drm_need_swiotlb);
> +
> +static void memcpy_fallback(struct dma_buf_map *dst,
> + const struct dma_buf_map *src,
> + unsigned long len)
> +{
> + if (!dst->is_iomem && !src->is_iomem) {
> + memcpy(dst->vaddr, src->vaddr, len);
> + } else if (!src->is_iomem) {
> + dma_buf_map_memcpy_to(dst, src->vaddr, len);
> + } else if (!dst->is_iomem) {
> + memcpy_fromio(dst->vaddr, src->vaddr_iomem, len);
> + } else {
> + /*
> + * Bounce size is not performance tuned, but using a
> + * bounce buffer like this is significantly faster than
> + * resorting to ioreadxx() + iowritexx().
> + */
> + char bounce[MEMCPY_BOUNCE_SIZE];
> + void __iomem *_src = src->vaddr_iomem;
> + void __iomem *_dst = dst->vaddr_iomem;
> +
> + while (len >= MEMCPY_BOUNCE_SIZE) {
> + memcpy_fromio(bounce, _src, MEMCPY_BOUNCE_SIZE);
> + memcpy_toio(_dst, bounce, MEMCPY_BOUNCE_SIZE);
> + _src += MEMCPY_BOUNCE_SIZE;
> + _dst += MEMCPY_BOUNCE_SIZE;
> + len -= MEMCPY_BOUNCE_SIZE;
> + }
> + if (len) {
> + memcpy_fromio(bounce, _src, MEMCPY_BOUNCE_SIZE);
> + memcpy_toio(_dst, bounce, MEMCPY_BOUNCE_SIZE);
> + }
> + }
> +}
> +
> +#ifdef CONFIG_X86
> +
> +static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
> +
> +static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
> +{
> + kernel_fpu_begin();
> +
> + while (len >= 4) {
> + asm("movntdqa (%0), %%xmm0\n"
> + "movntdqa 16(%0), %%xmm1\n"
> + "movntdqa 32(%0), %%xmm2\n"
> + "movntdqa 48(%0), %%xmm3\n"
> + "movaps %%xmm0, (%1)\n"
> + "movaps %%xmm1, 16(%1)\n"
> + "movaps %%xmm2, 32(%1)\n"
> + "movaps %%xmm3, 48(%1)\n"
> + :: "r" (src), "r" (dst) : "memory");
> + src += 64;
> + dst += 64;
> + len -= 4;
> + }
> + while (len--) {
> + asm("movntdqa (%0), %%xmm0\n"
> + "movaps %%xmm0, (%1)\n"
> + :: "r" (src), "r" (dst) : "memory");
> + src += 16;
> + dst += 16;
> + }
> +
> + kernel_fpu_end();
> +}
> +
> +/*
> + * __drm_memcpy_from_wc copies @len bytes from @src to @dst using
> + * non-temporal instructions where available. Note that all arguments
> + * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
> + * of 16.
> + */
> +static void __drm_memcpy_from_wc(void *dst, const void *src, unsigned long len)
> +{
> + if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
> + memcpy(dst, src, len);
> + else if (likely(len))
> + __memcpy_ntdqa(dst, src, len >> 4);
> +}
> +
> +/**
> + * drm_memcpy_from_wc - Perform the fastest available memcpy from a source
> + * that may be WC.
> + * @dst: The destination pointer
> + * @src: The source pointer
> + * @len: The size of the area o transfer in bytes
> + *
> + * Tries an arch optimized memcpy for prefetching reading out of a WC region,
> + * and if no such beast is available, falls back to a normal memcpy.
> + */
> +void drm_memcpy_from_wc(struct dma_buf_map *dst,
> + const struct dma_buf_map *src,
> + unsigned long len)
> +{
> + if (WARN_ON(in_interrupt())) {
> + memcpy_fallback(dst, src, len);
> + return;
> + }
> +
> + if (static_branch_likely(&has_movntdqa)) {
> + __drm_memcpy_from_wc(dst->is_iomem ?
> + (void __force *)dst->vaddr_iomem :
> + dst->vaddr,
> + src->is_iomem ?
> + (void const __force *)src->vaddr_iomem :
> + src->vaddr,
> + len);
> + return;
> + }
> +
> + memcpy_fallback(dst, src, len);
> +}
> +EXPORT_SYMBOL(drm_memcpy_from_wc);
> +
> +/**
> + * drm_memcpy_init_early - One time initialization of the WC memcpy code
> + */
> +void drm_memcpy_init_early(void)
> +{
> + /*
> + * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions
> + * emulation. So don't enable movntdqa in hypervisor guest.
> + */
> + if (static_cpu_has(X86_FEATURE_XMM4_1) &&
> + !boot_cpu_has(X86_FEATURE_HYPERVISOR))
> + static_branch_enable(&has_movntdqa);
> +}
> +#else
> +void drm_memcpy_from_wc(struct dma_buf_map *dst,
> + const struct dma_buf_map *src,
> + unsigned long len)
> +{
> + WARN_ON(in_interrupt());
> +
> + memcpy_fallback(dst, src, len);
> +}
> +
> +void drm_memcpy_init_early(void)
> +{
> +}
> +#endif /* CONFIG_X86 */
> diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
> index 3d8d68a98b95..8804ec7d3215 100644
> --- a/drivers/gpu/drm/drm_drv.c
> +++ b/drivers/gpu/drm/drm_drv.c
> @@ -35,6 +35,7 @@
> #include <linux/slab.h>
> #include <linux/srcu.h>
>
> +#include <drm/drm_cache.h>
> #include <drm/drm_client.h>
> #include <drm/drm_color_mgmt.h>
> #include <drm/drm_drv.h>
> @@ -1041,6 +1042,7 @@ static int __init drm_core_init(void)
>
> drm_connector_ida_init();
> idr_init(&drm_minors_idr);
> + drm_memcpy_init_early();
>
> ret = drm_sysfs_init();
> if (ret < 0) {
> diff --git a/include/drm/drm_cache.h b/include/drm/drm_cache.h
> index e9ad4863d915..cc9de1632dd3 100644
> --- a/include/drm/drm_cache.h
> +++ b/include/drm/drm_cache.h
> @@ -35,6 +35,8 @@
>
> #include <linux/scatterlist.h>
>
> +struct dma_buf_map;
> +
> void drm_clflush_pages(struct page *pages[], unsigned long num_pages);
> void drm_clflush_sg(struct sg_table *st);
> void drm_clflush_virt_range(void *addr, unsigned long length);
> @@ -70,4 +72,9 @@ static inline bool drm_arch_can_wc_memory(void)
> #endif
> }
>
> +void drm_memcpy_init_early(void);
> +
> +void drm_memcpy_from_wc(struct dma_buf_map *dst,
> + const struct dma_buf_map *src,
> + unsigned long len);
> #endif
next prev parent reply other threads:[~2021-05-31 12:37 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-31 12:19 [PATCH v7 00/15] Move LMEM (VRAM) management over to TTM Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 01/15] drm/i915: Untangle the vma pages_mutex Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 02/15] drm/i915: Don't free shared locks while shared Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 03/15] drm/i915: Fix i915_sg_page_sizes to record dma segments rather than physical pages Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 04/15] drm/i915/ttm Initialize the ttm device and memory managers Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 05/15] drm/i915/ttm: Embed a ttm buffer object in the i915 gem object Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 06/15] drm/ttm: Add a generic TTM memcpy move for page-based iomem Thomas Hellström
2021-05-31 12:36 ` Christian König
2021-05-31 13:08 ` Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 07/15] drm: Add a prefetching memcpy_from_wc Thomas Hellström
2021-05-31 12:37 ` Christian König [this message]
2021-05-31 12:19 ` [PATCH v7 08/15] drm/ttm: Use drm_memcpy_from_wc for TTM bo moves Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 09/15] drm/ttm: Document and optimize ttm_bo_pipeline_gutting() Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 10/15] drm/ttm, drm/amdgpu: Allow the driver some control over swapping Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 11/15] drm/i915/ttm: Introduce a TTM i915 gem object backend Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 12/15] drm/i915/lmem: Verify checks for lmem residency Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 13/15] drm/i915: Disable mmap ioctl for gen12+ Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 14/15] drm/vma: Add a driver_private member to vma_node Thomas Hellström
2021-05-31 12:19 ` [PATCH v7 15/15] drm/i915: Use ttm mmap handling for ttm bo's Thomas Hellström
2021-05-31 17:17 ` Thomas Hellström (Intel)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=d77b1ae8-1062-ce5d-6bce-48d55717b304@amd.com \
--to=christian.koenig@amd.com \
--cc=dri-devel@lists.freedesktop.org \
--cc=intel-gfx@lists.freedesktop.org \
--cc=thomas.hellstrom@linux.intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).