From: Alistair Popple <apopple@nvidia.com>
To: <akpm@linux-foundation.org>, <Felix.Kuehling@amd.com>,
<linux-mm@kvack.org>, <rcampbell@nvidia.com>,
<linux-ext4@vger.kernel.org>, <linux-xfs@vger.kernel.org>,
Alex Sierra <alex.sierra@amd.com>
Cc: <amd-gfx@lists.freedesktop.org>,
<dri-devel@lists.freedesktop.org>, <hch@lst.de>, <jgg@nvidia.com>,
<jglisse@redhat.com>, <willy@infradead.org>
Subject: Re: [PATCH v3 09/10] tools: update hmm-test to support device coherent type
Date: Thu, 20 Jan 2022 17:14:49 +1100 [thread overview]
Message-ID: <2346143.zzXRFGWsWK@nvdebian> (raw)
In-Reply-To: <20220110223201.31024-10-alex.sierra@amd.com>
On Tuesday, 11 January 2022 9:32:00 AM AEDT Alex Sierra wrote:
> Test cases such as migrate_fault and migrate_multiple, were modified to
> explicit migrate from device to sys memory without the need of page
> faults, when using device coherent type.
>
> Snapshot test case updated to read memory device type first and based
> on that, get the proper returned results migrate_ping_pong test case
Where is the migrate_ping_pong test? Did you perhaps forget to add it? :-)
> added to test explicit migration from device to sys memory for both
> private and coherent zone types.
>
> Helpers to migrate from device to sys memory and vicerversa
> were also added.
>
> Signed-off-by: Alex Sierra <alex.sierra@amd.com>
> ---
> v2:
> Set FIXTURE_VARIANT to add multiple device types to the FIXTURE. This
> will run all the tests for each device type (private and coherent) in
> case both existed during hmm-test driver probed.
> ---
> tools/testing/selftests/vm/hmm-tests.c | 122 ++++++++++++++++++++-----
> 1 file changed, 101 insertions(+), 21 deletions(-)
>
> diff --git a/tools/testing/selftests/vm/hmm-tests.c b/tools/testing/selftests/vm/hmm-tests.c
> index 864f126ffd78..8eb81dfba4b3 100644
> --- a/tools/testing/selftests/vm/hmm-tests.c
> +++ b/tools/testing/selftests/vm/hmm-tests.c
> @@ -44,6 +44,14 @@ struct hmm_buffer {
> int fd;
> uint64_t cpages;
> uint64_t faults;
> + int zone_device_type;
> +};
> +
> +enum {
> + HMM_PRIVATE_DEVICE_ONE,
> + HMM_PRIVATE_DEVICE_TWO,
> + HMM_COHERENCE_DEVICE_ONE,
> + HMM_COHERENCE_DEVICE_TWO,
> };
>
> #define TWOMEG (1 << 21)
> @@ -60,6 +68,21 @@ FIXTURE(hmm)
> unsigned int page_shift;
> };
>
> +FIXTURE_VARIANT(hmm)
> +{
> + int device_number;
> +};
> +
> +FIXTURE_VARIANT_ADD(hmm, hmm_device_private)
> +{
> + .device_number = HMM_PRIVATE_DEVICE_ONE,
> +};
> +
> +FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent)
> +{
> + .device_number = HMM_COHERENCE_DEVICE_ONE,
> +};
> +
> FIXTURE(hmm2)
> {
> int fd0;
> @@ -68,6 +91,24 @@ FIXTURE(hmm2)
> unsigned int page_shift;
> };
>
> +FIXTURE_VARIANT(hmm2)
> +{
> + int device_number0;
> + int device_number1;
> +};
> +
> +FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private)
> +{
> + .device_number0 = HMM_PRIVATE_DEVICE_ONE,
> + .device_number1 = HMM_PRIVATE_DEVICE_TWO,
> +};
> +
> +FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent)
> +{
> + .device_number0 = HMM_COHERENCE_DEVICE_ONE,
> + .device_number1 = HMM_COHERENCE_DEVICE_TWO,
> +};
> +
> static int hmm_open(int unit)
> {
> char pathname[HMM_PATH_MAX];
> @@ -81,12 +122,19 @@ static int hmm_open(int unit)
> return fd;
> }
>
> +static bool hmm_is_coherent_type(int dev_num)
> +{
> + return (dev_num >= HMM_COHERENCE_DEVICE_ONE);
> +}
> +
> FIXTURE_SETUP(hmm)
> {
> self->page_size = sysconf(_SC_PAGE_SIZE);
> self->page_shift = ffs(self->page_size) - 1;
>
> - self->fd = hmm_open(0);
> + self->fd = hmm_open(variant->device_number);
> + if (self->fd < 0 && hmm_is_coherent_type(variant->device_number))
> + SKIP(exit(0), "DEVICE_COHERENT not available");
> ASSERT_GE(self->fd, 0);
> }
>
> @@ -95,9 +143,11 @@ FIXTURE_SETUP(hmm2)
> self->page_size = sysconf(_SC_PAGE_SIZE);
> self->page_shift = ffs(self->page_size) - 1;
>
> - self->fd0 = hmm_open(0);
> + self->fd0 = hmm_open(variant->device_number0);
> + if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0))
> + SKIP(exit(0), "DEVICE_COHERENT not available");
> ASSERT_GE(self->fd0, 0);
> - self->fd1 = hmm_open(1);
> + self->fd1 = hmm_open(variant->device_number1);
> ASSERT_GE(self->fd1, 0);
> }
>
> @@ -144,6 +194,7 @@ static int hmm_dmirror_cmd(int fd,
> }
> buffer->cpages = cmd.cpages;
> buffer->faults = cmd.faults;
> + buffer->zone_device_type = cmd.zone_device_type;
>
> return 0;
> }
> @@ -211,6 +262,20 @@ static void hmm_nanosleep(unsigned int n)
> nanosleep(&t, NULL);
> }
>
> +static int hmm_migrate_sys_to_dev(int fd,
> + struct hmm_buffer *buffer,
> + unsigned long npages)
> +{
> + return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages);
> +}
> +
> +static int hmm_migrate_dev_to_sys(int fd,
> + struct hmm_buffer *buffer,
> + unsigned long npages)
> +{
> + return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages);
> +}
> +
> /*
> * Simple NULL test of device open/close.
> */
> @@ -875,7 +940,7 @@ TEST_F(hmm, migrate)
> ptr[i] = i;
>
> /* Migrate memory to device. */
> - ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages);
> + ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
> ASSERT_EQ(ret, 0);
> ASSERT_EQ(buffer->cpages, npages);
>
> @@ -923,7 +988,7 @@ TEST_F(hmm, migrate_fault)
> ptr[i] = i;
>
> /* Migrate memory to device. */
> - ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages);
> + ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
> ASSERT_EQ(ret, 0);
> ASSERT_EQ(buffer->cpages, npages);
>
> @@ -936,7 +1001,7 @@ TEST_F(hmm, migrate_fault)
> ASSERT_EQ(ptr[i], i);
>
> /* Migrate memory to the device again. */
> - ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages);
> + ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
> ASSERT_EQ(ret, 0);
> ASSERT_EQ(buffer->cpages, npages);
>
> @@ -976,7 +1041,7 @@ TEST_F(hmm, migrate_shared)
> ASSERT_NE(buffer->ptr, MAP_FAILED);
>
> /* Migrate memory to device. */
> - ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages);
> + ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
> ASSERT_EQ(ret, -ENOENT);
>
> hmm_buffer_free(buffer);
> @@ -1015,7 +1080,7 @@ TEST_F(hmm2, migrate_mixed)
> p = buffer->ptr;
>
> /* Migrating a protected area should be an error. */
> - ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, npages);
> + ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
> ASSERT_EQ(ret, -EINVAL);
>
> /* Punch a hole after the first page address. */
> @@ -1023,7 +1088,7 @@ TEST_F(hmm2, migrate_mixed)
> ASSERT_EQ(ret, 0);
>
> /* We expect an error if the vma doesn't cover the range. */
> - ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 3);
> + ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3);
> ASSERT_EQ(ret, -EINVAL);
>
> /* Page 2 will be a read-only zero page. */
> @@ -1055,13 +1120,13 @@ TEST_F(hmm2, migrate_mixed)
>
> /* Now try to migrate pages 2-5 to device 1. */
> buffer->ptr = p + 2 * self->page_size;
> - ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 4);
> + ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4);
> ASSERT_EQ(ret, 0);
> ASSERT_EQ(buffer->cpages, 4);
>
> /* Page 5 won't be migrated to device 0 because it's on device 1. */
> buffer->ptr = p + 5 * self->page_size;
> - ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_MIGRATE, buffer, 1);
> + ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
> ASSERT_EQ(ret, -ENOENT);
> buffer->ptr = p;
>
> @@ -1070,8 +1135,12 @@ TEST_F(hmm2, migrate_mixed)
> }
>
> /*
> - * Migrate anonymous memory to device private memory and fault it back to system
> - * memory multiple times.
> + * Migrate anonymous memory to device memory and back to system memory
> + * multiple times. In case of private zone configuration, this is done
> + * through fault pages accessed by CPU. In case of coherent zone configuration,
> + * the pages from the device should be explicitly migrated back to system memory.
> + * The reason is Coherent device zone has coherent access by CPU, therefore
> + * it will not generate any page fault.
> */
> TEST_F(hmm, migrate_multiple)
> {
> @@ -1107,8 +1176,7 @@ TEST_F(hmm, migrate_multiple)
> ptr[i] = i;
>
> /* Migrate memory to device. */
> - ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer,
> - npages);
> + ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
> ASSERT_EQ(ret, 0);
> ASSERT_EQ(buffer->cpages, npages);
>
> @@ -1116,7 +1184,12 @@ TEST_F(hmm, migrate_multiple)
> for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
> ASSERT_EQ(ptr[i], i);
>
> - /* Fault pages back to system memory and check them. */
> + /* Migrate back to system memory and check them. */
> + if (hmm_is_coherent_type(variant->device_number)) {
> + ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);
So I think this will still pass even if nothing migrates so as mentioned on
the previous patch I think we need to check for the number of pages that
actually migrated. Alternatively I suppose you could do a snapshot and check
that, but that seems like it would be harder. Otherwise I think this looks
good.
> + ASSERT_EQ(ret, 0);
> + }
> +
> for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
> ASSERT_EQ(ptr[i], i);
>
> @@ -1312,13 +1385,13 @@ TEST_F(hmm2, snapshot)
>
> /* Page 5 will be migrated to device 0. */
> buffer->ptr = p + 5 * self->page_size;
> - ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_MIGRATE, buffer, 1);
> + ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
> ASSERT_EQ(ret, 0);
> ASSERT_EQ(buffer->cpages, 1);
>
> /* Page 6 will be migrated to device 1. */
> buffer->ptr = p + 6 * self->page_size;
> - ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 1);
> + ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1);
> ASSERT_EQ(ret, 0);
> ASSERT_EQ(buffer->cpages, 1);
>
> @@ -1335,9 +1408,16 @@ TEST_F(hmm2, snapshot)
> ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ);
> ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ);
> ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE);
> - ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |
> - HMM_DMIRROR_PROT_WRITE);
> - ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);
> + if (!hmm_is_coherent_type(variant->device_number0)) {
> + ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |
> + HMM_DMIRROR_PROT_WRITE);
> + ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);
> + } else {
> + ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL |
> + HMM_DMIRROR_PROT_WRITE);
> + ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE |
> + HMM_DMIRROR_PROT_WRITE);
> + }
>
> hmm_buffer_free(buffer);
> }
>
next prev parent reply other threads:[~2022-01-20 6:15 UTC|newest]
Thread overview: 24+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-01-10 22:31 [PATCH v3 00/10] Add MEMORY_DEVICE_COHERENT for coherent device memory mapping Alex Sierra
2022-01-10 22:31 ` [PATCH v3 01/10] mm: add zone device coherent type memory support Alex Sierra
2022-01-20 4:08 ` Alistair Popple
2022-01-10 22:31 ` [PATCH v3 02/10] mm: add device coherent vma selection for memory migration Alex Sierra
2022-01-10 22:31 ` [PATCH v3 03/10] mm/gup: fail get_user_pages for LONGTERM dev coherent type Alex Sierra
2022-01-20 12:36 ` Joao Martins
2022-01-20 13:18 ` Alistair Popple
2022-01-10 22:31 ` [PATCH v3 04/10] drm/amdkfd: add SPM support for SVM Alex Sierra
2022-01-10 22:31 ` [PATCH v3 05/10] drm/amdkfd: coherent type as sys mem on migration to ram Alex Sierra
2022-01-10 22:31 ` [PATCH v3 06/10] lib: test_hmm add ioctl to get zone device type Alex Sierra
2022-01-20 5:01 ` Alistair Popple
2022-01-10 22:31 ` [PATCH v3 07/10] lib: test_hmm add module param for " Alex Sierra
2022-01-20 5:23 ` Alistair Popple
2022-01-10 22:31 ` [PATCH v3 08/10] lib: add support for device coherent type in test_hmm Alex Sierra
2022-01-20 6:00 ` Alistair Popple
2022-01-10 22:32 ` [PATCH v3 09/10] tools: update hmm-test to support device coherent type Alex Sierra
2022-01-20 6:14 ` Alistair Popple [this message]
2022-01-27 3:22 ` Sierra Guiza, Alejandro (Alex)
2022-01-10 22:32 ` [PATCH v3 10/10] tools: update test_hmm script to support SP config Alex Sierra
2022-01-20 6:17 ` Alistair Popple
2022-01-12 11:06 ` [PATCH v3 00/10] Add MEMORY_DEVICE_COHERENT for coherent device memory mapping Alistair Popple
2022-01-20 6:33 ` Alistair Popple
2022-01-12 11:16 ` David Hildenbrand
2022-01-12 16:08 ` Felix Kuehling
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=2346143.zzXRFGWsWK@nvdebian \
--to=apopple@nvidia.com \
--cc=Felix.Kuehling@amd.com \
--cc=akpm@linux-foundation.org \
--cc=alex.sierra@amd.com \
--cc=amd-gfx@lists.freedesktop.org \
--cc=dri-devel@lists.freedesktop.org \
--cc=hch@lst.de \
--cc=jgg@nvidia.com \
--cc=jglisse@redhat.com \
--cc=linux-ext4@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-xfs@vger.kernel.org \
--cc=rcampbell@nvidia.com \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).