From: Felix Kuehling <Felix.Kuehling@amd.com>
To: amd-gfx@lists.freedesktop.org, dri-devel@lists.freedesktop.org
Cc: Philip Yang <Philip.Yang@amd.com>
Subject: [PATCH 11/34] drm/amdkfd: svm range eviction and restore
Date: Wed, 14 Apr 2021 21:23:14 -0400 [thread overview]
Message-ID: <20210415012337.1755-12-Felix.Kuehling@amd.com> (raw)
In-Reply-To: <20210415012337.1755-1-Felix.Kuehling@amd.com>
HMM interval notifier callback notify CPU page table will be updated,
stop process queues if the updated address belongs to svm range
registered in process svms objects tree. Scheduled restore work to
update GPU page table using new pages address in the updated svm range.
The restore worker flushes any deferred work to make sure it restores
an up-to-date svm_range_list.
Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 +
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 1 +
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 135 +++++++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 +
4 files changed, 140 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 0d19a13fc227..1b829eef9e50 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -738,6 +738,8 @@ struct svm_range_list {
struct work_struct deferred_list_work;
struct list_head deferred_range_list;
spinlock_t deferred_list_lock;
+ atomic_t evicted_ranges;
+ struct delayed_work restore_work;
};
/* Process data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 1f3d4f5c64a8..3bcde43ccd70 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1064,6 +1064,7 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
cancel_delayed_work_sync(&p->eviction_work);
cancel_delayed_work_sync(&p->restore_work);
+ cancel_delayed_work_sync(&p->svms.restore_work);
mutex_lock(&p->mutex);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 55ca39fe46a8..1ffdc2422d67 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -22,6 +22,7 @@
*/
#include <linux/types.h>
+#include <linux/sched/task.h>
#include "amdgpu_sync.h"
#include "amdgpu_object.h"
#include "amdgpu_vm.h"
@@ -29,6 +30,8 @@
#include "kfd_priv.h"
#include "kfd_svm.h"
+#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1
+
static bool
svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
const struct mmu_notifier_range *range,
@@ -251,6 +254,7 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
INIT_LIST_HEAD(&prange->insert_list);
INIT_LIST_HEAD(&prange->deferred_list);
INIT_LIST_HEAD(&prange->child_list);
+ atomic_set(&prange->invalid, 0);
mutex_init(&prange->lock);
svm_range_set_default_attributes(&prange->preferred_loc,
&prange->prefetch_loc,
@@ -963,6 +967,129 @@ svm_range_list_lock_and_flush_work(struct svm_range_list *svms,
goto retry_flush_work;
}
+static void svm_range_restore_work(struct work_struct *work)
+{
+ struct delayed_work *dwork = to_delayed_work(work);
+ struct amdkfd_process_info *process_info;
+ struct svm_range_list *svms;
+ struct svm_range *prange;
+ struct kfd_process *p;
+ struct mm_struct *mm;
+ int evicted_ranges;
+ int invalid;
+ int r;
+
+ svms = container_of(dwork, struct svm_range_list, restore_work);
+ evicted_ranges = atomic_read(&svms->evicted_ranges);
+ if (!evicted_ranges)
+ return;
+
+ pr_debug("restore svm ranges\n");
+
+ /* kfd_process_notifier_release destroys this worker thread. So during
+ * the lifetime of this thread, kfd_process and mm will be valid.
+ */
+ p = container_of(svms, struct kfd_process, svms);
+ process_info = p->kgd_process_info;
+ mm = p->mm;
+ if (!mm)
+ return;
+
+ mutex_lock(&process_info->lock);
+ svm_range_list_lock_and_flush_work(svms, mm);
+ mutex_lock(&svms->lock);
+
+ evicted_ranges = atomic_read(&svms->evicted_ranges);
+
+ list_for_each_entry(prange, &svms->list, list) {
+ invalid = atomic_read(&prange->invalid);
+ if (!invalid)
+ continue;
+
+ pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n",
+ prange->svms, prange, prange->start, prange->last,
+ invalid);
+
+ r = svm_range_validate_and_map(mm, prange, MAX_GPU_INSTANCE,
+ false, true);
+ if (r) {
+ pr_debug("failed %d to map 0x%lx to gpus\n", r,
+ prange->start);
+ goto unlock_out;
+ }
+
+ if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid)
+ goto unlock_out;
+ }
+
+ if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) !=
+ evicted_ranges)
+ goto unlock_out;
+
+ evicted_ranges = 0;
+
+ r = kgd2kfd_resume_mm(mm);
+ if (r) {
+ /* No recovery from this failure. Probably the CP is
+ * hanging. No point trying again.
+ */
+ pr_debug("failed %d to resume KFD\n", r);
+ }
+
+ pr_debug("restore svm ranges successfully\n");
+
+unlock_out:
+ mutex_unlock(&svms->lock);
+ mmap_write_unlock(mm);
+ mutex_unlock(&process_info->lock);
+
+ /* If validation failed, reschedule another attempt */
+ if (evicted_ranges) {
+ pr_debug("reschedule to restore svm range\n");
+ schedule_delayed_work(&svms->restore_work,
+ msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
+ }
+}
+
+/**
+ * svm_range_evict - evict svm range
+ *
+ * Stop all queues of the process to ensure GPU doesn't access the memory, then
+ * return to let CPU evict the buffer and proceed CPU pagetable update.
+ *
+ * Don't need use lock to sync cpu pagetable invalidation with GPU execution.
+ * If invalidation happens while restore work is running, restore work will
+ * restart to ensure to get the latest CPU pages mapping to GPU, then start
+ * the queues.
+ */
+static int
+svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
+ unsigned long start, unsigned long last)
+{
+ struct svm_range_list *svms = prange->svms;
+ int evicted_ranges;
+ int r = 0;
+
+ atomic_inc(&prange->invalid);
+ evicted_ranges = atomic_inc_return(&svms->evicted_ranges);
+ if (evicted_ranges != 1)
+ return r;
+
+ pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n",
+ prange->svms, prange->start, prange->last);
+
+ /* First eviction, stop the queues */
+ r = kgd2kfd_quiesce_mm(mm);
+ if (r)
+ pr_debug("failed to quiesce KFD\n");
+
+ pr_debug("schedule to restore svm %p ranges\n", svms);
+ schedule_delayed_work(&svms->restore_work,
+ msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
+
+ return r;
+}
+
static struct svm_range *svm_range_clone(struct svm_range *old)
{
struct svm_range *new;
@@ -1331,6 +1458,11 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
* svm_range_cpu_invalidate_pagetables - interval notifier callback
*
* MMU range unmap notifier to remove svm ranges
+ *
+ * If GPU vm fault retry is not enabled, evict the svm range, then restore
+ * work will update GPU mapping.
+ * If GPU vm fault retry is enabled, unmap the svm range from GPU, vm fault
+ * will update GPU mapping.
*/
static bool
svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
@@ -1364,6 +1496,7 @@ svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
svm_range_unmap_from_cpu(mni->mm, prange, start, last);
break;
default:
+ svm_range_evict(prange, mni->mm, start, last);
break;
}
@@ -1389,6 +1522,8 @@ int svm_range_list_init(struct kfd_process *p)
svms->objects = RB_ROOT_CACHED;
mutex_init(&svms->lock);
INIT_LIST_HEAD(&svms->list);
+ atomic_set(&svms->evicted_ranges, 0);
+ INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work);
INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work);
INIT_LIST_HEAD(&svms->deferred_range_list);
spin_lock_init(&svms->deferred_list_lock);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index 5949890bf48c..3c94899c5c40 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -67,6 +67,7 @@ struct svm_work_list_item {
* @perfetch_loc: last prefetch location, 0 for CPU, or GPU id
* @actual_loc: the actual location, 0 for CPU, or GPU id
* @granularity:migration granularity, log2 num pages
+ * @invalid: not 0 means cpu page table is invalidated
* @notifier: register mmu interval notifier
* @work_item: deferred work item information
* @deferred_list: list header used to add range to deferred list
@@ -97,6 +98,7 @@ struct svm_range {
uint32_t prefetch_loc;
uint32_t actual_loc;
uint8_t granularity;
+ atomic_t invalid;
struct mmu_interval_notifier notifier;
struct svm_work_list_item work_item;
struct list_head deferred_list;
--
2.31.1
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
next prev parent reply other threads:[~2021-04-15 1:24 UTC|newest]
Thread overview: 36+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-15 1:23 [PATCH 00/34] Add HMM-based SVM memory manager to KFD v5 Felix Kuehling
2021-04-15 1:23 ` [PATCH 01/34] drm/amdkfd: helper to convert gpu id and idx Felix Kuehling
2021-04-15 1:23 ` [PATCH 02/34] drm/amdkfd: add svm ioctl API Felix Kuehling
2021-04-15 1:23 ` [PATCH 03/34] drm/amdkfd: register svm range Felix Kuehling
2021-04-15 1:23 ` [PATCH 04/34] drm/amdkfd: add svm ioctl GET_ATTR op Felix Kuehling
2021-04-15 1:23 ` [PATCH 05/34] drm/amdgpu: add common HMM get pages function Felix Kuehling
2021-04-15 1:23 ` [PATCH 06/34] drm/amdkfd: support larger svm range allocation Felix Kuehling
2021-04-15 1:23 ` [PATCH 07/34] drm/amdkfd: validate svm range system memory Felix Kuehling
2021-04-15 1:23 ` [PATCH 08/34] drm/amdkfd: deregister svm range Felix Kuehling
2021-04-15 1:23 ` [PATCH 09/34] drm/amdgpu: export vm update mapping interface Felix Kuehling
2021-04-15 1:23 ` [PATCH 10/34] drm/amdkfd: map svm range to GPUs Felix Kuehling
2021-04-15 1:23 ` Felix Kuehling [this message]
2021-04-15 1:23 ` [PATCH 12/34] drm/amdgpu: Enable retry faults unconditionally on Aldebaran Felix Kuehling
2021-04-15 1:23 ` [PATCH 13/34] drm/amdkfd: add xnack enabled flag to kfd_process Felix Kuehling
2021-04-15 1:23 ` [PATCH 14/34] drm/amdkfd: add ioctl to configure and query xnack retries Felix Kuehling
2021-04-15 1:23 ` [PATCH 15/34] drm/amdkfd: register HMM device private zone Felix Kuehling
2021-04-15 1:23 ` [PATCH 16/34] drm/amdkfd: validate vram svm range from TTM Felix Kuehling
2021-04-15 1:23 ` [PATCH 17/34] drm/amdkfd: support xgmi same hive mapping Felix Kuehling
2021-04-15 1:23 ` [PATCH 18/34] drm/amdkfd: copy memory through gart table Felix Kuehling
2021-04-15 1:23 ` [PATCH 19/34] drm/amdkfd: HMM migrate ram to vram Felix Kuehling
2021-04-15 1:23 ` [PATCH 20/34] drm/amdkfd: HMM migrate vram to ram Felix Kuehling
2021-04-15 1:23 ` [PATCH 21/34] drm/amdkfd: invalidate tables on page retry fault Felix Kuehling
2021-04-15 1:23 ` [PATCH 22/34] drm/amdgpu: enable 48-bit IH timestamp counter Felix Kuehling
2021-04-15 1:23 ` [PATCH 23/34] drm/amdkfd: page table restore through svm API Felix Kuehling
2021-04-15 1:23 ` [PATCH 24/34] drm/amdkfd: SVM API call to restore page tables Felix Kuehling
2021-04-15 1:23 ` [PATCH 25/34] drm/amdkfd: add svm_bo reference for eviction fence Felix Kuehling
2021-04-15 1:23 ` [PATCH 26/34] drm/amdgpu: add param bit flag to create SVM BOs Felix Kuehling
2021-04-15 1:23 ` [PATCH 27/34] drm/amdkfd: add svm_bo eviction mechanism support Felix Kuehling
2021-04-15 1:23 ` [PATCH 28/34] drm/amdgpu: svm bo enable_signal call condition Felix Kuehling
2021-04-15 1:23 ` [PATCH 29/34] drm/amdgpu: add svm_bo eviction to enable_signal cb Felix Kuehling
2021-04-15 1:23 ` [PATCH 30/34] drm/amdkfd: refine migration policy with xnack on Felix Kuehling
2021-04-15 1:23 ` [PATCH 31/34] drm/amdkfd: add svm range validate timestamp Felix Kuehling
2021-04-15 1:23 ` [PATCH 32/34] drm/amdkfd: multiple gpu migrate vram to vram Felix Kuehling
2021-04-15 1:23 ` [PATCH 33/34] drm/amdkfd: Add SVM API support capability bits Felix Kuehling
2021-04-15 1:23 ` [PATCH 34/34] drm/amdkfd: Add CONFIG_HSA_AMD_SVM Felix Kuehling
-- strict thread matches above, loose matches on Subject: below --
2021-04-06 1:45 [PATCH 00/34] Add HMM-based SVM memory manager to KFD v4 Felix Kuehling
2021-04-06 1:46 ` [PATCH 11/34] drm/amdkfd: svm range eviction and restore Felix Kuehling
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210415012337.1755-12-Felix.Kuehling@amd.com \
--to=felix.kuehling@amd.com \
--cc=Philip.Yang@amd.com \
--cc=amd-gfx@lists.freedesktop.org \
--cc=dri-devel@lists.freedesktop.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).