* [PATCH 01/15] drm/i915: Hold irq-off for the entire fake lock period
2019-08-23 13:26 vm->mutex Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 02/15] drm/i915/gtt: Preallocate Braswell top-level page directory Chris Wilson
` (16 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Sadly lockdep records when the irqs are re-enabled and then marks up the
fake lock as being irq-unsafe. Our hand is forced and so we must mark up
the entire fake lock critical section as irq-off.
Hopefully this is the last tweak required!
v2: Not quite, we need to mark the timeline spinlock as irqsafe. That
was a genuine bug being hidden by the earlier lockdep splat.
Fixes: d67739268cf0 ("drm/i915/gt: Mark up the nested engine-pm timeline lock as irqsafe")
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Mika Kuoppala <mika.kuoppala@linux.intel.com>
Reviewed-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
---
drivers/gpu/drm/i915/gt/intel_engine_pm.c | 28 +++++++++++++++--------
drivers/gpu/drm/i915/gt/intel_reset.c | 9 ++++----
drivers/gpu/drm/i915/gt/intel_timeline.c | 10 ++++----
drivers/gpu/drm/i915/i915_gem.c | 15 ++++++------
drivers/gpu/drm/i915/i915_request.c | 9 ++++----
5 files changed, 42 insertions(+), 29 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
index a372d4ea9370..65b5ca74b394 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -39,27 +39,32 @@ static int __engine_unpark(struct intel_wakeref *wf)
#if IS_ENABLED(CONFIG_LOCKDEP)
-static inline void __timeline_mark_lock(struct intel_context *ce)
+static inline unsigned long __timeline_mark_lock(struct intel_context *ce)
{
unsigned long flags;
local_irq_save(flags);
mutex_acquire(&ce->timeline->mutex.dep_map, 2, 0, _THIS_IP_);
- local_irq_restore(flags);
+
+ return flags;
}
-static inline void __timeline_mark_unlock(struct intel_context *ce)
+static inline void __timeline_mark_unlock(struct intel_context *ce,
+ unsigned long flags)
{
mutex_release(&ce->timeline->mutex.dep_map, 0, _THIS_IP_);
+ local_irq_restore(flags);
}
#else
-static inline void __timeline_mark_lock(struct intel_context *ce)
+static inline unsigned long __timeline_mark_lock(struct intel_context *ce)
{
+ return 0;
}
-static inline void __timeline_mark_unlock(struct intel_context *ce)
+static inline void __timeline_mark_unlock(struct intel_context *ce,
+ unsigned long flags)
{
}
@@ -68,6 +73,8 @@ static inline void __timeline_mark_unlock(struct intel_context *ce)
static bool switch_to_kernel_context(struct intel_engine_cs *engine)
{
struct i915_request *rq;
+ unsigned long flags;
+ bool result = true;
/* Already inside the kernel context, safe to power down. */
if (engine->wakeref_serial == engine->serial)
@@ -89,12 +96,12 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine)
* retiring the last request, thus all rings should be empty and
* all timelines idle.
*/
- __timeline_mark_lock(engine->kernel_context);
+ flags = __timeline_mark_lock(engine->kernel_context);
rq = __i915_request_create(engine->kernel_context, GFP_NOWAIT);
if (IS_ERR(rq))
/* Context switch failed, hope for the best! Maybe reset? */
- return true;
+ goto out_unlock;
intel_timeline_enter(rq->timeline);
@@ -110,9 +117,10 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine)
__intel_wakeref_defer_park(&engine->wakeref);
__i915_request_queue(rq, NULL);
- __timeline_mark_unlock(engine->kernel_context);
-
- return false;
+ result = false;
+out_unlock:
+ __timeline_mark_unlock(engine->kernel_context, flags);
+ return result;
}
static int __engine_park(struct intel_wakeref *wf)
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index 077716442c90..b9d84d52e986 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -792,6 +792,7 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
{
struct intel_gt_timelines *timelines = >->timelines;
struct intel_timeline *tl;
+ unsigned long flags;
if (!test_bit(I915_WEDGED, >->reset.flags))
return true;
@@ -811,7 +812,7 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
*
* No more can be submitted until we reset the wedged bit.
*/
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
list_for_each_entry(tl, &timelines->active_list, link) {
struct i915_request *rq;
@@ -819,7 +820,7 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
if (!rq)
continue;
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
/*
* All internal dependencies (i915_requests) will have
@@ -832,10 +833,10 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
i915_request_put(rq);
/* Restart iteration after droping lock */
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
tl = list_entry(&timelines->active_list, typeof(*tl), link);
}
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
intel_gt_sanitize(gt, false);
diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c
index 02fbe11b671b..9cb01d9828f1 100644
--- a/drivers/gpu/drm/i915/gt/intel_timeline.c
+++ b/drivers/gpu/drm/i915/gt/intel_timeline.c
@@ -337,6 +337,7 @@ int intel_timeline_pin(struct intel_timeline *tl)
void intel_timeline_enter(struct intel_timeline *tl)
{
struct intel_gt_timelines *timelines = &tl->gt->timelines;
+ unsigned long flags;
lockdep_assert_held(&tl->mutex);
@@ -345,14 +346,15 @@ void intel_timeline_enter(struct intel_timeline *tl)
return;
GEM_BUG_ON(!tl->active_count); /* overflow? */
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
list_add(&tl->link, &timelines->active_list);
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
}
void intel_timeline_exit(struct intel_timeline *tl)
{
struct intel_gt_timelines *timelines = &tl->gt->timelines;
+ unsigned long flags;
lockdep_assert_held(&tl->mutex);
@@ -360,9 +362,9 @@ void intel_timeline_exit(struct intel_timeline *tl)
if (--tl->active_count)
return;
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
list_del(&tl->link);
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
/*
* Since this timeline is idle, all bariers upon which we were waiting
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index eb31b69a316a..ec9a46c276de 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -889,12 +889,13 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915)
static long
wait_for_timelines(struct drm_i915_private *i915,
- unsigned int flags, long timeout)
+ unsigned int wait, long timeout)
{
struct intel_gt_timelines *timelines = &i915->gt.timelines;
struct intel_timeline *tl;
+ unsigned long flags;
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
list_for_each_entry(tl, &timelines->active_list, link) {
struct i915_request *rq;
@@ -902,7 +903,7 @@ wait_for_timelines(struct drm_i915_private *i915,
if (!rq)
continue;
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
/*
* "Race-to-idle".
@@ -913,19 +914,19 @@ wait_for_timelines(struct drm_i915_private *i915,
* want to complete as quickly as possible to avoid prolonged
* stalls, so allow the gpu to boost to maximum clocks.
*/
- if (flags & I915_WAIT_FOR_IDLE_BOOST)
+ if (wait & I915_WAIT_FOR_IDLE_BOOST)
gen6_rps_boost(rq);
- timeout = i915_request_wait(rq, flags, timeout);
+ timeout = i915_request_wait(rq, wait, timeout);
i915_request_put(rq);
if (timeout < 0)
return timeout;
/* restart after reacquiring the lock */
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
tl = list_entry(&timelines->active_list, typeof(*tl), link);
}
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
return timeout;
}
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index f1a0a57fc6fc..a53777dd371c 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -1465,9 +1465,10 @@ bool i915_retire_requests(struct drm_i915_private *i915)
{
struct intel_gt_timelines *timelines = &i915->gt.timelines;
struct intel_timeline *tl, *tn;
+ unsigned long flags;
LIST_HEAD(free);
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
if (!mutex_trylock(&tl->mutex))
continue;
@@ -1475,11 +1476,11 @@ bool i915_retire_requests(struct drm_i915_private *i915)
intel_timeline_get(tl);
GEM_BUG_ON(!tl->active_count);
tl->active_count++; /* pin the list element */
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
retire_requests(tl);
- spin_lock(&timelines->lock);
+ spin_lock_irqsave(&timelines->lock, flags);
/* Resume iteration after dropping lock */
list_safe_reset_next(tl, tn, link);
@@ -1494,7 +1495,7 @@ bool i915_retire_requests(struct drm_i915_private *i915)
list_add(&tl->link, &free);
}
}
- spin_unlock(&timelines->lock);
+ spin_unlock_irqrestore(&timelines->lock, flags);
list_for_each_entry_safe(tl, tn, &free, link)
__intel_timeline_free(&tl->kref);
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 02/15] drm/i915/gtt: Preallocate Braswell top-level page directory
2019-08-23 13:26 vm->mutex Chris Wilson
2019-08-23 13:26 ` [PATCH 01/15] drm/i915: Hold irq-off for the entire fake lock period Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 14:14 ` [PATCH v3] Do we need to posting-read after ggtt insert-page? Chris Wilson
2019-08-23 14:14 ` [PATCH v3] drm/i915/gtt: Preallocate Braswell top-level page directory Chris Wilson
2019-08-23 13:26 ` [PATCH 03/15] drm/i915/selftests: Teach igt_gpu_fill_dw() to take intel_context Chris Wilson
` (15 subsequent siblings)
17 siblings, 2 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
In order for the Braswell top-level PD to remain the same from the time
of request construction to its submission onto HW, as we may be
asynchronously rewriting the page tables (thus changing the expected
register state after having already stored the old addresses in the
request), the top level PD must be preallocated.
So wave goodbye to our lazy allocation of those 4x2 pages.
v2: A little bit of write-flushing required (presumably it always has
been required, but now we are more susceptible and it is showing up!)
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Mika Kuoppala <mika.kuoppala@linux.intel.com>
Reviewed-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
---
drivers/gpu/drm/i915/gem/i915_gem_context.c | 8 ++-
drivers/gpu/drm/i915/gt/intel_lrc.c | 59 +--------------------
drivers/gpu/drm/i915/i915_gem_gtt.c | 10 ++--
3 files changed, 13 insertions(+), 64 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 1cdfe05514c3..863b5cc1ffa9 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -1003,12 +1003,18 @@ static int emit_ppgtt_update(struct i915_request *rq, void *data)
intel_ring_advance(rq, cs);
} else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+ int err;
+
+ /* Magic required to prevent forcewake errors! */
+ err = engine->emit_flush(rq, EMIT_INVALIDATE);
+ if (err)
+ return err;
cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
if (IS_ERR(cs))
return PTR_ERR(cs);
- *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES);
+ *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;;
for (i = GEN8_3LVL_PDPES; i--; ) {
const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index d42584439f51..d14569f7637d 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -1871,60 +1871,6 @@ static int gen8_emit_init_breadcrumb(struct i915_request *rq)
return 0;
}
-static int emit_pdps(struct i915_request *rq)
-{
- const struct intel_engine_cs * const engine = rq->engine;
- struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(rq->hw_context->vm);
- int err, i;
- u32 *cs;
-
- GEM_BUG_ON(intel_vgpu_active(rq->i915));
-
- /*
- * Beware ye of the dragons, this sequence is magic!
- *
- * Small changes to this sequence can cause anything from
- * GPU hangs to forcewake errors and machine lockups!
- */
-
- /* Flush any residual operations from the context load */
- err = engine->emit_flush(rq, EMIT_FLUSH);
- if (err)
- return err;
-
- /* Magic required to prevent forcewake errors! */
- err = engine->emit_flush(rq, EMIT_INVALIDATE);
- if (err)
- return err;
-
- cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
- if (IS_ERR(cs))
- return PTR_ERR(cs);
-
- /* Ensure the LRI have landed before we invalidate & continue */
- *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
- for (i = GEN8_3LVL_PDPES; i--; ) {
- const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
- u32 base = engine->mmio_base;
-
- *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
- *cs++ = upper_32_bits(pd_daddr);
- *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
- *cs++ = lower_32_bits(pd_daddr);
- }
- *cs++ = MI_NOOP;
-
- intel_ring_advance(rq, cs);
-
- /* Be doubly sure the LRI have landed before proceeding */
- err = engine->emit_flush(rq, EMIT_FLUSH);
- if (err)
- return err;
-
- /* Re-invalidate the TLB for luck */
- return engine->emit_flush(rq, EMIT_INVALIDATE);
-}
-
static int execlists_request_alloc(struct i915_request *request)
{
int ret;
@@ -1947,10 +1893,7 @@ static int execlists_request_alloc(struct i915_request *request)
*/
/* Unconditionally invalidate GPU caches and TLBs. */
- if (i915_vm_is_4lvl(request->hw_context->vm))
- ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
- else
- ret = emit_pdps(request);
+ ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 0b81e0b64393..43d0de15bb21 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -162,6 +162,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
pte_flags |= PTE_READ_ONLY;
vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
+ wmb();
return 0;
}
@@ -1420,6 +1421,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
set_pd_entry(pd, idx, pde);
atomic_inc(px_used(pde)); /* keep pinned */
}
+ wmb();
return 0;
}
@@ -1507,11 +1509,9 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
}
if (!i915_vm_is_4lvl(&ppgtt->vm)) {
- if (intel_vgpu_active(i915)) {
- err = gen8_preallocate_top_level_pdp(ppgtt);
- if (err)
- goto err_free_pd;
- }
+ err = gen8_preallocate_top_level_pdp(ppgtt);
+ if (err)
+ goto err_free_pd;
}
ppgtt->vm.insert_entries = gen8_ppgtt_insert;
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v3] Do we need to posting-read after ggtt insert-page?
2019-08-23 13:26 ` [PATCH 02/15] drm/i915/gtt: Preallocate Braswell top-level page directory Chris Wilson
@ 2019-08-23 14:14 ` Chris Wilson
2019-08-23 14:14 ` Chris Wilson
2019-08-23 14:14 ` [PATCH v3] drm/i915/gtt: Preallocate Braswell top-level page directory Chris Wilson
1 sibling, 1 reply; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 14:14 UTC (permalink / raw)
To: intel-gfx
See gem_set_tiling_vs_pwrite/vs_blit
---
drivers/gpu/drm/i915/i915_gem_gtt.c | 26 +++++++++-----------------
1 file changed, 9 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 2680b388dc33..a08cbbf16bac 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -127,6 +127,15 @@ static void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
* WCB of the writes into the GGTT before it triggers the invalidate.
*/
intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+
+ /*
+ * Make sure the internal GAM fifo has been cleared of all GTT
+ * writes before exiting stop_machine(). This guarantees that
+ * any aperture accesses waiting to start in another process
+ * cannot back up behind the GTT writes causing a hang.
+ * The register can be any arbitrary GAM register.
+ */
+ intel_uncore_posting_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
}
static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
@@ -2271,20 +2280,6 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
gen8_set_pte(>t_base[i], scratch_pte);
}
-static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
-{
- struct drm_i915_private *dev_priv = vm->i915;
-
- /*
- * Make sure the internal GAM fifo has been cleared of all GTT
- * writes before exiting stop_machine(). This guarantees that
- * any aperture accesses waiting to start in another process
- * cannot back up behind the GTT writes causing a hang.
- * The register can be any arbitrary GAM register.
- */
- POSTING_READ(GFX_FLSH_CNTL_GEN6);
-}
-
struct insert_page {
struct i915_address_space *vm;
dma_addr_t addr;
@@ -2297,7 +2292,6 @@ static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
struct insert_page *arg = _arg;
gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
- bxt_vtd_ggtt_wa(arg->vm);
return 0;
}
@@ -2325,7 +2319,6 @@ static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
struct insert_entries *arg = _arg;
gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags);
- bxt_vtd_ggtt_wa(arg->vm);
return 0;
}
@@ -2351,7 +2344,6 @@ static int bxt_vtd_ggtt_clear_range__cb(void *_arg)
struct clear_range *arg = _arg;
gen8_ggtt_clear_range(arg->vm, arg->start, arg->length);
- bxt_vtd_ggtt_wa(arg->vm);
return 0;
}
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v3] drm/i915/gtt: Preallocate Braswell top-level page directory
2019-08-23 13:26 ` [PATCH 02/15] drm/i915/gtt: Preallocate Braswell top-level page directory Chris Wilson
2019-08-23 14:14 ` [PATCH v3] Do we need to posting-read after ggtt insert-page? Chris Wilson
@ 2019-08-23 14:14 ` Chris Wilson
1 sibling, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 14:14 UTC (permalink / raw)
To: intel-gfx
In order for the Braswell top-level PD to remain the same from the time
of request construction to its submission onto HW, as we may be
asynchronously rewriting the page tables (thus changing the expected
register state after having already stored the old addresses in the
request), the top level PD must be preallocated.
So wave goodbye to our lazy allocation of those 4x2 pages.
v2: A little bit of write-flushing required (presumably it always has
been required, but now we are more susceptible and it is showing up!)
v3: Put back the forced-PD-reload on every batch, we can't survive
without it and explicitly marking the context for PD reload makes
Braswell turn nasty.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Mika Kuoppala <mika.kuoppala@linux.intel.com>
Reviewed-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
---
drivers/gpu/drm/i915/gem/i915_gem_context.c | 8 +++++++-
drivers/gpu/drm/i915/i915_gem_gtt.c | 10 +++++-----
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 1cdfe05514c3..863b5cc1ffa9 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -1003,12 +1003,18 @@ static int emit_ppgtt_update(struct i915_request *rq, void *data)
intel_ring_advance(rq, cs);
} else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+ int err;
+
+ /* Magic required to prevent forcewake errors! */
+ err = engine->emit_flush(rq, EMIT_INVALIDATE);
+ if (err)
+ return err;
cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
if (IS_ERR(cs))
return PTR_ERR(cs);
- *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES);
+ *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;;
for (i = GEN8_3LVL_PDPES; i--; ) {
const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 0b81e0b64393..43d0de15bb21 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -162,6 +162,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
pte_flags |= PTE_READ_ONLY;
vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
+ wmb();
return 0;
}
@@ -1420,6 +1421,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
set_pd_entry(pd, idx, pde);
atomic_inc(px_used(pde)); /* keep pinned */
}
+ wmb();
return 0;
}
@@ -1507,11 +1509,9 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
}
if (!i915_vm_is_4lvl(&ppgtt->vm)) {
- if (intel_vgpu_active(i915)) {
- err = gen8_preallocate_top_level_pdp(ppgtt);
- if (err)
- goto err_free_pd;
- }
+ err = gen8_preallocate_top_level_pdp(ppgtt);
+ if (err)
+ goto err_free_pd;
}
ppgtt->vm.insert_entries = gen8_ppgtt_insert;
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread
* [PATCH 03/15] drm/i915/selftests: Teach igt_gpu_fill_dw() to take intel_context
2019-08-23 13:26 vm->mutex Chris Wilson
2019-08-23 13:26 ` [PATCH 01/15] drm/i915: Hold irq-off for the entire fake lock period Chris Wilson
2019-08-23 13:26 ` [PATCH 02/15] drm/i915/gtt: Preallocate Braswell top-level page directory Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 16:45 ` [PATCH] " Chris Wilson
2019-08-23 13:26 ` [PATCH 04/15] dma-fence: Serialise signal enabling (dma_fence_enable_sw_signaling) Chris Wilson
` (14 subsequent siblings)
17 siblings, 1 reply; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx; +Cc: Matthew Auld
Avoid having to pass around (ctx, engine) everywhere by passing the
actual intel_context we intend to use. Today we preach this lesson to
igt_gpu_fill_dw and its callers' callers.
The immediate benefit for the GEM selftests is that we aim to use the
GEM context as the control, the source of the engines on which to test
the GEM context.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Matthew Auld <matthew.auld@intel.com>
Reviewed-by: Matthew Auld <matthew.auld@intel.com>
---
.../gpu/drm/i915/gem/selftests/huge_pages.c | 92 ++++++++++---------
.../drm/i915/gem/selftests/i915_gem_context.c | 70 ++++++++------
.../drm/i915/gem/selftests/igt_gem_utils.c | 26 +++---
.../drm/i915/gem/selftests/igt_gem_utils.h | 13 ++-
4 files changed, 109 insertions(+), 92 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
index 8de83c6d81f5..a0a272556289 100644
--- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
@@ -879,9 +879,8 @@ static int igt_mock_ppgtt_64K(void *arg)
return err;
}
-static int gpu_write(struct i915_vma *vma,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
+static int gpu_write(struct intel_context *ce,
+ struct i915_vma *vma,
u32 dw,
u32 val)
{
@@ -893,7 +892,7 @@ static int gpu_write(struct i915_vma *vma,
if (err)
return err;
- return igt_gpu_fill_dw(vma, ctx, engine, dw * sizeof(u32),
+ return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32),
vma->size >> PAGE_SHIFT, val);
}
@@ -929,18 +928,16 @@ static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
return err;
}
-static int __igt_write_huge(struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
+static int __igt_write_huge(struct intel_context *ce,
struct drm_i915_gem_object *obj,
u64 size, u64 offset,
u32 dword, u32 val)
{
- struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm;
unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
struct i915_vma *vma;
int err;
- vma = i915_vma_instance(obj, vm, NULL);
+ vma = i915_vma_instance(obj, ce->vm, NULL);
if (IS_ERR(vma))
return PTR_ERR(vma);
@@ -954,7 +951,7 @@ static int __igt_write_huge(struct i915_gem_context *ctx,
* The ggtt may have some pages reserved so
* refrain from erroring out.
*/
- if (err == -ENOSPC && i915_is_ggtt(vm))
+ if (err == -ENOSPC && i915_is_ggtt(ce->vm))
err = 0;
goto out_vma_close;
@@ -964,7 +961,7 @@ static int __igt_write_huge(struct i915_gem_context *ctx,
if (err)
goto out_vma_unpin;
- err = gpu_write(vma, ctx, engine, dword, val);
+ err = gpu_write(ce, vma, dword, val);
if (err) {
pr_err("gpu-write failed at offset=%llx\n", offset);
goto out_vma_unpin;
@@ -987,14 +984,13 @@ static int __igt_write_huge(struct i915_gem_context *ctx,
static int igt_write_huge(struct i915_gem_context *ctx,
struct drm_i915_gem_object *obj)
{
- struct drm_i915_private *i915 = to_i915(obj->base.dev);
- struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
- static struct intel_engine_cs *engines[I915_NUM_ENGINES];
- struct intel_engine_cs *engine;
+ struct i915_gem_engines *engines;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
I915_RND_STATE(prng);
IGT_TIMEOUT(end_time);
unsigned int max_page_size;
- unsigned int id;
+ unsigned int count;
u64 max;
u64 num;
u64 size;
@@ -1009,18 +1005,16 @@ static int igt_write_huge(struct i915_gem_context *ctx,
size = round_up(size, I915_GTT_PAGE_SIZE_2M);
max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
- max = div_u64((vm->total - size), max_page_size);
+ max = div_u64((ctx->vm->total - size), max_page_size);
n = 0;
- for_each_engine(engine, i915, id) {
- if (!intel_engine_can_store_dword(engine)) {
- pr_info("store-dword-imm not supported on engine=%u\n",
- id);
+ for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
+ count++;
+ if (!intel_engine_can_store_dword(ce->engine))
continue;
- }
- engines[n++] = engine;
+ n++;
}
-
+ i915_gem_context_unlock_engines(ctx);
if (!n)
return 0;
@@ -1029,7 +1023,7 @@ static int igt_write_huge(struct i915_gem_context *ctx,
* randomized order, lets also make feeding to the same engine a few
* times in succession a possibility by enlarging the permutation array.
*/
- order = i915_random_order(n * I915_NUM_ENGINES, &prng);
+ order = i915_random_order(count * count, &prng);
if (!order)
return -ENOMEM;
@@ -1039,13 +1033,17 @@ static int igt_write_huge(struct i915_gem_context *ctx,
* offset = 0.
*/
i = 0;
+ engines = i915_gem_context_lock_engines(ctx);
for_each_prime_number_from(num, 0, max) {
u64 offset_low = num * max_page_size;
u64 offset_high = (max - num) * max_page_size;
u32 dword = offset_in_page(num) / 4;
+ struct intel_context *ce;
- engine = engines[order[i] % n];
- i = (i + 1) % (n * I915_NUM_ENGINES);
+ ce = engines->engines[order[i] % engines->num_engines];
+ i = (i + 1) % (count * count);
+ if (!intel_engine_can_store_dword(ce->engine))
+ continue;
/*
* In order to utilize 64K pages we need to both pad the vma
@@ -1057,22 +1055,23 @@ static int igt_write_huge(struct i915_gem_context *ctx,
offset_low = round_down(offset_low,
I915_GTT_PAGE_SIZE_2M);
- err = __igt_write_huge(ctx, engine, obj, size, offset_low,
+ err = __igt_write_huge(ce, obj, size, offset_low,
dword, num + 1);
if (err)
break;
- err = __igt_write_huge(ctx, engine, obj, size, offset_high,
+ err = __igt_write_huge(ce, obj, size, offset_high,
dword, num + 1);
if (err)
break;
if (igt_timeout(end_time,
- "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
- __func__, engine->id, offset_low, offset_high,
+ "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
+ __func__, ce->engine->name, offset_low, offset_high,
max_page_size))
break;
}
+ i915_gem_context_unlock_engines(ctx);
kfree(order);
@@ -1316,10 +1315,11 @@ static int igt_ppgtt_pin_update(void *arg)
unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
struct i915_address_space *vm = ctx->vm;
struct drm_i915_gem_object *obj;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
struct i915_vma *vma;
unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
struct intel_engine_cs *engine;
- enum intel_engine_id id;
unsigned int n;
int first, last;
int err;
@@ -1419,14 +1419,18 @@ static int igt_ppgtt_pin_update(void *arg)
*/
n = 0;
- for_each_engine(engine, dev_priv, id) {
+ for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
if (!intel_engine_can_store_dword(engine))
continue;
- err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf);
+ err = gpu_write(ce, vma, n++, 0xdeadbeaf);
if (err)
- goto out_unpin;
+ break;
}
+ i915_gem_context_unlock_engines(ctx);
+ if (err)
+ goto out_unpin;
+
while (n--) {
err = cpu_check(obj, n, 0xdeadbeaf);
if (err)
@@ -1507,8 +1511,8 @@ static int igt_shrink_thp(void *arg)
struct drm_i915_private *i915 = ctx->i915;
struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
struct drm_i915_gem_object *obj;
- struct intel_engine_cs *engine;
- enum intel_engine_id id;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
struct i915_vma *vma;
unsigned int flags = PIN_USER;
unsigned int n;
@@ -1548,16 +1552,19 @@ static int igt_shrink_thp(void *arg)
goto out_unpin;
n = 0;
- for_each_engine(engine, i915, id) {
- if (!intel_engine_can_store_dword(engine))
+
+ for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
+ if (!intel_engine_can_store_dword(ce->engine))
continue;
- err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf);
+ err = gpu_write(ce, vma, n++, 0xdeadbeaf);
if (err)
- goto out_unpin;
+ break;
}
-
+ i915_gem_context_unlock_engines(ctx);
i915_vma_unpin(vma);
+ if (err)
+ goto out_close;
/*
* Now that the pages are *unpinned* shrink-all should invoke
@@ -1583,10 +1590,9 @@ static int igt_shrink_thp(void *arg)
while (n--) {
err = cpu_check(obj, n, 0xdeadbeaf);
if (err)
- goto out_unpin;
+ break;
}
-
out_unpin:
i915_vma_unpin(vma);
out_close:
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
index 3e6f4a65d356..3adb60c2fd1f 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
@@ -166,19 +166,17 @@ static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
}
-static int gpu_fill(struct drm_i915_gem_object *obj,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
+static int gpu_fill(struct intel_context *ce,
+ struct drm_i915_gem_object *obj,
unsigned int dw)
{
- struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm;
struct i915_vma *vma;
int err;
- GEM_BUG_ON(obj->base.size > vm->total);
- GEM_BUG_ON(!intel_engine_can_store_dword(engine));
+ GEM_BUG_ON(obj->base.size > ce->vm->total);
+ GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
- vma = i915_vma_instance(obj, vm, NULL);
+ vma = i915_vma_instance(obj, ce->vm, NULL);
if (IS_ERR(vma))
return PTR_ERR(vma);
@@ -200,9 +198,7 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
* whilst checking that each context provides a unique view
* into the object.
*/
- err = igt_gpu_fill_dw(vma,
- ctx,
- engine,
+ err = igt_gpu_fill_dw(ce, vma,
(dw * real_page_count(obj)) << PAGE_SHIFT |
(dw * sizeof(u32)),
real_page_count(obj),
@@ -305,22 +301,21 @@ static int file_add_object(struct drm_file *file,
}
static struct drm_i915_gem_object *
-create_test_object(struct i915_gem_context *ctx,
+create_test_object(struct i915_address_space *vm,
struct drm_file *file,
struct list_head *objects)
{
struct drm_i915_gem_object *obj;
- struct i915_address_space *vm = ctx->vm ?: &ctx->i915->ggtt.vm;
u64 size;
int err;
/* Keep in GEM's good graces */
- i915_retire_requests(ctx->i915);
+ i915_retire_requests(vm->i915);
size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
- obj = huge_gem_object(ctx->i915, DW_PER_PAGE * PAGE_SIZE, size);
+ obj = huge_gem_object(vm->i915, DW_PER_PAGE * PAGE_SIZE, size);
if (IS_ERR(obj))
return obj;
@@ -393,6 +388,7 @@ static int igt_ctx_exec(void *arg)
dw = 0;
while (!time_after(jiffies, end_time)) {
struct i915_gem_context *ctx;
+ struct intel_context *ce;
ctx = live_context(i915, file);
if (IS_ERR(ctx)) {
@@ -400,15 +396,20 @@ static int igt_ctx_exec(void *arg)
goto out_unlock;
}
+ ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
+
if (!obj) {
- obj = create_test_object(ctx, file, &objects);
+ obj = create_test_object(ce->vm, file, &objects);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
+ intel_context_put(ce);
goto out_unlock;
}
}
- err = gpu_fill(obj, ctx, engine, dw);
+ err = gpu_fill(ce, obj, dw);
+ intel_context_put(ce);
+
if (err) {
pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
ndwords, dw, max_dwords(obj),
@@ -509,6 +510,7 @@ static int igt_shared_ctx_exec(void *arg)
ncontexts = 0;
while (!time_after(jiffies, end_time)) {
struct i915_gem_context *ctx;
+ struct intel_context *ce;
ctx = kernel_context(i915);
if (IS_ERR(ctx)) {
@@ -518,22 +520,26 @@ static int igt_shared_ctx_exec(void *arg)
__assign_ppgtt(ctx, parent->vm);
+ ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
if (!obj) {
- obj = create_test_object(parent, file, &objects);
+ obj = create_test_object(parent->vm, file, &objects);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
+ intel_context_put(ce);
kernel_context_close(ctx);
goto out_test;
}
}
- err = gpu_fill(obj, ctx, engine, dw);
+ err = gpu_fill(ce, obj, dw);
+ intel_context_put(ce);
+ kernel_context_close(ctx);
+
if (err) {
pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
ndwords, dw, max_dwords(obj),
engine->name, ctx->hw_id,
yesno(!!ctx->vm), err);
- kernel_context_close(ctx);
goto out_test;
}
@@ -544,8 +550,6 @@ static int igt_shared_ctx_exec(void *arg)
ndwords++;
ncontexts++;
-
- kernel_context_close(ctx);
}
pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
ncontexts, engine->name, ndwords);
@@ -604,6 +608,8 @@ static struct i915_vma *rpcs_query_batch(struct i915_vma *vma)
__i915_gem_object_flush_map(obj, 0, 64);
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(vma->vm->gt);
+
vma = i915_vma_instance(obj, vma->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
@@ -1082,17 +1088,19 @@ static int igt_ctx_readonly(void *arg)
ndwords = 0;
dw = 0;
while (!time_after(jiffies, end_time)) {
- struct intel_engine_cs *engine;
- unsigned int id;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
- for_each_engine(engine, i915, id) {
- if (!intel_engine_can_store_dword(engine))
+ for_each_gem_engine(ce,
+ i915_gem_context_lock_engines(ctx), it) {
+ if (!intel_engine_can_store_dword(ce->engine))
continue;
if (!obj) {
- obj = create_test_object(ctx, file, &objects);
+ obj = create_test_object(ce->vm, file, &objects);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
+ i915_gem_context_unlock_engines(ctx);
goto out_unlock;
}
@@ -1100,12 +1108,13 @@ static int igt_ctx_readonly(void *arg)
i915_gem_object_set_readonly(obj);
}
- err = gpu_fill(obj, ctx, engine, dw);
+ err = gpu_fill(ce, obj, dw);
if (err) {
pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
ndwords, dw, max_dwords(obj),
- engine->name, ctx->hw_id,
+ ce->engine->name, ctx->hw_id,
yesno(!!ctx->vm), err);
+ i915_gem_context_unlock_engines(ctx);
goto out_unlock;
}
@@ -1115,6 +1124,7 @@ static int igt_ctx_readonly(void *arg)
}
ndwords++;
}
+ i915_gem_context_unlock_engines(ctx);
}
pr_info("Submitted %lu dwords (across %u engines)\n",
ndwords, RUNTIME_INFO(i915)->num_engines);
@@ -1197,6 +1207,8 @@ static int write_to_scratch(struct i915_gem_context *ctx,
__i915_gem_object_flush_map(obj, 0, 64);
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(engine->gt);
+
vma = i915_vma_instance(obj, ctx->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
@@ -1296,6 +1308,8 @@ static int read_from_scratch(struct i915_gem_context *ctx,
i915_gem_object_flush_map(obj);
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(engine->gt);
+
vma = i915_vma_instance(obj, ctx->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
diff --git a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
index 57ece53c1075..ee5dc13a30b3 100644
--- a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
+++ b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
@@ -9,6 +9,7 @@
#include "gem/i915_gem_context.h"
#include "gem/i915_gem_pm.h"
#include "gt/intel_context.h"
+#include "gt/intel_gt.h"
#include "i915_vma.h"
#include "i915_drv.h"
@@ -84,6 +85,8 @@ igt_emit_store_dw(struct i915_vma *vma,
*cmd = MI_BATCH_BUFFER_END;
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(vma->vm->gt);
+
vma = i915_vma_instance(obj, vma->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
@@ -101,40 +104,35 @@ igt_emit_store_dw(struct i915_vma *vma,
return ERR_PTR(err);
}
-int igt_gpu_fill_dw(struct i915_vma *vma,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
- u64 offset,
- unsigned long count,
- u32 val)
+int igt_gpu_fill_dw(struct intel_context *ce,
+ struct i915_vma *vma, u64 offset,
+ unsigned long count, u32 val)
{
- struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm;
struct i915_request *rq;
struct i915_vma *batch;
unsigned int flags;
int err;
- GEM_BUG_ON(vma->size > vm->total);
- GEM_BUG_ON(!intel_engine_can_store_dword(engine));
+ GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
GEM_BUG_ON(!i915_vma_is_pinned(vma));
batch = igt_emit_store_dw(vma, offset, count, val);
if (IS_ERR(batch))
return PTR_ERR(batch);
- rq = igt_request_alloc(ctx, engine);
+ rq = intel_context_create_request(ce);
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
goto err_batch;
}
flags = 0;
- if (INTEL_GEN(vm->i915) <= 5)
+ if (INTEL_GEN(ce->vm->i915) <= 5)
flags |= I915_DISPATCH_SECURE;
- err = engine->emit_bb_start(rq,
- batch->node.start, batch->node.size,
- flags);
+ err = rq->engine->emit_bb_start(rq,
+ batch->node.start, batch->node.size,
+ flags);
if (err)
goto err_request;
diff --git a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h
index 361a7ef866b0..4221cf84d175 100644
--- a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h
+++ b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h
@@ -11,9 +11,11 @@
struct i915_request;
struct i915_gem_context;
-struct intel_engine_cs;
struct i915_vma;
+struct intel_context;
+struct intel_engine_cs;
+
struct i915_request *
igt_request_alloc(struct i915_gem_context *ctx, struct intel_engine_cs *engine);
@@ -23,11 +25,8 @@ igt_emit_store_dw(struct i915_vma *vma,
unsigned long count,
u32 val);
-int igt_gpu_fill_dw(struct i915_vma *vma,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
- u64 offset,
- unsigned long count,
- u32 val);
+int igt_gpu_fill_dw(struct intel_context *ce,
+ struct i915_vma *vma, u64 offset,
+ unsigned long count, u32 val);
#endif /* __IGT_GEM_UTILS_H__ */
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH] drm/i915/selftests: Teach igt_gpu_fill_dw() to take intel_context
2019-08-23 13:26 ` [PATCH 03/15] drm/i915/selftests: Teach igt_gpu_fill_dw() to take intel_context Chris Wilson
@ 2019-08-23 16:45 ` Chris Wilson
0 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 16:45 UTC (permalink / raw)
To: intel-gfx; +Cc: Matthew Auld
Avoid having to pass around (ctx, engine) everywhere by passing the
actual intel_context we intend to use. Today we preach this lesson to
igt_gpu_fill_dw and its callers' callers.
The immediate benefit for the GEM selftests is that we aim to use the
GEM context as the control, the source of the engines on which to test
the GEM context.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Matthew Auld <matthew.auld@intel.com>
Reviewed-by: Matthew Auld <matthew.auld@intel.com>
---
.../gpu/drm/i915/gem/selftests/huge_pages.c | 92 ++++++++++---------
.../drm/i915/gem/selftests/i915_gem_context.c | 70 ++++++++------
.../drm/i915/gem/selftests/igt_gem_utils.c | 26 +++---
.../drm/i915/gem/selftests/igt_gem_utils.h | 13 ++-
4 files changed, 109 insertions(+), 92 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
index 8de83c6d81f5..5dc97e3a5a9d 100644
--- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
@@ -879,9 +879,8 @@ static int igt_mock_ppgtt_64K(void *arg)
return err;
}
-static int gpu_write(struct i915_vma *vma,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
+static int gpu_write(struct intel_context *ce,
+ struct i915_vma *vma,
u32 dw,
u32 val)
{
@@ -893,7 +892,7 @@ static int gpu_write(struct i915_vma *vma,
if (err)
return err;
- return igt_gpu_fill_dw(vma, ctx, engine, dw * sizeof(u32),
+ return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32),
vma->size >> PAGE_SHIFT, val);
}
@@ -929,18 +928,16 @@ static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
return err;
}
-static int __igt_write_huge(struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
+static int __igt_write_huge(struct intel_context *ce,
struct drm_i915_gem_object *obj,
u64 size, u64 offset,
u32 dword, u32 val)
{
- struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm;
unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
struct i915_vma *vma;
int err;
- vma = i915_vma_instance(obj, vm, NULL);
+ vma = i915_vma_instance(obj, ce->vm, NULL);
if (IS_ERR(vma))
return PTR_ERR(vma);
@@ -954,7 +951,7 @@ static int __igt_write_huge(struct i915_gem_context *ctx,
* The ggtt may have some pages reserved so
* refrain from erroring out.
*/
- if (err == -ENOSPC && i915_is_ggtt(vm))
+ if (err == -ENOSPC && i915_is_ggtt(ce->vm))
err = 0;
goto out_vma_close;
@@ -964,7 +961,7 @@ static int __igt_write_huge(struct i915_gem_context *ctx,
if (err)
goto out_vma_unpin;
- err = gpu_write(vma, ctx, engine, dword, val);
+ err = gpu_write(ce, vma, dword, val);
if (err) {
pr_err("gpu-write failed at offset=%llx\n", offset);
goto out_vma_unpin;
@@ -987,14 +984,13 @@ static int __igt_write_huge(struct i915_gem_context *ctx,
static int igt_write_huge(struct i915_gem_context *ctx,
struct drm_i915_gem_object *obj)
{
- struct drm_i915_private *i915 = to_i915(obj->base.dev);
- struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
- static struct intel_engine_cs *engines[I915_NUM_ENGINES];
- struct intel_engine_cs *engine;
+ struct i915_gem_engines *engines;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
I915_RND_STATE(prng);
IGT_TIMEOUT(end_time);
unsigned int max_page_size;
- unsigned int id;
+ unsigned int count;
u64 max;
u64 num;
u64 size;
@@ -1009,18 +1005,16 @@ static int igt_write_huge(struct i915_gem_context *ctx,
size = round_up(size, I915_GTT_PAGE_SIZE_2M);
max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
- max = div_u64((vm->total - size), max_page_size);
+ max = div_u64((ctx->vm->total - size), max_page_size);
n = 0;
- for_each_engine(engine, i915, id) {
- if (!intel_engine_can_store_dword(engine)) {
- pr_info("store-dword-imm not supported on engine=%u\n",
- id);
+ for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
+ count++;
+ if (!intel_engine_can_store_dword(ce->engine))
continue;
- }
- engines[n++] = engine;
+ n++;
}
-
+ i915_gem_context_unlock_engines(ctx);
if (!n)
return 0;
@@ -1029,7 +1023,7 @@ static int igt_write_huge(struct i915_gem_context *ctx,
* randomized order, lets also make feeding to the same engine a few
* times in succession a possibility by enlarging the permutation array.
*/
- order = i915_random_order(n * I915_NUM_ENGINES, &prng);
+ order = i915_random_order(count * count, &prng);
if (!order)
return -ENOMEM;
@@ -1039,13 +1033,17 @@ static int igt_write_huge(struct i915_gem_context *ctx,
* offset = 0.
*/
i = 0;
+ engines = i915_gem_context_lock_engines(ctx);
for_each_prime_number_from(num, 0, max) {
u64 offset_low = num * max_page_size;
u64 offset_high = (max - num) * max_page_size;
u32 dword = offset_in_page(num) / 4;
+ struct intel_context *ce;
- engine = engines[order[i] % n];
- i = (i + 1) % (n * I915_NUM_ENGINES);
+ ce = engines->engines[order[i] % engines->num_engines];
+ i = (i + 1) % (count * count);
+ if (!ce || !intel_engine_can_store_dword(ce->engine))
+ continue;
/*
* In order to utilize 64K pages we need to both pad the vma
@@ -1057,22 +1055,23 @@ static int igt_write_huge(struct i915_gem_context *ctx,
offset_low = round_down(offset_low,
I915_GTT_PAGE_SIZE_2M);
- err = __igt_write_huge(ctx, engine, obj, size, offset_low,
+ err = __igt_write_huge(ce, obj, size, offset_low,
dword, num + 1);
if (err)
break;
- err = __igt_write_huge(ctx, engine, obj, size, offset_high,
+ err = __igt_write_huge(ce, obj, size, offset_high,
dword, num + 1);
if (err)
break;
if (igt_timeout(end_time,
- "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
- __func__, engine->id, offset_low, offset_high,
+ "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
+ __func__, ce->engine->name, offset_low, offset_high,
max_page_size))
break;
}
+ i915_gem_context_unlock_engines(ctx);
kfree(order);
@@ -1316,10 +1315,11 @@ static int igt_ppgtt_pin_update(void *arg)
unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
struct i915_address_space *vm = ctx->vm;
struct drm_i915_gem_object *obj;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
struct i915_vma *vma;
unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
struct intel_engine_cs *engine;
- enum intel_engine_id id;
unsigned int n;
int first, last;
int err;
@@ -1419,14 +1419,18 @@ static int igt_ppgtt_pin_update(void *arg)
*/
n = 0;
- for_each_engine(engine, dev_priv, id) {
+ for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
if (!intel_engine_can_store_dword(engine))
continue;
- err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf);
+ err = gpu_write(ce, vma, n++, 0xdeadbeaf);
if (err)
- goto out_unpin;
+ break;
}
+ i915_gem_context_unlock_engines(ctx);
+ if (err)
+ goto out_unpin;
+
while (n--) {
err = cpu_check(obj, n, 0xdeadbeaf);
if (err)
@@ -1507,8 +1511,8 @@ static int igt_shrink_thp(void *arg)
struct drm_i915_private *i915 = ctx->i915;
struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
struct drm_i915_gem_object *obj;
- struct intel_engine_cs *engine;
- enum intel_engine_id id;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
struct i915_vma *vma;
unsigned int flags = PIN_USER;
unsigned int n;
@@ -1548,16 +1552,19 @@ static int igt_shrink_thp(void *arg)
goto out_unpin;
n = 0;
- for_each_engine(engine, i915, id) {
- if (!intel_engine_can_store_dword(engine))
+
+ for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
+ if (!intel_engine_can_store_dword(ce->engine))
continue;
- err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf);
+ err = gpu_write(ce, vma, n++, 0xdeadbeaf);
if (err)
- goto out_unpin;
+ break;
}
-
+ i915_gem_context_unlock_engines(ctx);
i915_vma_unpin(vma);
+ if (err)
+ goto out_close;
/*
* Now that the pages are *unpinned* shrink-all should invoke
@@ -1583,10 +1590,9 @@ static int igt_shrink_thp(void *arg)
while (n--) {
err = cpu_check(obj, n, 0xdeadbeaf);
if (err)
- goto out_unpin;
+ break;
}
-
out_unpin:
i915_vma_unpin(vma);
out_close:
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
index 3e6f4a65d356..3adb60c2fd1f 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
@@ -166,19 +166,17 @@ static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
}
-static int gpu_fill(struct drm_i915_gem_object *obj,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
+static int gpu_fill(struct intel_context *ce,
+ struct drm_i915_gem_object *obj,
unsigned int dw)
{
- struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm;
struct i915_vma *vma;
int err;
- GEM_BUG_ON(obj->base.size > vm->total);
- GEM_BUG_ON(!intel_engine_can_store_dword(engine));
+ GEM_BUG_ON(obj->base.size > ce->vm->total);
+ GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
- vma = i915_vma_instance(obj, vm, NULL);
+ vma = i915_vma_instance(obj, ce->vm, NULL);
if (IS_ERR(vma))
return PTR_ERR(vma);
@@ -200,9 +198,7 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
* whilst checking that each context provides a unique view
* into the object.
*/
- err = igt_gpu_fill_dw(vma,
- ctx,
- engine,
+ err = igt_gpu_fill_dw(ce, vma,
(dw * real_page_count(obj)) << PAGE_SHIFT |
(dw * sizeof(u32)),
real_page_count(obj),
@@ -305,22 +301,21 @@ static int file_add_object(struct drm_file *file,
}
static struct drm_i915_gem_object *
-create_test_object(struct i915_gem_context *ctx,
+create_test_object(struct i915_address_space *vm,
struct drm_file *file,
struct list_head *objects)
{
struct drm_i915_gem_object *obj;
- struct i915_address_space *vm = ctx->vm ?: &ctx->i915->ggtt.vm;
u64 size;
int err;
/* Keep in GEM's good graces */
- i915_retire_requests(ctx->i915);
+ i915_retire_requests(vm->i915);
size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
- obj = huge_gem_object(ctx->i915, DW_PER_PAGE * PAGE_SIZE, size);
+ obj = huge_gem_object(vm->i915, DW_PER_PAGE * PAGE_SIZE, size);
if (IS_ERR(obj))
return obj;
@@ -393,6 +388,7 @@ static int igt_ctx_exec(void *arg)
dw = 0;
while (!time_after(jiffies, end_time)) {
struct i915_gem_context *ctx;
+ struct intel_context *ce;
ctx = live_context(i915, file);
if (IS_ERR(ctx)) {
@@ -400,15 +396,20 @@ static int igt_ctx_exec(void *arg)
goto out_unlock;
}
+ ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
+
if (!obj) {
- obj = create_test_object(ctx, file, &objects);
+ obj = create_test_object(ce->vm, file, &objects);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
+ intel_context_put(ce);
goto out_unlock;
}
}
- err = gpu_fill(obj, ctx, engine, dw);
+ err = gpu_fill(ce, obj, dw);
+ intel_context_put(ce);
+
if (err) {
pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
ndwords, dw, max_dwords(obj),
@@ -509,6 +510,7 @@ static int igt_shared_ctx_exec(void *arg)
ncontexts = 0;
while (!time_after(jiffies, end_time)) {
struct i915_gem_context *ctx;
+ struct intel_context *ce;
ctx = kernel_context(i915);
if (IS_ERR(ctx)) {
@@ -518,22 +520,26 @@ static int igt_shared_ctx_exec(void *arg)
__assign_ppgtt(ctx, parent->vm);
+ ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
if (!obj) {
- obj = create_test_object(parent, file, &objects);
+ obj = create_test_object(parent->vm, file, &objects);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
+ intel_context_put(ce);
kernel_context_close(ctx);
goto out_test;
}
}
- err = gpu_fill(obj, ctx, engine, dw);
+ err = gpu_fill(ce, obj, dw);
+ intel_context_put(ce);
+ kernel_context_close(ctx);
+
if (err) {
pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
ndwords, dw, max_dwords(obj),
engine->name, ctx->hw_id,
yesno(!!ctx->vm), err);
- kernel_context_close(ctx);
goto out_test;
}
@@ -544,8 +550,6 @@ static int igt_shared_ctx_exec(void *arg)
ndwords++;
ncontexts++;
-
- kernel_context_close(ctx);
}
pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
ncontexts, engine->name, ndwords);
@@ -604,6 +608,8 @@ static struct i915_vma *rpcs_query_batch(struct i915_vma *vma)
__i915_gem_object_flush_map(obj, 0, 64);
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(vma->vm->gt);
+
vma = i915_vma_instance(obj, vma->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
@@ -1082,17 +1088,19 @@ static int igt_ctx_readonly(void *arg)
ndwords = 0;
dw = 0;
while (!time_after(jiffies, end_time)) {
- struct intel_engine_cs *engine;
- unsigned int id;
+ struct i915_gem_engines_iter it;
+ struct intel_context *ce;
- for_each_engine(engine, i915, id) {
- if (!intel_engine_can_store_dword(engine))
+ for_each_gem_engine(ce,
+ i915_gem_context_lock_engines(ctx), it) {
+ if (!intel_engine_can_store_dword(ce->engine))
continue;
if (!obj) {
- obj = create_test_object(ctx, file, &objects);
+ obj = create_test_object(ce->vm, file, &objects);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
+ i915_gem_context_unlock_engines(ctx);
goto out_unlock;
}
@@ -1100,12 +1108,13 @@ static int igt_ctx_readonly(void *arg)
i915_gem_object_set_readonly(obj);
}
- err = gpu_fill(obj, ctx, engine, dw);
+ err = gpu_fill(ce, obj, dw);
if (err) {
pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
ndwords, dw, max_dwords(obj),
- engine->name, ctx->hw_id,
+ ce->engine->name, ctx->hw_id,
yesno(!!ctx->vm), err);
+ i915_gem_context_unlock_engines(ctx);
goto out_unlock;
}
@@ -1115,6 +1124,7 @@ static int igt_ctx_readonly(void *arg)
}
ndwords++;
}
+ i915_gem_context_unlock_engines(ctx);
}
pr_info("Submitted %lu dwords (across %u engines)\n",
ndwords, RUNTIME_INFO(i915)->num_engines);
@@ -1197,6 +1207,8 @@ static int write_to_scratch(struct i915_gem_context *ctx,
__i915_gem_object_flush_map(obj, 0, 64);
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(engine->gt);
+
vma = i915_vma_instance(obj, ctx->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
@@ -1296,6 +1308,8 @@ static int read_from_scratch(struct i915_gem_context *ctx,
i915_gem_object_flush_map(obj);
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(engine->gt);
+
vma = i915_vma_instance(obj, ctx->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
diff --git a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
index 57ece53c1075..ee5dc13a30b3 100644
--- a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
+++ b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
@@ -9,6 +9,7 @@
#include "gem/i915_gem_context.h"
#include "gem/i915_gem_pm.h"
#include "gt/intel_context.h"
+#include "gt/intel_gt.h"
#include "i915_vma.h"
#include "i915_drv.h"
@@ -84,6 +85,8 @@ igt_emit_store_dw(struct i915_vma *vma,
*cmd = MI_BATCH_BUFFER_END;
i915_gem_object_unpin_map(obj);
+ intel_gt_chipset_flush(vma->vm->gt);
+
vma = i915_vma_instance(obj, vma->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
@@ -101,40 +104,35 @@ igt_emit_store_dw(struct i915_vma *vma,
return ERR_PTR(err);
}
-int igt_gpu_fill_dw(struct i915_vma *vma,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
- u64 offset,
- unsigned long count,
- u32 val)
+int igt_gpu_fill_dw(struct intel_context *ce,
+ struct i915_vma *vma, u64 offset,
+ unsigned long count, u32 val)
{
- struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm;
struct i915_request *rq;
struct i915_vma *batch;
unsigned int flags;
int err;
- GEM_BUG_ON(vma->size > vm->total);
- GEM_BUG_ON(!intel_engine_can_store_dword(engine));
+ GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
GEM_BUG_ON(!i915_vma_is_pinned(vma));
batch = igt_emit_store_dw(vma, offset, count, val);
if (IS_ERR(batch))
return PTR_ERR(batch);
- rq = igt_request_alloc(ctx, engine);
+ rq = intel_context_create_request(ce);
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
goto err_batch;
}
flags = 0;
- if (INTEL_GEN(vm->i915) <= 5)
+ if (INTEL_GEN(ce->vm->i915) <= 5)
flags |= I915_DISPATCH_SECURE;
- err = engine->emit_bb_start(rq,
- batch->node.start, batch->node.size,
- flags);
+ err = rq->engine->emit_bb_start(rq,
+ batch->node.start, batch->node.size,
+ flags);
if (err)
goto err_request;
diff --git a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h
index 361a7ef866b0..4221cf84d175 100644
--- a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h
+++ b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h
@@ -11,9 +11,11 @@
struct i915_request;
struct i915_gem_context;
-struct intel_engine_cs;
struct i915_vma;
+struct intel_context;
+struct intel_engine_cs;
+
struct i915_request *
igt_request_alloc(struct i915_gem_context *ctx, struct intel_engine_cs *engine);
@@ -23,11 +25,8 @@ igt_emit_store_dw(struct i915_vma *vma,
unsigned long count,
u32 val);
-int igt_gpu_fill_dw(struct i915_vma *vma,
- struct i915_gem_context *ctx,
- struct intel_engine_cs *engine,
- u64 offset,
- unsigned long count,
- u32 val);
+int igt_gpu_fill_dw(struct intel_context *ce,
+ struct i915_vma *vma, u64 offset,
+ unsigned long count, u32 val);
#endif /* __IGT_GEM_UTILS_H__ */
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread
* [PATCH 04/15] dma-fence: Serialise signal enabling (dma_fence_enable_sw_signaling)
2019-08-23 13:26 vm->mutex Chris Wilson
` (2 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 03/15] drm/i915/selftests: Teach igt_gpu_fill_dw() to take intel_context Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 05/15] drm/i915: Make shrink/unshrink be atomic Chris Wilson
` (13 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Make dma_fence_enable_sw_signaling() behave like its
dma_fence_add_callback() and dma_fence_default_wait() counterparts and
perform the test to enable signaling under the fence->lock, along with
the action to do so. This ensure that should an implementation be trying
to flush the cb_list (by signaling) on retirement before freeing the
fence, it can do so in a race-free manner.
See also 0fc89b6802ba ("dma-fence: Simply wrap dma_fence_signal_locked
with dma_fence_signal").
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
drivers/dma-buf/dma-fence.c | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 2c136aee3e79..587727089134 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -285,19 +285,18 @@ void dma_fence_enable_sw_signaling(struct dma_fence *fence)
{
unsigned long flags;
+ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+ return;
+
+ spin_lock_irqsave(fence->lock, flags);
if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
&fence->flags) &&
- !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) &&
fence->ops->enable_signaling) {
trace_dma_fence_enable_signal(fence);
-
- spin_lock_irqsave(fence->lock, flags);
-
if (!fence->ops->enable_signaling(fence))
dma_fence_signal_locked(fence);
-
- spin_unlock_irqrestore(fence->lock, flags);
}
+ spin_unlock_irqrestore(fence->lock, flags);
}
EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 05/15] drm/i915: Make shrink/unshrink be atomic
2019-08-23 13:26 vm->mutex Chris Wilson
` (3 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 04/15] dma-fence: Serialise signal enabling (dma_fence_enable_sw_signaling) Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 06/15] drm/i915: Mark up obj->pin_global as being atomic Chris Wilson
` (12 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Add an atomic counter and always take the spinlock around the pin/unpin
events, so that we can perform the list manipulation concurrently.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
drivers/gpu/drm/i915/gem/i915_gem_domain.c | 3 +-
.../gpu/drm/i915/gem/i915_gem_object_types.h | 1 +
drivers/gpu/drm/i915/gem/i915_gem_pages.c | 1 +
drivers/gpu/drm/i915/gem/i915_gem_shrinker.c | 36 +++++++++++--------
drivers/gpu/drm/i915/gt/intel_context.c | 2 +-
5 files changed, 26 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index 9c58e8fac1d9..ef33001e8ddb 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -500,7 +500,8 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
spin_lock_irqsave(&i915->mm.obj_lock, flags);
- if (obj->mm.madv == I915_MADV_WILLNEED)
+ if (obj->mm.madv == I915_MADV_WILLNEED &&
+ !atomic_read(&obj->mm.shrink_pin))
list_move_tail(&obj->mm.link, &i915->mm.shrink_list);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index ede0eb4218a8..511fce7913ec 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -158,6 +158,7 @@ struct drm_i915_gem_object {
struct {
struct mutex lock; /* protects the pages and their use */
atomic_t pages_pin_count;
+ atomic_t shrink_pin;
struct sg_table *pages;
void *mapping;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
index 18f0ce0135c1..2e941f093a20 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
@@ -71,6 +71,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
list = &i915->mm.shrink_list;
list_add_tail(&obj->mm.link, list);
+ atomic_set(&obj->mm.shrink_pin, 0);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
index edd21d14e64f..94970f60d7f6 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
@@ -523,46 +523,52 @@ void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915,
void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj)
{
+ struct drm_i915_private *i915 = obj_to_i915(obj);
+ unsigned long flags;
+
/*
* We can only be called while the pages are pinned or when
* the pages are released. If pinned, we should only be called
* from a single caller under controlled conditions; and on release
* only one caller may release us. Neither the two may cross.
*/
- if (!list_empty(&obj->mm.link)) { /* pinned by caller */
- struct drm_i915_private *i915 = obj_to_i915(obj);
- unsigned long flags;
-
- spin_lock_irqsave(&i915->mm.obj_lock, flags);
- GEM_BUG_ON(list_empty(&obj->mm.link));
+ if (atomic_add_unless(&obj->mm.shrink_pin, 1, 0))
+ return;
+ spin_lock_irqsave(&i915->mm.obj_lock, flags);
+ if (!atomic_fetch_inc(&obj->mm.shrink_pin) &&
+ !list_empty(&obj->mm.link)) {
list_del_init(&obj->mm.link);
i915->mm.shrink_count--;
i915->mm.shrink_memory -= obj->base.size;
-
- spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
+ spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
static void __i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj,
struct list_head *head)
{
+ struct drm_i915_private *i915 = obj_to_i915(obj);
+ unsigned long flags;
+
GEM_BUG_ON(!i915_gem_object_has_pages(obj));
- GEM_BUG_ON(!list_empty(&obj->mm.link));
+ if (!i915_gem_object_is_shrinkable(obj))
+ return;
- if (i915_gem_object_is_shrinkable(obj)) {
- struct drm_i915_private *i915 = obj_to_i915(obj);
- unsigned long flags;
+ if (atomic_add_unless(&obj->mm.shrink_pin, -1, 1))
+ return;
- spin_lock_irqsave(&i915->mm.obj_lock, flags);
- GEM_BUG_ON(!kref_read(&obj->base.refcount));
+ spin_lock_irqsave(&i915->mm.obj_lock, flags);
+ GEM_BUG_ON(!kref_read(&obj->base.refcount));
+ if (atomic_dec_and_test(&obj->mm.shrink_pin)) {
+ GEM_BUG_ON(!list_empty(&obj->mm.link));
list_add_tail(&obj->mm.link, head);
i915->mm.shrink_count++;
i915->mm.shrink_memory += obj->base.size;
- spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
+ spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj)
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index f55691d151ae..c0495811f493 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -134,8 +134,8 @@ static int __context_pin_state(struct i915_vma *vma)
static void __context_unpin_state(struct i915_vma *vma)
{
- __i915_vma_unpin(vma);
i915_vma_make_shrinkable(vma);
+ __i915_vma_unpin(vma);
}
static void __intel_context_retire(struct i915_active *active)
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 06/15] drm/i915: Mark up obj->pin_global as being atomic
2019-08-23 13:26 vm->mutex Chris Wilson
` (4 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 05/15] drm/i915: Make shrink/unshrink be atomic Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 16:00 ` Mika Kuoppala
2019-08-23 13:26 ` [PATCH 07/15] drm/i915: Only track bound elements of the GTT Chris Wilson
` (11 subsequent siblings)
17 siblings, 1 reply; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Coming up next, we will want to manipulate the pin_global counter
outside of the principle locks, so convert it to an atomic.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
drivers/gpu/drm/i915/gem/i915_gem_domain.c | 12 ++++++------
drivers/gpu/drm/i915/gem/i915_gem_object.h | 3 ++-
drivers/gpu/drm/i915/gem/i915_gem_object_types.h | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_shrinker.c | 2 +-
drivers/gpu/drm/i915/i915_debugfs.c | 4 ++--
5 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index ef33001e8ddb..b7be9ad2b62c 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -27,7 +27,7 @@ static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
{
- if (!READ_ONCE(obj->pin_global))
+ if (!atomic_read(&obj->pin_global))
return;
i915_gem_object_lock(obj);
@@ -425,7 +425,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
/* Mark the global pin early so that we account for the
* display coherency whilst setting up the cache domains.
*/
- obj->pin_global++;
+ atomic_inc(&obj->pin_global);
/* The display engine is not coherent with the LLC cache on gen6. As
* a result, we make sure that the pinning that is about to occur is
@@ -475,7 +475,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
return vma;
err_unpin_global:
- obj->pin_global--;
+ atomic_dec(&obj->pin_global);
return vma;
}
@@ -515,11 +515,11 @@ i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
assert_object_held(obj);
- if (WARN_ON(obj->pin_global == 0))
+ if (GEM_WARN_ON(!atomic_read(&obj->pin_global)))
return;
- if (--obj->pin_global == 0)
- vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
+ if (atomic_dec_and_test(&obj->pin_global))
+ vma->display_alignment = I915_GTT_MIN_ALIGNMENT; /* XXX race */
/* Bump the LRU to try and avoid premature eviction whilst flipping */
i915_gem_object_bump_inactive_ggtt(obj);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 5efb9936e05b..11bb1a412e4f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -406,7 +406,8 @@ static inline bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
return true;
- return obj->pin_global; /* currently in use by HW, keep flushed */
+ /* Currently in use by HW (display engine)? Keep flushed. */
+ return atomic_read(&obj->pin_global);
}
static inline void __start_cpu_write(struct drm_i915_gem_object *obj)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index 511fce7913ec..a09e74e27734 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -153,7 +153,7 @@ struct drm_i915_gem_object {
/** Count of VMA actually bound by this object */
atomic_t bind_count;
/** Count of how many global VMA are currently pinned for use by HW */
- unsigned int pin_global;
+ atomic_t pin_global;
struct {
struct mutex lock; /* protects the pages and their use */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
index 94970f60d7f6..841f18386b54 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
@@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
* To simplify the scan, and to avoid walking the list of vma under the
* object, we just check the count of its permanently pinned.
*/
- if (READ_ONCE(obj->pin_global))
+ if (atomic_read(&obj->pin_global))
return false;
/* We can only return physical pages to the system if we can either
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index e103fcba6435..c36538f1b36c 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -79,7 +79,7 @@ static int i915_capabilities(struct seq_file *m, void *data)
static char get_pin_flag(struct drm_i915_gem_object *obj)
{
- return obj->pin_global ? 'p' : ' ';
+ return atomic_read(&obj->pin_global) ? 'p' : ' ';
}
static char get_tiling_flag(struct drm_i915_gem_object *obj)
@@ -221,7 +221,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
seq_printf(m, " (pinned x %d)", pin_count);
if (obj->stolen)
seq_printf(m, " (stolen: %08llx)", obj->stolen->start);
- if (obj->pin_global)
+ if (atomic_read(&obj->pin_global))
seq_printf(m, " (global)");
engine = i915_gem_object_last_write_engine(obj);
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [PATCH 06/15] drm/i915: Mark up obj->pin_global as being atomic
2019-08-23 13:26 ` [PATCH 06/15] drm/i915: Mark up obj->pin_global as being atomic Chris Wilson
@ 2019-08-23 16:00 ` Mika Kuoppala
2019-08-23 16:03 ` Chris Wilson
0 siblings, 1 reply; 25+ messages in thread
From: Mika Kuoppala @ 2019-08-23 16:00 UTC (permalink / raw)
To: Chris Wilson, intel-gfx
Chris Wilson <chris@chris-wilson.co.uk> writes:
> Coming up next, we will want to manipulate the pin_global counter
> outside of the principle locks, so convert it to an atomic.
>
I am not yet up to speed on pin_global but
being part of a series where where this needs
to be atomic, the patch does what it says on the tin.
Reviewed-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> ---
> drivers/gpu/drm/i915/gem/i915_gem_domain.c | 12 ++++++------
> drivers/gpu/drm/i915/gem/i915_gem_object.h | 3 ++-
> drivers/gpu/drm/i915/gem/i915_gem_object_types.h | 2 +-
> drivers/gpu/drm/i915/gem/i915_gem_shrinker.c | 2 +-
> drivers/gpu/drm/i915/i915_debugfs.c | 4 ++--
> 5 files changed, 12 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
> index ef33001e8ddb..b7be9ad2b62c 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
> @@ -27,7 +27,7 @@ static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
>
> void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
> {
> - if (!READ_ONCE(obj->pin_global))
> + if (!atomic_read(&obj->pin_global))
> return;
>
> i915_gem_object_lock(obj);
> @@ -425,7 +425,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
> /* Mark the global pin early so that we account for the
> * display coherency whilst setting up the cache domains.
> */
> - obj->pin_global++;
> + atomic_inc(&obj->pin_global);
>
> /* The display engine is not coherent with the LLC cache on gen6. As
> * a result, we make sure that the pinning that is about to occur is
> @@ -475,7 +475,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
> return vma;
>
> err_unpin_global:
> - obj->pin_global--;
> + atomic_dec(&obj->pin_global);
> return vma;
> }
>
> @@ -515,11 +515,11 @@ i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
>
> assert_object_held(obj);
>
> - if (WARN_ON(obj->pin_global == 0))
> + if (GEM_WARN_ON(!atomic_read(&obj->pin_global)))
> return;
>
> - if (--obj->pin_global == 0)
> - vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
> + if (atomic_dec_and_test(&obj->pin_global))
> + vma->display_alignment = I915_GTT_MIN_ALIGNMENT; /* XXX race */
>
> /* Bump the LRU to try and avoid premature eviction whilst flipping */
> i915_gem_object_bump_inactive_ggtt(obj);
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
> index 5efb9936e05b..11bb1a412e4f 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
> @@ -406,7 +406,8 @@ static inline bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
> if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
> return true;
>
> - return obj->pin_global; /* currently in use by HW, keep flushed */
> + /* Currently in use by HW (display engine)? Keep flushed. */
> + return atomic_read(&obj->pin_global);
> }
>
> static inline void __start_cpu_write(struct drm_i915_gem_object *obj)
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
> index 511fce7913ec..a09e74e27734 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
> @@ -153,7 +153,7 @@ struct drm_i915_gem_object {
> /** Count of VMA actually bound by this object */
> atomic_t bind_count;
> /** Count of how many global VMA are currently pinned for use by HW */
> - unsigned int pin_global;
> + atomic_t pin_global;
>
> struct {
> struct mutex lock; /* protects the pages and their use */
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
> index 94970f60d7f6..841f18386b54 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
> @@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
> * To simplify the scan, and to avoid walking the list of vma under the
> * object, we just check the count of its permanently pinned.
> */
> - if (READ_ONCE(obj->pin_global))
> + if (atomic_read(&obj->pin_global))
> return false;
>
> /* We can only return physical pages to the system if we can either
> diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
> index e103fcba6435..c36538f1b36c 100644
> --- a/drivers/gpu/drm/i915/i915_debugfs.c
> +++ b/drivers/gpu/drm/i915/i915_debugfs.c
> @@ -79,7 +79,7 @@ static int i915_capabilities(struct seq_file *m, void *data)
>
> static char get_pin_flag(struct drm_i915_gem_object *obj)
> {
> - return obj->pin_global ? 'p' : ' ';
> + return atomic_read(&obj->pin_global) ? 'p' : ' ';
> }
>
> static char get_tiling_flag(struct drm_i915_gem_object *obj)
> @@ -221,7 +221,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
> seq_printf(m, " (pinned x %d)", pin_count);
> if (obj->stolen)
> seq_printf(m, " (stolen: %08llx)", obj->stolen->start);
> - if (obj->pin_global)
> + if (atomic_read(&obj->pin_global))
> seq_printf(m, " (global)");
>
> engine = i915_gem_object_last_write_engine(obj);
> --
> 2.23.0
>
> _______________________________________________
> Intel-gfx mailing list
> Intel-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [PATCH 06/15] drm/i915: Mark up obj->pin_global as being atomic
2019-08-23 16:00 ` Mika Kuoppala
@ 2019-08-23 16:03 ` Chris Wilson
0 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 16:03 UTC (permalink / raw)
To: Mika Kuoppala, intel-gfx
Quoting Mika Kuoppala (2019-08-23 17:00:08)
> Chris Wilson <chris@chris-wilson.co.uk> writes:
>
> > Coming up next, we will want to manipulate the pin_global counter
> > outside of the principle locks, so convert it to an atomic.
> >
>
> I am not yet up to speed on pin_global but
> being part of a series where where this needs
> to be atomic, the patch does what it says on the tin.
pin_global really started off life as a means to keep the shrinker at
bay. It then grew as a marker for display objects in GEM, but really we
can use obj->frontbuffer now for that role. And we already use
obj->frontbuffer to delay shrinking of those objects.
Hmm. That might work.
-Chris
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 25+ messages in thread
* [PATCH 07/15] drm/i915: Only track bound elements of the GTT
2019-08-23 13:26 vm->mutex Chris Wilson
` (5 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 06/15] drm/i915: Mark up obj->pin_global as being atomic Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 08/15] drm/i915: Make i915_vma.flags atomic_t for mutex reduction Chris Wilson
` (10 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx; +Cc: Matthew Auld
The premise here is to simply avoiding having to acquire the vm->mutex
inside vma create/destroy to update the vm->unbound_lists, to avoid some
nasty lock recursions later.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Matthew Auld <matthew.auld@intel.com>
---
drivers/gpu/drm/i915/gem/i915_gem_stolen.c | 2 +-
drivers/gpu/drm/i915/i915_gem_gtt.c | 23 ++++---------------
drivers/gpu/drm/i915/i915_gem_gtt.h | 5 ----
drivers/gpu/drm/i915/i915_vma.c | 12 ++--------
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 2 +-
5 files changed, 8 insertions(+), 36 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index aa533b4ab5f5..2e1bfd5e4adf 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -689,7 +689,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
__i915_vma_set_map_and_fenceable(vma);
mutex_lock(&ggtt->vm.mutex);
- list_move_tail(&vma->vm_link, &ggtt->vm.bound_list);
+ list_add_tail(&vma->vm_link, &ggtt->vm.bound_list);
mutex_unlock(&ggtt->vm.mutex);
GEM_BUG_ON(i915_gem_object_is_shrinkable(obj));
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 43d0de15bb21..b46015060b40 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -499,19 +499,12 @@ static void i915_address_space_fini(struct i915_address_space *vm)
static void ppgtt_destroy_vma(struct i915_address_space *vm)
{
- struct list_head *phases[] = {
- &vm->bound_list,
- &vm->unbound_list,
- NULL,
- }, **phase;
+ struct i915_vma *vma, *vn;
mutex_lock(&vm->i915->drm.struct_mutex);
- for (phase = phases; *phase; phase++) {
- struct i915_vma *vma, *vn;
-
- list_for_each_entry_safe(vma, vn, *phase, vm_link)
- i915_vma_destroy(vma);
- }
+ list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link)
+ i915_vma_destroy(vma);
+ GEM_BUG_ON(!list_empty(&vm->bound_list));
mutex_unlock(&vm->i915->drm.struct_mutex);
}
@@ -522,9 +515,6 @@ static void __i915_vm_release(struct work_struct *work)
ppgtt_destroy_vma(vm);
- GEM_BUG_ON(!list_empty(&vm->bound_list));
- GEM_BUG_ON(!list_empty(&vm->unbound_list));
-
vm->cleanup(vm);
i915_address_space_fini(vm);
@@ -563,7 +553,6 @@ static void i915_address_space_init(struct i915_address_space *vm, int subclass)
stash_init(&vm->free_pages);
- INIT_LIST_HEAD(&vm->unbound_list);
INIT_LIST_HEAD(&vm->bound_list);
}
@@ -1889,10 +1878,6 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
INIT_LIST_HEAD(&vma->obj_link);
INIT_LIST_HEAD(&vma->closed_link);
- mutex_lock(&vma->vm->mutex);
- list_add(&vma->vm_link, &vma->vm->unbound_list);
- mutex_unlock(&vma->vm->mutex);
-
return vma;
}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index b97a47fc7a68..61bbbfb964b1 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -320,11 +320,6 @@ struct i915_address_space {
*/
struct list_head bound_list;
- /**
- * List of vma that are not unbound.
- */
- struct list_head unbound_list;
-
struct pagestash free_pages;
/* Global GTT */
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index e0e677b2a3a9..8fc2d44c20f5 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -218,10 +218,6 @@ vma_create(struct drm_i915_gem_object *obj,
spin_unlock(&obj->vma.lock);
- mutex_lock(&vm->mutex);
- list_add(&vma->vm_link, &vm->unbound_list);
- mutex_unlock(&vm->mutex);
-
return vma;
err_vma:
@@ -659,7 +655,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level));
mutex_lock(&vma->vm->mutex);
- list_move_tail(&vma->vm_link, &vma->vm->bound_list);
+ list_add_tail(&vma->vm_link, &vma->vm->bound_list);
mutex_unlock(&vma->vm->mutex);
if (vma->obj) {
@@ -687,7 +683,7 @@ i915_vma_remove(struct i915_vma *vma)
mutex_lock(&vma->vm->mutex);
drm_mm_remove_node(&vma->node);
- list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
+ list_del(&vma->vm_link);
mutex_unlock(&vma->vm->mutex);
/*
@@ -800,10 +796,6 @@ static void __i915_vma_destroy(struct i915_vma *vma)
GEM_BUG_ON(vma->node.allocated);
GEM_BUG_ON(vma->fence);
- mutex_lock(&vma->vm->mutex);
- list_del(&vma->vm_link);
- mutex_unlock(&vma->vm->mutex);
-
if (vma->obj) {
struct drm_i915_gem_object *obj = vma->obj;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index 31a51ca1ddcb..a90c9be95f8c 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -1237,7 +1237,7 @@ static void track_vma_bind(struct i915_vma *vma)
vma->pages = obj->mm.pages;
mutex_lock(&vma->vm->mutex);
- list_move_tail(&vma->vm_link, &vma->vm->bound_list);
+ list_add_tail(&vma->vm_link, &vma->vm->bound_list);
mutex_unlock(&vma->vm->mutex);
}
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 08/15] drm/i915: Make i915_vma.flags atomic_t for mutex reduction
2019-08-23 13:26 vm->mutex Chris Wilson
` (6 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 07/15] drm/i915: Only track bound elements of the GTT Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 09/15] drm/i915: Mark up address spaces that may need to allocate Chris Wilson
` (9 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
drivers/gpu/drm/i915/gem/i915_gem_object.c | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_stolen.c | 2 +-
drivers/gpu/drm/i915/i915_gem_gtt.c | 14 ++---
drivers/gpu/drm/i915/i915_vma.c | 29 +++++-----
drivers/gpu/drm/i915/i915_vma.h | 63 +++++++++++++---------
drivers/gpu/drm/i915/selftests/mock_gtt.c | 4 +-
6 files changed, 65 insertions(+), 49 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index d7855dc5a5c5..0ef60dae23a7 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -163,7 +163,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
list_for_each_entry_safe(vma, vn, &obj->vma.list, obj_link) {
GEM_BUG_ON(i915_vma_is_active(vma));
- vma->flags &= ~I915_VMA_PIN_MASK;
+ atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
i915_vma_destroy(vma);
}
GEM_BUG_ON(!list_empty(&obj->vma.list));
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index 2e1bfd5e4adf..0d81de1461b4 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -685,7 +685,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
vma->pages = obj->mm.pages;
- vma->flags |= I915_VMA_GLOBAL_BIND;
+ set_bit(I915_VMA_GLOBAL_BIND_BIT, __i915_vma_flags(vma));
__i915_vma_set_map_and_fenceable(vma);
mutex_lock(&ggtt->vm.mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index b46015060b40..f7782278db78 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -149,7 +149,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
u32 pte_flags;
int err;
- if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
+ if (!i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) {
err = vma->vm->allocate_va_range(vma->vm,
vma->node.start, vma->size);
if (err)
@@ -1872,7 +1872,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
vma->size = size;
vma->fence_size = size;
- vma->flags = I915_VMA_GGTT;
+ atomic_set(&vma->flags, I915_VMA_GGTT);
vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */
INIT_LIST_HEAD(&vma->obj_link);
@@ -2431,7 +2431,7 @@ static int ggtt_bind_vma(struct i915_vma *vma,
* GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
* upgrade to both bound if we bind either to avoid double-binding.
*/
- vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
+ atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
return 0;
}
@@ -2461,7 +2461,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
if (flags & I915_VMA_LOCAL_BIND) {
struct i915_ppgtt *alias = i915_vm_to_ggtt(vma->vm)->alias;
- if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
+ if (!i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) {
ret = alias->vm.allocate_va_range(&alias->vm,
vma->node.start,
vma->size);
@@ -2489,7 +2489,7 @@ static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
{
struct drm_i915_private *i915 = vma->vm->i915;
- if (vma->flags & I915_VMA_GLOBAL_BIND) {
+ if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
struct i915_address_space *vm = vma->vm;
intel_wakeref_t wakeref;
@@ -2497,7 +2497,7 @@ static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
vm->clear_range(vm, vma->node.start, vma->size);
}
- if (vma->flags & I915_VMA_LOCAL_BIND) {
+ if (i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) {
struct i915_address_space *vm =
&i915_vm_to_ggtt(vma->vm)->alias->vm;
@@ -3297,7 +3297,7 @@ static void ggtt_restore_mappings(struct i915_ggtt *ggtt)
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
struct drm_i915_gem_object *obj = vma->obj;
- if (!(vma->flags & I915_VMA_GLOBAL_BIND))
+ if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
continue;
mutex_unlock(&ggtt->vm.mutex);
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 8fc2d44c20f5..6c4aa9471af3 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -171,7 +171,7 @@ vma_create(struct drm_i915_gem_object *obj,
i915_gem_object_get_stride(obj));
GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
- vma->flags |= I915_VMA_GGTT;
+ __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
}
spin_lock(&obj->vma.lock);
@@ -321,7 +321,8 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
if (flags & PIN_USER)
bind_flags |= I915_VMA_LOCAL_BIND;
- vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
+ vma_flags = atomic_read(&vma->flags);
+ vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
if (flags & PIN_UPDATE)
bind_flags |= vma_flags;
else
@@ -336,7 +337,7 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
if (ret)
return ret;
- vma->flags |= bind_flags;
+ atomic_or(bind_flags, &vma->flags);
return 0;
}
@@ -355,7 +356,7 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
}
GEM_BUG_ON(!i915_vma_is_ggtt(vma));
- GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0);
+ GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND));
ptr = vma->iomap;
if (ptr == NULL) {
@@ -468,9 +469,9 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end;
if (mappable && fenceable)
- vma->flags |= I915_VMA_CAN_FENCE;
+ set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
else
- vma->flags &= ~I915_VMA_CAN_FENCE;
+ clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
}
static bool color_differs(struct drm_mm_node *node, unsigned long color)
@@ -543,7 +544,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
int ret;
GEM_BUG_ON(i915_vma_is_closed(vma));
- GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
+ GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
size = max(size, vma->size);
@@ -677,7 +678,7 @@ static void
i915_vma_remove(struct i915_vma *vma)
{
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
- GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
+ GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
vma->ops->clear_pages(vma);
@@ -708,7 +709,7 @@ i915_vma_remove(struct i915_vma *vma)
int __i915_vma_do_pin(struct i915_vma *vma,
u64 size, u64 alignment, u64 flags)
{
- const unsigned int bound = vma->flags;
+ const unsigned int bound = atomic_read(&vma->flags);
int ret;
lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
@@ -731,9 +732,9 @@ int __i915_vma_do_pin(struct i915_vma *vma,
if (ret)
goto err_remove;
- GEM_BUG_ON((vma->flags & I915_VMA_BIND_MASK) == 0);
+ GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_BIND_MASK));
- if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
+ if ((bound ^ atomic_read(&vma->flags)) & I915_VMA_GLOBAL_BIND)
__i915_vma_set_map_and_fenceable(vma);
GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
@@ -743,7 +744,7 @@ int __i915_vma_do_pin(struct i915_vma *vma,
if ((bound & I915_VMA_BIND_MASK) == 0) {
i915_vma_remove(vma);
GEM_BUG_ON(vma->pages);
- GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK);
+ GEM_BUG_ON(atomic_read(&vma->flags) & I915_VMA_BIND_MASK);
}
err_unpin:
__i915_vma_unpin(vma);
@@ -986,7 +987,7 @@ int i915_vma_unbind(struct i915_vma *vma)
mutex_unlock(&vma->vm->mutex);
__i915_vma_iounmap(vma);
- vma->flags &= ~I915_VMA_CAN_FENCE;
+ clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
}
GEM_BUG_ON(vma->fence);
GEM_BUG_ON(i915_vma_has_userfault(vma));
@@ -995,7 +996,7 @@ int i915_vma_unbind(struct i915_vma *vma)
trace_i915_vma_unbind(vma);
vma->ops->unbind_vma(vma);
}
- vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
+ atomic_and(~I915_VMA_BIND_MASK, &vma->flags);
i915_vma_remove(vma);
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 889fc7cb910a..38c6f3427e8c 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -72,7 +72,7 @@ struct i915_vma {
* that exist in the ctx->handle_vmas LUT for this vma.
*/
atomic_t open_count;
- unsigned long flags;
+ atomic_t flags;
/**
* How many users have pinned this object in GTT space.
*
@@ -97,18 +97,29 @@ struct i915_vma {
* users.
*/
#define I915_VMA_PIN_MASK 0xff
-#define I915_VMA_PIN_OVERFLOW BIT(8)
+#define I915_VMA_PIN_OVERFLOW_BIT 8
+#define I915_VMA_PIN_OVERFLOW ((int)BIT(I915_VMA_PIN_OVERFLOW_BIT))
/** Flags and address space this VMA is bound to */
-#define I915_VMA_GLOBAL_BIND BIT(9)
-#define I915_VMA_LOCAL_BIND BIT(10)
-#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
+#define I915_VMA_GLOBAL_BIND_BIT 9
+#define I915_VMA_LOCAL_BIND_BIT 10
-#define I915_VMA_GGTT BIT(11)
-#define I915_VMA_CAN_FENCE BIT(12)
+#define I915_VMA_GLOBAL_BIND ((int)BIT(I915_VMA_GLOBAL_BIND_BIT))
+#define I915_VMA_LOCAL_BIND ((int)BIT(I915_VMA_LOCAL_BIND_BIT))
+
+#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | \
+ I915_VMA_LOCAL_BIND | \
+ I915_VMA_PIN_OVERFLOW)
+
+#define I915_VMA_GGTT_BIT 11
+#define I915_VMA_CAN_FENCE_BIT 12
#define I915_VMA_USERFAULT_BIT 13
-#define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT)
-#define I915_VMA_GGTT_WRITE BIT(14)
+#define I915_VMA_GGTT_WRITE_BIT 14
+
+#define I915_VMA_GGTT ((int)BIT(I915_VMA_GGTT_BIT))
+#define I915_VMA_CAN_FENCE ((int)BIT(I915_VMA_CAN_FENCE_BIT))
+#define I915_VMA_USERFAULT ((int)BIT(I915_VMA_USERFAULT_BIT))
+#define I915_VMA_GGTT_WRITE ((int)BIT(I915_VMA_GGTT_WRITE_BIT))
struct i915_active active;
@@ -162,48 +173,52 @@ int __must_check i915_vma_move_to_active(struct i915_vma *vma,
struct i915_request *rq,
unsigned int flags);
+#define __i915_vma_flags(v) ((unsigned long *)&(v)->flags)
+
static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
{
- return vma->flags & I915_VMA_GGTT;
+ return test_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
}
static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma)
{
- return vma->flags & I915_VMA_GGTT_WRITE;
+ return test_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma));
}
static inline void i915_vma_set_ggtt_write(struct i915_vma *vma)
{
GEM_BUG_ON(!i915_vma_is_ggtt(vma));
- vma->flags |= I915_VMA_GGTT_WRITE;
+ set_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma));
}
-static inline void i915_vma_unset_ggtt_write(struct i915_vma *vma)
+static inline bool i915_vma_unset_ggtt_write(struct i915_vma *vma)
{
- vma->flags &= ~I915_VMA_GGTT_WRITE;
+ return test_and_clear_bit(I915_VMA_GGTT_WRITE_BIT,
+ __i915_vma_flags(vma));
}
void i915_vma_flush_writes(struct i915_vma *vma);
static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
{
- return vma->flags & I915_VMA_CAN_FENCE;
+ return test_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
}
static inline bool i915_vma_set_userfault(struct i915_vma *vma)
{
GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
- return __test_and_set_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
+ return __test_and_set_bit(I915_VMA_USERFAULT_BIT,
+ __i915_vma_flags(vma));
}
static inline void i915_vma_unset_userfault(struct i915_vma *vma)
{
- return __clear_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
+ return __clear_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));
}
static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
{
- return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
+ return test_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));
}
static inline bool i915_vma_is_closed(const struct i915_vma *vma)
@@ -330,7 +345,7 @@ i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
/* Pin early to prevent the shrinker/eviction logic from destroying
* our vma as we insert and bind.
*/
- if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) {
+ if (likely(((atomic_inc_return(&vma->flags) ^ flags) & I915_VMA_BIND_MASK) == 0)) {
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
return 0;
@@ -341,7 +356,7 @@ i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
- return vma->flags & I915_VMA_PIN_MASK;
+ return atomic_read(&vma->flags) & I915_VMA_PIN_MASK;
}
static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
@@ -351,13 +366,13 @@ static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
static inline void __i915_vma_pin(struct i915_vma *vma)
{
- vma->flags++;
- GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+ atomic_inc(&vma->flags);
+ GEM_BUG_ON(atomic_read(&vma->flags) & I915_VMA_PIN_OVERFLOW);
}
static inline void __i915_vma_unpin(struct i915_vma *vma)
{
- vma->flags--;
+ atomic_dec(&vma->flags);
}
static inline void i915_vma_unpin(struct i915_vma *vma)
@@ -370,7 +385,7 @@ static inline void i915_vma_unpin(struct i915_vma *vma)
static inline bool i915_vma_is_bound(const struct i915_vma *vma,
unsigned int where)
{
- return vma->flags & where;
+ return atomic_read(&vma->flags) & where;
}
/**
diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c
index e62a67e0f79c..366335981086 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c
@@ -43,7 +43,7 @@ static int mock_bind_ppgtt(struct i915_vma *vma,
u32 flags)
{
GEM_BUG_ON(flags & I915_VMA_GLOBAL_BIND);
- vma->flags |= I915_VMA_LOCAL_BIND;
+ set_bit(I915_VMA_LOCAL_BIND_BIT, __i915_vma_flags(vma));
return 0;
}
@@ -86,7 +86,7 @@ static int mock_bind_ggtt(struct i915_vma *vma,
enum i915_cache_level cache_level,
u32 flags)
{
- vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
+ atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
return 0;
}
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 09/15] drm/i915: Mark up address spaces that may need to allocate
2019-08-23 13:26 vm->mutex Chris Wilson
` (7 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 08/15] drm/i915: Make i915_vma.flags atomic_t for mutex reduction Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 10/15] drm/i915: Pull i915_vma_pin under the vm->mutex Chris Wilson
` (8 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Since we cannot allocate underneath the vm->mutex (it is used in the
direct-reclaim paths), we need to shift the allocations off into a
mutexless worker with fence recursion prevention. To know when we need
this protection, we mark up the address spaces that do allocate before
insertion.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_gem_gtt.c | 3 +++
drivers/gpu/drm/i915/i915_gem_gtt.h | 2 ++
2 files changed, 5 insertions(+)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index f7782278db78..342a84566e61 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -1503,6 +1503,7 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
goto err_free_pd;
}
+ ppgtt->vm.bind_alloc = I915_VMA_LOCAL_BIND;
ppgtt->vm.insert_entries = gen8_ppgtt_insert;
ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;
ppgtt->vm.clear_range = gen8_ppgtt_clear;
@@ -1950,6 +1951,7 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
ppgtt_init(&ppgtt->base, &i915->gt);
ppgtt->base.vm.top = 1;
+ ppgtt->base.vm.bind_alloc = I915_VMA_LOCAL_BIND;
ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
@@ -2581,6 +2583,7 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
goto err_ppgtt;
ggtt->alias = ppgtt;
+ ggtt->vm.bind_alloc |= ppgtt->vm.bind_alloc;
GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma);
ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 61bbbfb964b1..0f14722fd163 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -305,6 +305,8 @@ struct i915_address_space {
u64 total; /* size addr space maps (ex. 2GB for ggtt) */
u64 reserved; /* size addr space reserved */
+ unsigned int bind_alloc;
+
bool closed;
struct mutex mutex; /* protects vma and our lists */
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 10/15] drm/i915: Pull i915_vma_pin under the vm->mutex
2019-08-23 13:26 vm->mutex Chris Wilson
` (8 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 09/15] drm/i915: Mark up address spaces that may need to allocate Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 11/15] drm/i915: Push the i915_active.retire into a worker Chris Wilson
` (7 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Replace the struct_mutex requirement for pinning the i915_vma with the
local vm->mutex instead. Note that the vm->mutex is tainted by the
shrinker (we require unbinding from inside fs-reclaim) and so we cannot
allocate while holding that mutex. Instead we have to preallocate
workers to do allocate and apply the PTE updates after we have we
reserved their slot in the drm_mm (using fences to order the PTE writes
with the GPU work and with later unbind).
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
drivers/gpu/drm/i915/display/intel_display.c | 30 +-
drivers/gpu/drm/i915/display/intel_fbdev.c | 8 +-
drivers/gpu/drm/i915/display/intel_overlay.c | 11 +-
drivers/gpu/drm/i915/gem/i915_gem_context.c | 20 +-
drivers/gpu/drm/i915/gem/i915_gem_domain.c | 19 +-
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 9 +-
drivers/gpu/drm/i915/gem/i915_gem_mman.c | 21 +-
drivers/gpu/drm/i915/gem/i915_gem_object.c | 33 +-
drivers/gpu/drm/i915/gem/i915_gem_object.h | 5 +
drivers/gpu/drm/i915/gem/i915_gem_shrinker.c | 67 ----
drivers/gpu/drm/i915/gem/i915_gem_stolen.c | 5 +-
drivers/gpu/drm/i915/gem/i915_gem_tiling.c | 13 +-
drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 27 +-
.../gpu/drm/i915/gem/selftests/huge_pages.c | 23 +-
.../drm/i915/gem/selftests/i915_gem_mman.c | 2 -
drivers/gpu/drm/i915/gt/intel_gt.c | 5 +-
drivers/gpu/drm/i915/gt/selftest_hangcheck.c | 19 +-
drivers/gpu/drm/i915/gvt/aperture_gm.c | 12 +-
drivers/gpu/drm/i915/i915_active.c | 66 ++++
drivers/gpu/drm/i915/i915_active.h | 2 +
drivers/gpu/drm/i915/i915_active_types.h | 5 +
drivers/gpu/drm/i915/i915_gem.c | 77 ++--
drivers/gpu/drm/i915/i915_gem_evict.c | 20 +-
drivers/gpu/drm/i915/i915_gem_fence_reg.c | 4 +-
drivers/gpu/drm/i915/i915_gem_gtt.c | 100 +++--
drivers/gpu/drm/i915/i915_gem_gtt.h | 31 +-
drivers/gpu/drm/i915/i915_perf.c | 26 +-
drivers/gpu/drm/i915/i915_vma.c | 341 +++++++++++-------
drivers/gpu/drm/i915/i915_vma.h | 59 ++-
.../gpu/drm/i915/selftests/i915_gem_evict.c | 36 +-
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 88 +++--
drivers/gpu/drm/i915/selftests/i915_vma.c | 2 +
32 files changed, 617 insertions(+), 569 deletions(-)
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index b51d1ceb8739..e2d166f1da6a 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -2079,8 +2079,6 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
unsigned int pinctl;
u32 alignment;
- WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-
alignment = intel_surf_alignment(fb, 0);
/* Note that the w/a also requires 64 PTE of padding following the
@@ -2161,8 +2159,6 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
-
i915_gem_object_lock(vma->obj);
if (flags & PLANE_HAS_FENCE)
i915_vma_unpin_fence(vma);
@@ -3063,12 +3059,10 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
return false;
}
- mutex_lock(&dev->struct_mutex);
obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
base_aligned,
base_aligned,
size_aligned);
- mutex_unlock(&dev->struct_mutex);
if (!obj)
return false;
@@ -3230,13 +3224,11 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
intel_state->color_plane[0].stride =
intel_fb_pitch(fb, 0, intel_state->base.rotation);
- mutex_lock(&dev->struct_mutex);
intel_state->vma =
intel_pin_and_fence_fb_obj(fb,
&intel_state->view,
intel_plane_uses_fence(intel_state),
&intel_state->flags);
- mutex_unlock(&dev->struct_mutex);
if (IS_ERR(intel_state->vma)) {
DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
intel_crtc->pipe, PTR_ERR(intel_state->vma));
@@ -14330,8 +14322,6 @@ static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
* bits. Some older platforms need special physical address handling for
* cursor planes.
*
- * Must be called with struct_mutex held.
- *
* Returns 0 on success, negative error code on failure.
*/
int
@@ -14388,15 +14378,8 @@ intel_prepare_plane_fb(struct drm_plane *plane,
if (ret)
return ret;
- ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
- if (ret) {
- i915_gem_object_unpin_pages(obj);
- return ret;
- }
-
ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
- mutex_unlock(&dev_priv->drm.struct_mutex);
i915_gem_object_unpin_pages(obj);
if (ret)
return ret;
@@ -14445,8 +14428,6 @@ intel_prepare_plane_fb(struct drm_plane *plane,
* @old_state: the state from the previous modeset
*
* Cleans up a framebuffer that has just been removed from a plane.
- *
- * Must be called with struct_mutex held.
*/
void
intel_cleanup_plane_fb(struct drm_plane *plane,
@@ -14462,9 +14443,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
}
/* Should only be called after a successful intel_prepare_plane_fb()! */
- mutex_lock(&dev_priv->drm.struct_mutex);
intel_plane_unpin_fb(to_intel_plane_state(old_state));
- mutex_unlock(&dev_priv->drm.struct_mutex);
}
int
@@ -14664,7 +14643,6 @@ intel_legacy_cursor_update(struct drm_plane *plane,
u32 src_w, u32 src_h,
struct drm_modeset_acquire_ctx *ctx)
{
- struct drm_i915_private *dev_priv = to_i915(crtc->dev);
struct drm_plane_state *old_plane_state, *new_plane_state;
struct intel_plane *intel_plane = to_intel_plane(plane);
struct intel_crtc_state *crtc_state =
@@ -14730,13 +14708,9 @@ intel_legacy_cursor_update(struct drm_plane *plane,
if (ret)
goto out_free;
- ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
- if (ret)
- goto out_free;
-
ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
if (ret)
- goto out_unlock;
+ goto out_free;
intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_FLIP);
intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->fb),
@@ -14766,8 +14740,6 @@ intel_legacy_cursor_update(struct drm_plane *plane,
intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
-out_unlock:
- mutex_unlock(&dev_priv->drm.struct_mutex);
out_free:
if (new_crtc_state)
intel_crtc_destroy_state(crtc, &new_crtc_state->base);
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c
index d59eee5c5d9c..a3dea3f2dacd 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c
@@ -204,7 +204,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
sizes->fb_height = intel_fb->base.height;
}
- mutex_lock(&dev->struct_mutex);
wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
/* Pin the GGTT vma for our access via info->screen_base.
@@ -266,7 +265,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
ifbdev->vma_flags = flags;
intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
- mutex_unlock(&dev->struct_mutex);
vga_switcheroo_client_fb_set(pdev, info);
return 0;
@@ -274,7 +272,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
intel_unpin_fb_vma(vma, flags);
out_unlock:
intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
- mutex_unlock(&dev->struct_mutex);
return ret;
}
@@ -291,11 +288,8 @@ static void intel_fbdev_destroy(struct intel_fbdev *ifbdev)
drm_fb_helper_fini(&ifbdev->helper);
- if (ifbdev->vma) {
- mutex_lock(&ifbdev->helper.dev->struct_mutex);
+ if (ifbdev->vma)
intel_unpin_fb_vma(ifbdev->vma, ifbdev->vma_flags);
- mutex_unlock(&ifbdev->helper.dev->struct_mutex);
- }
if (ifbdev->fb)
drm_framebuffer_remove(&ifbdev->fb->base);
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
index 29edfc343716..4f36557b3f3b 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.c
+++ b/drivers/gpu/drm/i915/display/intel_overlay.c
@@ -1303,15 +1303,11 @@ static int get_registers(struct intel_overlay *overlay, bool use_phys)
struct i915_vma *vma;
int err;
- mutex_lock(&i915->drm.struct_mutex);
-
obj = i915_gem_object_create_stolen(i915, PAGE_SIZE);
if (obj == NULL)
obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
- if (IS_ERR(obj)) {
- err = PTR_ERR(obj);
- goto err_unlock;
- }
+ if (IS_ERR(obj))
+ return PTR_ERR(obj);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
if (IS_ERR(vma)) {
@@ -1332,13 +1328,10 @@ static int get_registers(struct intel_overlay *overlay, bool use_phys)
}
overlay->reg_bo = obj;
- mutex_unlock(&i915->drm.struct_mutex);
return 0;
err_put_bo:
i915_gem_object_put(obj);
-err_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
return err;
}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 863b5cc1ffa9..23c4a888041f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -313,8 +313,6 @@ static void i915_gem_context_free(struct i915_gem_context *ctx)
GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
release_hw_id(ctx);
- if (ctx->vm)
- i915_vm_put(ctx->vm);
free_engines(rcu_access_pointer(ctx->engines));
mutex_destroy(&ctx->engines_mutex);
@@ -379,9 +377,13 @@ void i915_gem_context_release(struct kref *ref)
static void context_close(struct i915_gem_context *ctx)
{
+ i915_gem_context_set_closed(ctx);
+
+ if (ctx->vm)
+ i915_vm_close(ctx->vm);
+
mutex_lock(&ctx->mutex);
- i915_gem_context_set_closed(ctx);
ctx->file_priv = ERR_PTR(-EBADF);
/*
@@ -474,7 +476,7 @@ __set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm)
GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old));
- ctx->vm = i915_vm_get(vm);
+ ctx->vm = i915_vm_open(vm);
context_apply_all(ctx, __apply_ppgtt, vm);
return old;
@@ -488,7 +490,7 @@ static void __assign_ppgtt(struct i915_gem_context *ctx,
vm = __set_ppgtt(ctx, vm);
if (vm)
- i915_vm_put(vm);
+ i915_vm_close(vm);
}
static void __set_timeline(struct intel_timeline **dst,
@@ -953,7 +955,7 @@ static int get_ppgtt(struct drm_i915_file_private *file_priv,
if (ret < 0)
goto err_unlock;
- i915_vm_get(vm);
+ i915_vm_open(vm);
args->size = 0;
args->value = ret;
@@ -973,7 +975,7 @@ static void set_ppgtt_barrier(void *data)
if (INTEL_GEN(old->i915) < 8)
gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old));
- i915_vm_put(old);
+ i915_vm_close(old);
}
static int emit_ppgtt_update(struct i915_request *rq, void *data)
@@ -1093,8 +1095,8 @@ static int set_ppgtt(struct drm_i915_file_private *file_priv,
set_ppgtt_barrier,
old);
if (err) {
- i915_vm_put(__set_ppgtt(ctx, old));
- i915_vm_put(old);
+ i915_vm_close(__set_ppgtt(ctx, old));
+ i915_vm_close(old);
}
unlock:
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index b7be9ad2b62c..51c22e044eee 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -202,7 +202,11 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
i915_gem_valid_gtt_space(vma, cache_level))
continue;
- ret = i915_vma_unbind(vma);
+ ret = mutex_lock_interruptible(&vma->vm->mutex);
+ if (!ret) {
+ ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ }
if (ret)
return ret;
@@ -288,7 +292,12 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
if (!drm_mm_node_allocated(&vma->node))
continue;
- ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
+ /* Wait for an earlier async bind */
+ ret = i915_active_wait(&vma->active);
+ if (ret)
+ return ret;
+
+ ret = i915_vma_bind(vma, cache_level, PIN_UPDATE, NULL);
if (ret)
return ret;
}
@@ -389,16 +398,11 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
if (ret)
goto out;
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- goto out;
-
ret = i915_gem_object_lock_interruptible(obj);
if (ret == 0) {
ret = i915_gem_object_set_cache_level(obj, level);
i915_gem_object_unlock(obj);
}
- mutex_unlock(&i915->drm.struct_mutex);
out:
i915_gem_object_put(obj);
@@ -491,6 +495,7 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
if (!drm_mm_node_allocated(&vma->node))
continue;
+ GEM_BUG_ON(vma->vm != &i915->ggtt.vm);
list_move_tail(&vma->vm_link, &vma->vm->bound_list);
}
mutex_unlock(&i915->ggtt.vm.mutex);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index b5f6937369ea..05a3e24248a5 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -549,8 +549,11 @@ eb_add_vma(struct i915_execbuffer *eb,
eb_unreserve_vma(vma, vma->exec_flags);
list_add_tail(&vma->exec_link, &eb->unbound);
- if (drm_mm_node_allocated(&vma->node))
+ if (drm_mm_node_allocated(&vma->node)) {
+ mutex_lock(&vma->vm->mutex);
err = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ }
if (unlikely(err))
vma->exec_flags = NULL;
}
@@ -697,7 +700,9 @@ static int eb_reserve(struct i915_execbuffer *eb)
case 1:
/* Too fragmented, unbind everything and retry */
+ mutex_lock(&eb->context->vm->mutex);
err = i915_gem_evict_vm(eb->context->vm);
+ mutex_unlock(&eb->context->vm->mutex);
if (err)
return err;
break;
@@ -1388,7 +1393,7 @@ eb_relocate_entry(struct i915_execbuffer *eb,
if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
IS_GEN(eb->i915, 6)) {
err = i915_vma_bind(target, target->obj->cache_level,
- PIN_GLOBAL);
+ PIN_GLOBAL, NULL);
if (WARN_ONCE(err,
"Unexpected failure to bind target VMA!"))
return err;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 595539a09e38..0440dbfda295 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -251,16 +251,6 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
goto err_rpm;
}
- ret = i915_mutex_lock_interruptible(dev);
- if (ret)
- goto err_reset;
-
- /* Access to snoopable pages through the GTT is incoherent. */
- if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
- ret = -EFAULT;
- goto err_unlock;
- }
-
/* Now pin it into the GTT as needed */
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
PIN_MAPPABLE |
@@ -290,7 +280,13 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
}
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
- goto err_unlock;
+ goto err_reset;
+ }
+
+ /* Access to snoopable pages through the GTT is incoherent. */
+ if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
+ ret = -EFAULT;
+ goto err_unpin;
}
ret = i915_vma_pin_fence(vma);
@@ -319,13 +315,10 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
i915_vma_set_ggtt_write(vma);
-
err_fence:
i915_vma_unpin_fence(vma);
err_unpin:
__i915_vma_unpin(vma);
-err_unlock:
- mutex_unlock(&dev->struct_mutex);
err_reset:
intel_gt_reset_unlock(ggtt->vm.gt, srcu);
err_rpm:
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index 0ef60dae23a7..dbf9be9a79f4 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -155,21 +155,30 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
llist_for_each_entry_safe(obj, on, freed, freed) {
- struct i915_vma *vma, *vn;
-
trace_i915_gem_object_destroy(obj);
- mutex_lock(&i915->drm.struct_mutex);
-
- list_for_each_entry_safe(vma, vn, &obj->vma.list, obj_link) {
- GEM_BUG_ON(i915_vma_is_active(vma));
- atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
- i915_vma_destroy(vma);
+ if (!list_empty(&obj->vma.list)) {
+ struct i915_vma *vma;
+
+ /*
+ * Note that the vma keeps an object reference while
+ * it is active, so it *should* not sleep while we
+ * destroy it. Our debug code errs insits it *might*.
+ * For the moment, play along.
+ */
+ spin_lock(&obj->vma.lock);
+ while ((vma = list_first_entry_or_null(&obj->vma.list,
+ struct i915_vma,
+ obj_link))) {
+ GEM_BUG_ON(vma->obj != obj);
+ spin_unlock(&obj->vma.lock);
+
+ i915_vma_destroy(vma);
+
+ spin_lock(&obj->vma.lock);
+ }
+ spin_unlock(&obj->vma.lock);
}
- GEM_BUG_ON(!list_empty(&obj->vma.list));
- GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma.tree));
-
- mutex_unlock(&i915->drm.struct_mutex);
GEM_BUG_ON(atomic_read(&obj->bind_count));
GEM_BUG_ON(obj->userfault_count);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 11bb1a412e4f..8dfd3e170d32 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -106,6 +106,11 @@ static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj)
dma_resv_lock(obj->base.resv, NULL);
}
+static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj)
+{
+ return dma_resv_trylock(obj->base.resv);
+}
+
static inline int
i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj)
{
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
index 841f18386b54..cb9d5c2586bd 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
@@ -16,40 +16,6 @@
#include "i915_trace.h"
-static bool shrinker_lock(struct drm_i915_private *i915,
- unsigned int flags,
- bool *unlock)
-{
- struct mutex *m = &i915->drm.struct_mutex;
-
- switch (mutex_trylock_recursive(m)) {
- case MUTEX_TRYLOCK_RECURSIVE:
- *unlock = false;
- return true;
-
- case MUTEX_TRYLOCK_FAILED:
- *unlock = false;
- if (flags & I915_SHRINK_ACTIVE &&
- mutex_lock_killable_nested(m, I915_MM_SHRINKER) == 0)
- *unlock = true;
- return *unlock;
-
- case MUTEX_TRYLOCK_SUCCESS:
- *unlock = true;
- return true;
- }
-
- BUG();
-}
-
-static void shrinker_unlock(struct drm_i915_private *i915, bool unlock)
-{
- if (!unlock)
- return;
-
- mutex_unlock(&i915->drm.struct_mutex);
-}
-
static bool swap_available(void)
{
return get_nr_swap_pages() > 0;
@@ -162,10 +128,6 @@ i915_gem_shrink(struct drm_i915_private *i915,
intel_wakeref_t wakeref = 0;
unsigned long count = 0;
unsigned long scanned = 0;
- bool unlock;
-
- if (!shrinker_lock(i915, shrink, &unlock))
- return 0;
/*
* When shrinking the active list, we should also consider active
@@ -275,8 +237,6 @@ i915_gem_shrink(struct drm_i915_private *i915,
if (shrink & I915_SHRINK_BOUND)
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
- shrinker_unlock(i915, unlock);
-
if (nr_scanned)
*nr_scanned += scanned;
return count;
@@ -346,13 +306,9 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
struct drm_i915_private *i915 =
container_of(shrinker, struct drm_i915_private, mm.shrinker);
unsigned long freed;
- bool unlock;
sc->nr_scanned = 0;
- if (!shrinker_lock(i915, 0, &unlock))
- return SHRINK_STOP;
-
freed = i915_gem_shrink(i915,
sc->nr_to_scan,
&sc->nr_scanned,
@@ -373,8 +329,6 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
}
}
- shrinker_unlock(i915, unlock);
-
return sc->nr_scanned ? freed : SHRINK_STOP;
}
@@ -426,10 +380,6 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
struct i915_vma *vma, *next;
unsigned long freed_pages = 0;
intel_wakeref_t wakeref;
- bool unlock;
-
- if (!shrinker_lock(i915, 0, &unlock))
- return NOTIFY_DONE;
with_intel_runtime_pm(&i915->runtime_pm, wakeref)
freed_pages += i915_gem_shrink(i915, -1UL, NULL,
@@ -446,15 +396,11 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
if (!vma->iomap || i915_vma_is_active(vma))
continue;
- mutex_unlock(&i915->ggtt.vm.mutex);
if (i915_vma_unbind(vma) == 0)
freed_pages += count;
- mutex_lock(&i915->ggtt.vm.mutex);
}
mutex_unlock(&i915->ggtt.vm.mutex);
- shrinker_unlock(i915, unlock);
-
*(unsigned long *)ptr += freed_pages;
return NOTIFY_DONE;
}
@@ -497,22 +443,9 @@ void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915,
fs_reclaim_acquire(GFP_KERNEL);
- /*
- * As we invariably rely on the struct_mutex within the shrinker,
- * but have a complicated recursion dance, taint all the mutexes used
- * within the shrinker with the struct_mutex. For completeness, we
- * taint with all subclass of struct_mutex, even though we should
- * only need tainting by I915_MM_NORMAL to catch possible ABBA
- * deadlocks from using struct_mutex inside @mutex.
- */
- mutex_acquire(&i915->drm.struct_mutex.dep_map,
- I915_MM_SHRINKER, 0, _RET_IP_);
-
mutex_acquire(&mutex->dep_map, 0, 0, _RET_IP_);
mutex_release(&mutex->dep_map, 0, _RET_IP_);
- mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_);
-
fs_reclaim_release(GFP_KERNEL);
if (unlock)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index 0d81de1461b4..158ddd6e735f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -621,8 +621,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return NULL;
- lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
&stolen_offset, >t_offset, &size);
@@ -674,11 +672,13 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
* setting up the GTT space. The actual reservation will occur
* later.
*/
+ mutex_lock(&ggtt->vm.mutex);
ret = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
size, gtt_offset, obj->cache_level,
0);
if (ret) {
DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n");
+ mutex_unlock(&ggtt->vm.mutex);
goto err_pages;
}
@@ -688,7 +688,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
set_bit(I915_VMA_GLOBAL_BIND_BIT, __i915_vma_flags(vma));
__i915_vma_set_map_and_fenceable(vma);
- mutex_lock(&ggtt->vm.mutex);
list_add_tail(&vma->vm_link, &ggtt->vm.bound_list);
mutex_unlock(&ggtt->vm.mutex);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
index ca0c2f451742..32004401aa96 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
@@ -181,22 +181,25 @@ static int
i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
int tiling_mode, unsigned int stride)
{
+ struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt;
struct i915_vma *vma;
int ret;
if (tiling_mode == I915_TILING_NONE)
return 0;
+ mutex_lock(&ggtt->vm.mutex);
for_each_ggtt_vma(vma, obj) {
if (i915_vma_fence_prepare(vma, tiling_mode, stride))
continue;
ret = i915_vma_unbind(vma);
if (ret)
- return ret;
+ break;
}
+ mutex_unlock(&ggtt->vm.mutex);
- return 0;
+ return ret;
}
int
@@ -212,7 +215,6 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
- lockdep_assert_held(&i915->drm.struct_mutex);
if ((tiling | stride) == obj->tiling_and_stride)
return 0;
@@ -364,12 +366,7 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
}
}
- err = mutex_lock_interruptible(&dev->struct_mutex);
- if (err)
- goto err;
-
err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
- mutex_unlock(&dev->struct_mutex);
/* We have to maintain this existing ABI... */
args->stride = i915_gem_object_get_stride(obj);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
index 74da35611d7c..cd36236e3faf 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
@@ -92,7 +92,6 @@ userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
struct i915_mmu_notifier *mn =
container_of(_mn, struct i915_mmu_notifier, mn);
struct interval_tree_node *it;
- struct mutex *unlock = NULL;
unsigned long end;
int ret = 0;
@@ -129,33 +128,13 @@ userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
}
spin_unlock(&mn->lock);
- if (!unlock) {
- unlock = &mn->mm->i915->drm.struct_mutex;
-
- switch (mutex_trylock_recursive(unlock)) {
- default:
- case MUTEX_TRYLOCK_FAILED:
- if (mutex_lock_killable_nested(unlock, I915_MM_SHRINKER)) {
- i915_gem_object_put(obj);
- return -EINTR;
- }
- /* fall through */
- case MUTEX_TRYLOCK_SUCCESS:
- break;
-
- case MUTEX_TRYLOCK_RECURSIVE:
- unlock = ERR_PTR(-EEXIST);
- break;
- }
- }
-
ret = i915_gem_object_unbind(obj,
I915_GEM_OBJECT_UNBIND_ACTIVE);
if (ret == 0)
ret = __i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
i915_gem_object_put(obj);
if (ret)
- goto unlock;
+ return ret;
spin_lock(&mn->lock);
@@ -168,10 +147,6 @@ userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
}
spin_unlock(&mn->lock);
-unlock:
- if (!IS_ERR_OR_NULL(unlock))
- mutex_unlock(unlock);
-
return ret;
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
index a0a272556289..5ef0a3b44d90 100644
--- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
@@ -25,6 +25,17 @@ static const unsigned int page_sizes[] = {
I915_GTT_PAGE_SIZE_4K,
};
+static int unlocked_vma_unbind(struct i915_vma *vma)
+{
+ int ret;
+
+ mutex_lock(&vma->vm->mutex);
+ ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+
+ return ret;
+}
+
static unsigned int get_largest_page_size(struct drm_i915_private *i915,
u64 rem)
{
@@ -333,7 +344,11 @@ static int igt_check_page_sizes(struct i915_vma *vma)
struct drm_i915_private *i915 = vma->vm->i915;
unsigned int supported = INTEL_INFO(i915)->page_sizes;
struct drm_i915_gem_object *obj = vma->obj;
- int err = 0;
+ int err;
+
+ err = i915_active_wait(&vma->active);
+ if (err)
+ return err;
if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
@@ -526,7 +541,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg)
* pages.
*/
for (offset = 4096; offset < page_size; offset += 4096) {
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
if (err) {
i915_vma_close(vma);
goto out_unpin;
@@ -941,7 +956,7 @@ static int __igt_write_huge(struct intel_context *ce,
if (IS_ERR(vma))
return PTR_ERR(vma);
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
if (err)
goto out_vma_close;
@@ -1387,7 +1402,7 @@ static int igt_ppgtt_pin_update(void *arg)
goto out_unpin;
}
- err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE);
+ err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE, NULL);
if (err)
goto out_unpin;
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
index 1d27babff0ce..9c217dfe96a9 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
@@ -205,7 +205,6 @@ static int igt_partial_tiling(void *arg)
goto out;
}
- mutex_lock(&i915->drm.struct_mutex);
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
if (1) {
@@ -318,7 +317,6 @@ next_tiling: ;
out_unlock:
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
- mutex_unlock(&i915->drm.struct_mutex);
i915_gem_object_unpin_pages(obj);
out:
i915_gem_object_put(obj);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index d48ec9a76ed1..c2afffb94474 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -207,11 +207,12 @@ void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
struct intel_uncore *uncore = gt->uncore;
+ unsigned long flags;
- spin_lock_irq(&uncore->lock);
+ spin_lock_irqsave(&uncore->lock, flags);
intel_uncore_posting_read_fw(uncore,
RING_HEAD(RENDER_RING_BASE));
- spin_unlock_irq(&uncore->lock);
+ spin_unlock_irqrestore(&uncore->lock, flags);
}
}
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index a0098fc35921..e53eea1050f8 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -1127,15 +1127,14 @@ static int evict_vma(void *data)
{
struct evict_vma *arg = data;
struct i915_address_space *vm = arg->vma->vm;
- struct drm_i915_private *i915 = vm->i915;
struct drm_mm_node evict = arg->vma->node;
int err;
complete(&arg->completion);
- mutex_lock(&i915->drm.struct_mutex);
+ mutex_lock(&vm->mutex);
err = i915_gem_evict_for_node(vm, &evict, 0);
- mutex_unlock(&i915->drm.struct_mutex);
+ mutex_unlock(&vm->mutex);
return err;
}
@@ -1143,39 +1142,33 @@ static int evict_vma(void *data)
static int evict_fence(void *data)
{
struct evict_vma *arg = data;
- struct drm_i915_private *i915 = arg->vma->vm->i915;
int err;
complete(&arg->completion);
- mutex_lock(&i915->drm.struct_mutex);
-
/* Mark the fence register as dirty to force the mmio update. */
err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512);
if (err) {
pr_err("Invalid Y-tiling settings; err:%d\n", err);
- goto out_unlock;
+ return err;
}
err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE);
if (err) {
pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err);
- goto out_unlock;
+ return err;
}
err = i915_vma_pin_fence(arg->vma);
i915_vma_unpin(arg->vma);
if (err) {
pr_err("Unable to pin Y-tiled fence; err:%d\n", err);
- goto out_unlock;
+ return err;
}
i915_vma_unpin_fence(arg->vma);
-out_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
-
- return err;
+ return 0;
}
static int __igt_reset_evict_vma(struct intel_gt *gt,
diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c
index 5ff2437b2998..d996bbc7ea59 100644
--- a/drivers/gpu/drm/i915/gvt/aperture_gm.c
+++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c
@@ -61,14 +61,14 @@ static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm)
flags = PIN_MAPPABLE;
}
- mutex_lock(&dev_priv->drm.struct_mutex);
+ mutex_lock(&dev_priv->ggtt.vm.mutex);
mmio_hw_access_pre(dev_priv);
ret = i915_gem_gtt_insert(&dev_priv->ggtt.vm, node,
size, I915_GTT_PAGE_SIZE,
I915_COLOR_UNEVICTABLE,
start, end, flags);
mmio_hw_access_post(dev_priv);
- mutex_unlock(&dev_priv->drm.struct_mutex);
+ mutex_unlock(&dev_priv->ggtt.vm.mutex);
if (ret)
gvt_err("fail to alloc %s gm space from host\n",
high_gm ? "high" : "low");
@@ -98,9 +98,9 @@ static int alloc_vgpu_gm(struct intel_vgpu *vgpu)
return 0;
out_free_aperture:
- mutex_lock(&dev_priv->drm.struct_mutex);
+ mutex_lock(&dev_priv->ggtt.vm.mutex);
drm_mm_remove_node(&vgpu->gm.low_gm_node);
- mutex_unlock(&dev_priv->drm.struct_mutex);
+ mutex_unlock(&dev_priv->ggtt.vm.mutex);
return ret;
}
@@ -108,10 +108,10 @@ static void free_vgpu_gm(struct intel_vgpu *vgpu)
{
struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
- mutex_lock(&dev_priv->drm.struct_mutex);
+ mutex_lock(&dev_priv->ggtt.vm.mutex);
drm_mm_remove_node(&vgpu->gm.low_gm_node);
drm_mm_remove_node(&vgpu->gm.high_gm_node);
- mutex_unlock(&dev_priv->drm.struct_mutex);
+ mutex_unlock(&dev_priv->ggtt.vm.mutex);
}
/**
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index 48e16ad93bbd..5e9f826ff132 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -142,6 +142,7 @@ __active_retire(struct i915_active *ref)
if (!retire)
return;
+ GEM_BUG_ON(rcu_access_pointer(ref->excl));
rbtree_postorder_for_each_entry_safe(it, n, &root, node) {
GEM_BUG_ON(i915_active_request_isset(&it->base));
kmem_cache_free(global.slab_cache, it);
@@ -241,6 +242,8 @@ void __i915_active_init(struct drm_i915_private *i915,
ref->flags = 0;
ref->active = active;
ref->retire = retire;
+
+ ref->excl = NULL;
ref->tree = RB_ROOT;
ref->cache = NULL;
init_llist_head(&ref->preallocated_barriers);
@@ -337,6 +340,45 @@ int i915_active_ref(struct i915_active *ref,
return err;
}
+static void excl_cb(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+ struct i915_active *ref = container_of(cb, typeof(*ref), excl_cb);
+
+ RCU_INIT_POINTER(ref->excl, NULL);
+ dma_fence_put(f);
+
+ active_retire(ref);
+}
+
+void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f)
+{
+ GEM_BUG_ON(i915_active_is_idle(ref));
+
+ dma_fence_get(f);
+
+ rcu_read_lock();
+ if (rcu_access_pointer(ref->excl)) {
+ struct dma_fence *old;
+
+ old = dma_fence_get_rcu_safe(&ref->excl);
+ if (old) {
+ if (dma_fence_remove_callback(old, &ref->excl_cb))
+ atomic_dec(&ref->count);
+ dma_fence_put(old);
+ }
+ }
+ rcu_read_unlock();
+
+ atomic_inc(&ref->count);
+ rcu_assign_pointer(ref->excl, f);
+
+ if (dma_fence_add_callback(f, &ref->excl_cb, excl_cb)) {
+ RCU_INIT_POINTER(ref->excl, NULL);
+ atomic_dec(&ref->count);
+ dma_fence_put(f);
+ }
+}
+
int i915_active_acquire(struct i915_active *ref)
{
int err;
@@ -395,6 +437,25 @@ void i915_active_ungrab(struct i915_active *ref)
__active_ungrab(ref);
}
+static int excl_wait(struct i915_active *ref)
+{
+ struct dma_fence *old;
+ int err = 0;
+
+ if (!rcu_access_pointer(ref->excl))
+ return 0;
+
+ rcu_read_lock();
+ old = dma_fence_get_rcu_safe(&ref->excl);
+ rcu_read_unlock();
+ if (old) {
+ err = dma_fence_wait(old, true);
+ dma_fence_put(old);
+ }
+
+ return err;
+}
+
int i915_active_wait(struct i915_active *ref)
{
struct active_node *it, *n;
@@ -415,6 +476,10 @@ int i915_active_wait(struct i915_active *ref)
return 0;
}
+ err = excl_wait(ref);
+ if (err)
+ goto out;
+
rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
if (is_barrier(&it->base)) { /* unconnected idle-barrier */
err = -EBUSY;
@@ -426,6 +491,7 @@ int i915_active_wait(struct i915_active *ref)
break;
}
+out:
__active_retire(ref);
if (err)
return err;
diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h
index f95058f99057..bc1c9d5d4f63 100644
--- a/drivers/gpu/drm/i915/i915_active.h
+++ b/drivers/gpu/drm/i915/i915_active.h
@@ -373,6 +373,8 @@ int i915_active_ref(struct i915_active *ref,
struct intel_timeline *tl,
struct i915_request *rq);
+void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f);
+
int i915_active_wait(struct i915_active *ref);
int i915_request_await_active(struct i915_request *rq,
diff --git a/drivers/gpu/drm/i915/i915_active_types.h b/drivers/gpu/drm/i915/i915_active_types.h
index 1854e7d168c1..86e7a232ea3c 100644
--- a/drivers/gpu/drm/i915/i915_active_types.h
+++ b/drivers/gpu/drm/i915/i915_active_types.h
@@ -8,6 +8,7 @@
#define _I915_ACTIVE_TYPES_H_
#include <linux/atomic.h>
+#include <linux/dma-fence.h>
#include <linux/llist.h>
#include <linux/mutex.h>
#include <linux/rbtree.h>
@@ -51,6 +52,10 @@ struct i915_active {
struct mutex mutex;
atomic_t count;
+ /* Preallocated "exclusive" node */
+ struct dma_fence __rcu *excl;
+ struct dma_fence_cb excl_cb;
+
unsigned long flags;
#define I915_ACTIVE_GRAB_BIT 0
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index ec9a46c276de..85323cd08022 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -62,20 +62,31 @@
#include "intel_pm.h"
static int
-insert_mappable_node(struct i915_ggtt *ggtt,
- struct drm_mm_node *node, u32 size)
+insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
{
+ int err;
+
+ err = mutex_lock_interruptible(&ggtt->vm.mutex);
+ if (err)
+ return err;
+
memset(node, 0, sizeof(*node));
- return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
- size, 0, I915_COLOR_UNEVICTABLE,
- 0, ggtt->mappable_end,
- DRM_MM_INSERT_LOW);
+ err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
+ size, 0, I915_COLOR_UNEVICTABLE,
+ 0, ggtt->mappable_end,
+ DRM_MM_INSERT_LOW);
+
+ mutex_unlock(&ggtt->vm.mutex);
+
+ return err;
}
static void
-remove_mappable_node(struct drm_mm_node *node)
+remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
{
+ mutex_lock(&ggtt->vm.mutex);
drm_mm_remove_node(node);
+ mutex_unlock(&ggtt->vm.mutex);
}
int
@@ -87,7 +98,8 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
struct i915_vma *vma;
u64 pinned;
- mutex_lock(&ggtt->vm.mutex);
+ if (mutex_lock_interruptible(&ggtt->vm.mutex))
+ return -EINTR;
pinned = ggtt->vm.reserved;
list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
@@ -109,8 +121,6 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
LIST_HEAD(still_in_list);
int ret = 0;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
spin_lock(&obj->vma.lock);
while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
struct i915_vma,
@@ -120,8 +130,13 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
ret = -EBUSY;
if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
- !i915_vma_is_active(vma))
- ret = i915_vma_unbind(vma);
+ !i915_vma_is_active(vma)) {
+ ret = mutex_lock_interruptible(&vma->vm->mutex);
+ if (!ret) {
+ ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ }
+ }
spin_lock(&obj->vma.lock);
}
@@ -338,10 +353,6 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
u64 remain, offset;
int ret;
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- return ret;
-
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
vma = ERR_PTR(-ENODEV);
if (!i915_gem_object_is_tiled(obj))
@@ -355,12 +366,10 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
} else {
ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
if (ret)
- goto out_unlock;
+ goto out_rpm;
GEM_BUG_ON(!node.allocated);
}
- mutex_unlock(&i915->drm.struct_mutex);
-
ret = i915_gem_object_lock_interruptible(obj);
if (ret)
goto out_unpin;
@@ -414,17 +423,14 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
i915_gem_object_unlock_fence(obj, fence);
out_unpin:
- mutex_lock(&i915->drm.struct_mutex);
if (node.allocated) {
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
-out_unlock:
+out_rpm:
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
- mutex_unlock(&i915->drm.struct_mutex);
-
return ret;
}
@@ -531,10 +537,6 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
void __user *user_data;
int ret;
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- return ret;
-
if (i915_gem_object_has_struct_page(obj)) {
/*
* Avoid waking the device up if we can fallback, as
@@ -544,10 +546,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
* using the cache bypass of indirect GGTT access.
*/
wakeref = intel_runtime_pm_get_if_in_use(rpm);
- if (!wakeref) {
- ret = -EFAULT;
- goto out_unlock;
- }
+ if (!wakeref)
+ return -EFAULT;
} else {
/* No backing pages, no fallback, we must force GGTT access */
wakeref = intel_runtime_pm_get(rpm);
@@ -569,8 +569,6 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
GEM_BUG_ON(!node.allocated);
}
- mutex_unlock(&i915->drm.struct_mutex);
-
ret = i915_gem_object_lock_interruptible(obj);
if (ret)
goto out_unpin;
@@ -634,18 +632,15 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
i915_gem_object_unlock_fence(obj, fence);
out_unpin:
- mutex_lock(&i915->drm.struct_mutex);
intel_gt_flush_ggtt_writes(ggtt->vm.gt);
if (node.allocated) {
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
out_rpm:
intel_runtime_pm_put(rpm, wakeref);
-out_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
return ret;
}
@@ -967,8 +962,6 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
struct i915_vma *vma;
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
if (flags & PIN_MAPPABLE &&
(!view || view->type == I915_GGTT_VIEW_NORMAL)) {
/* If the required space is larger than the available
@@ -1022,7 +1015,9 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
i915_ggtt_offset(vma), alignment,
!!(flags & PIN_MAPPABLE),
i915_vma_is_map_and_fenceable(vma));
+ mutex_lock(&vma->vm->mutex);
ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
if (ret)
return ERR_PTR(ret);
}
@@ -1320,7 +1315,9 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
* from the GTT to prevent such accidents and reclaim the
* space.
*/
+ mutex_lock(&state->vm->mutex);
err = i915_vma_unbind(state);
+ mutex_unlock(&state->vm->mutex);
if (err)
goto out;
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 52c86c6e0673..722b03af3892 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -47,8 +47,7 @@ static int ggtt_flush(struct drm_i915_private *i915)
* bound by their active reference.
*/
return i915_gem_wait_for_idle(i915,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED,
+ I915_WAIT_INTERRUPTIBLE,
MAX_SCHEDULE_TIMEOUT);
}
@@ -104,7 +103,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
struct i915_vma *active;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict(vm, min_size, alignment, flags);
/*
@@ -127,15 +126,6 @@ i915_gem_evict_something(struct i915_address_space *vm,
min_size, alignment, cache_level,
start, end, mode);
- /*
- * Retire before we search the active list. Although we have
- * reasonable accuracy in our retirement lists, we may have
- * a stray pin (preventing eviction) that can only be resolved by
- * retiring.
- */
- if (!(flags & PIN_NONBLOCK))
- i915_retire_requests(dev_priv);
-
search_again:
active = NULL;
INIT_LIST_HEAD(&eviction_list);
@@ -269,7 +259,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
bool check_color;
int ret = 0;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
@@ -375,7 +365,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
struct i915_vma *vma, *next;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict_vm(vm);
/* Switch back to the default context in order to unpin
@@ -390,7 +380,6 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
}
INIT_LIST_HEAD(&eviction_list);
- mutex_lock(&vm->mutex);
list_for_each_entry(vma, &vm->bound_list, vm_link) {
if (i915_vma_is_pinned(vma))
continue;
@@ -398,7 +387,6 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
__i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
- mutex_unlock(&vm->mutex);
ret = 0;
list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.c b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
index 615a9f4ef30c..1e93f2aafc80 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence_reg.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
@@ -331,13 +331,15 @@ static struct i915_fence_reg *fence_find(struct drm_i915_private *i915)
return ERR_PTR(-EDEADLK);
}
-static int __i915_vma_pin_fence(struct i915_vma *vma)
+int __i915_vma_pin_fence(struct i915_vma *vma)
{
struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm);
struct i915_fence_reg *fence;
struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL;
int err;
+ lockdep_assert_held(&vma->vm->mutex);
+
/* Just update our place in the LRU if our fence is getting reused. */
if (vma->fence) {
fence = vma->fence;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 342a84566e61..8f742f3eec60 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -144,16 +144,18 @@ static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt)
static int ppgtt_bind_vma(struct i915_vma *vma,
enum i915_cache_level cache_level,
- u32 unused)
+ u32 flags)
{
u32 pte_flags;
int err;
- if (!i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) {
+ if (flags & I915_VMA_ALLOC) {
err = vma->vm->allocate_va_range(vma->vm,
vma->node.start, vma->size);
if (err)
return err;
+
+ set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma));
}
/* Applicable to VLV, and gen8+ */
@@ -161,6 +163,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
if (i915_gem_object_is_readonly(vma->obj))
pte_flags |= PTE_READ_ONLY;
+ GEM_BUG_ON(!test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)));
vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
wmb();
@@ -169,7 +172,8 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
static void ppgtt_unbind_vma(struct i915_vma *vma)
{
- vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
+ if (test_and_clear_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)))
+ vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
}
static int ppgtt_set_pages(struct i915_vma *vma)
@@ -497,15 +501,25 @@ static void i915_address_space_fini(struct i915_address_space *vm)
mutex_destroy(&vm->mutex);
}
-static void ppgtt_destroy_vma(struct i915_address_space *vm)
+void __i915_vm_close(struct i915_address_space *vm)
{
struct i915_vma *vma, *vn;
- mutex_lock(&vm->i915->drm.struct_mutex);
- list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link)
+ mutex_lock(&vm->mutex);
+ list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
+ struct drm_i915_gem_object *obj = vma->obj;
+
+ if (!kref_get_unless_zero(&obj->base.refcount))
+ continue;
+
+ atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
+ WARN_ON(i915_vma_unbind(vma));
i915_vma_destroy(vma);
+
+ i915_gem_object_put(obj);
+ }
GEM_BUG_ON(!list_empty(&vm->bound_list));
- mutex_unlock(&vm->i915->drm.struct_mutex);
+ mutex_unlock(&vm->mutex);
}
static void __i915_vm_release(struct work_struct *work)
@@ -513,8 +527,6 @@ static void __i915_vm_release(struct work_struct *work)
struct i915_address_space *vm =
container_of(work, struct i915_address_space, rcu.work);
- ppgtt_destroy_vma(vm);
-
vm->cleanup(vm);
i915_address_space_fini(vm);
@@ -529,7 +541,6 @@ void i915_vm_release(struct kref *kref)
GEM_BUG_ON(i915_is_ggtt(vm));
trace_i915_ppgtt_release(vm);
- vm->closed = true;
queue_rcu_work(vm->i915->wq, &vm->rcu);
}
@@ -537,6 +548,7 @@ static void i915_address_space_init(struct i915_address_space *vm, int subclass)
{
kref_init(&vm->ref);
INIT_RCU_WORK(&vm->rcu, __i915_vm_release);
+ atomic_set(&vm->open, 1);
/*
* The vm->mutex must be reclaim safe (for use in the shrinker).
@@ -1774,12 +1786,8 @@ static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
{
struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
- struct drm_i915_private *i915 = vm->i915;
- /* FIXME remove the struct_mutex to bring the locking under control */
- mutex_lock(&i915->drm.struct_mutex);
i915_vma_destroy(ppgtt->vma);
- mutex_unlock(&i915->drm.struct_mutex);
gen6_ppgtt_free_pd(ppgtt);
free_scratch(vm);
@@ -1867,7 +1875,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
i915_active_init(i915, &vma->active, NULL, NULL);
- vma->vm = &ggtt->vm;
+ vma->vm = i915_vm_get(&ggtt->vm);
vma->ops = &pd_vma_ops;
vma->private = ppgtt;
@@ -1887,7 +1895,7 @@ int gen6_ppgtt_pin(struct i915_ppgtt *base)
struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
int err;
- GEM_BUG_ON(ppgtt->base.vm.closed);
+ GEM_BUG_ON(!atomic_read(&ppgtt->base.vm.open));
/*
* Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
@@ -2463,14 +2471,18 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
if (flags & I915_VMA_LOCAL_BIND) {
struct i915_ppgtt *alias = i915_vm_to_ggtt(vma->vm)->alias;
- if (!i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) {
+ if (flags & I915_VMA_ALLOC) {
ret = alias->vm.allocate_va_range(&alias->vm,
vma->node.start,
vma->size);
if (ret)
return ret;
+
+ set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma));
}
+ GEM_BUG_ON(!test_bit(I915_VMA_ALLOC_BIT,
+ __i915_vma_flags(vma)));
alias->vm.insert_entries(&alias->vm, vma,
cache_level, pte_flags);
}
@@ -2600,22 +2612,16 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
{
- struct drm_i915_private *i915 = ggtt->vm.i915;
struct i915_ppgtt *ppgtt;
- mutex_lock(&i915->drm.struct_mutex);
-
ppgtt = fetch_and_zero(&ggtt->alias);
if (!ppgtt)
- goto out;
+ return;
i915_vm_put(&ppgtt->vm);
ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma;
ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
-
-out:
- mutex_unlock(&i915->drm.struct_mutex);
}
static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
@@ -2732,15 +2738,14 @@ int i915_init_ggtt(struct drm_i915_private *i915)
static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
{
- struct drm_i915_private *i915 = ggtt->vm.i915;
struct i915_vma *vma, *vn;
- ggtt->vm.closed = true;
+ atomic_set(&ggtt->vm.open, 0);
rcu_barrier(); /* flush the RCU'ed__i915_vm_release */
- flush_workqueue(i915->wq);
+ flush_workqueue(ggtt->vm.i915->wq);
- mutex_lock(&i915->drm.struct_mutex);
+ mutex_lock(&ggtt->vm.mutex);
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
WARN_ON(i915_vma_unbind(vma));
@@ -2749,15 +2754,12 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
drm_mm_remove_node(&ggtt->error_capture);
ggtt_release_guc_top(ggtt);
-
- if (drm_mm_initialized(&ggtt->vm.mm)) {
- intel_vgt_deballoon(ggtt);
- i915_address_space_fini(&ggtt->vm);
- }
+ intel_vgt_deballoon(ggtt);
ggtt->vm.cleanup(&ggtt->vm);
- mutex_unlock(&i915->drm.struct_mutex);
+ mutex_unlock(&ggtt->vm.mutex);
+ i915_address_space_fini(&ggtt->vm);
arch_phys_wc_del(ggtt->mtrr);
io_mapping_fini(&ggtt->iomap);
@@ -3186,9 +3188,6 @@ int i915_ggtt_probe_hw(struct drm_i915_private *i915)
static int ggtt_init_hw(struct i915_ggtt *ggtt)
{
struct drm_i915_private *i915 = ggtt->vm.i915;
- int ret = 0;
-
- mutex_lock(&i915->drm.struct_mutex);
i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
@@ -3204,18 +3203,14 @@ static int ggtt_init_hw(struct i915_ggtt *ggtt)
ggtt->gmadr.start,
ggtt->mappable_end)) {
ggtt->vm.cleanup(&ggtt->vm);
- ret = -EIO;
- goto out;
+ return -EIO;
}
ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, ggtt->mappable_end);
i915_ggtt_init_fences(ggtt);
-out:
- mutex_unlock(&i915->drm.struct_mutex);
-
- return ret;
+ return 0;
}
/**
@@ -3287,6 +3282,7 @@ static void ggtt_restore_mappings(struct i915_ggtt *ggtt)
{
struct i915_vma *vma, *vn;
bool flush = false;
+ int open;
intel_gt_check_and_clear_faults(ggtt->vm.gt);
@@ -3294,7 +3290,9 @@ static void ggtt_restore_mappings(struct i915_ggtt *ggtt)
/* First fill our portion of the GTT with scratch pages */
ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
- ggtt->vm.closed = true; /* skip rewriting PTE on VMA unbind */
+
+ /* Skip rewriting PTE on VMA unbind. */
+ open = atomic_xchg(&ggtt->vm.open, 0);
/* clflush objects bound into the GGTT and rebind them. */
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
@@ -3303,24 +3301,19 @@ static void ggtt_restore_mappings(struct i915_ggtt *ggtt)
if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
continue;
- mutex_unlock(&ggtt->vm.mutex);
-
if (!i915_vma_unbind(vma))
- goto lock;
+ continue;
WARN_ON(i915_vma_bind(vma,
obj ? obj->cache_level : 0,
- PIN_UPDATE));
+ PIN_UPDATE, NULL));
if (obj) { /* only used during resume => exclusive access */
flush |= fetch_and_zero(&obj->write_domain);
obj->read_domains |= I915_GEM_DOMAIN_GTT;
}
-
-lock:
- mutex_lock(&ggtt->vm.mutex);
}
- ggtt->vm.closed = false;
+ atomic_set(&ggtt->vm.open, open);
ggtt->invalidate(ggtt);
mutex_unlock(&ggtt->vm.mutex);
@@ -3712,7 +3705,8 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
u64 offset;
int err;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
+
GEM_BUG_ON(!size);
GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(alignment && !is_power_of_2(alignment));
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 0f14722fd163..0516c4d110bc 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -307,7 +307,7 @@ struct i915_address_space {
unsigned int bind_alloc;
- bool closed;
+ atomic_t open;
struct mutex mutex; /* protects vma and our lists */
#define VM_CLASS_GGTT 0
@@ -574,6 +574,35 @@ static inline void i915_vm_put(struct i915_address_space *vm)
kref_put(&vm->ref, i915_vm_release);
}
+static inline struct i915_address_space *
+i915_vm_open(struct i915_address_space *vm)
+{
+ GEM_BUG_ON(!atomic_read(&vm->open));
+ atomic_inc(&vm->open);
+ return i915_vm_get(vm);
+}
+
+static inline bool
+i915_vm_tryopen(struct i915_address_space *vm)
+{
+ if (atomic_add_unless(&vm->open, 1, 0))
+ return i915_vm_get(vm);
+
+ return false;
+}
+
+void __i915_vm_close(struct i915_address_space *vm);
+
+static inline void
+i915_vm_close(struct i915_address_space *vm)
+{
+ GEM_BUG_ON(!atomic_read(&vm->open));
+ if (atomic_dec_and_test(&vm->open))
+ __i915_vm_close(vm);
+
+ i915_vm_put(vm);
+}
+
int gen6_ppgtt_pin(struct i915_ppgtt *base);
void gen6_ppgtt_unpin(struct i915_ppgtt *base);
void gen6_ppgtt_unpin_all(struct i915_ppgtt *base);
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index e42b86827d6b..e6aa56d82f9b 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1204,15 +1204,10 @@ static int i915_oa_read(struct i915_perf_stream *stream,
static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
{
struct i915_gem_engines_iter it;
- struct drm_i915_private *i915 = stream->dev_priv;
struct i915_gem_context *ctx = stream->ctx;
struct intel_context *ce;
int err;
- err = i915_mutex_lock_interruptible(&i915->drm);
- if (err)
- return ERR_PTR(err);
-
for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
if (ce->engine->class != RENDER_CLASS)
continue;
@@ -1229,10 +1224,6 @@ static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
}
i915_gem_context_unlock_engines(ctx);
- mutex_unlock(&i915->drm.struct_mutex);
- if (err)
- return ERR_PTR(err);
-
return stream->pinned_ctx;
}
@@ -1347,15 +1338,9 @@ static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
static void
free_oa_buffer(struct i915_perf_stream *stream)
{
- struct drm_i915_private *i915 = stream->dev_priv;
-
- mutex_lock(&i915->drm.struct_mutex);
-
i915_vma_unpin_and_release(&stream->oa_buffer.vma,
I915_VMA_RELEASE_MAP);
- mutex_unlock(&i915->drm.struct_mutex);
-
stream->oa_buffer.vaddr = NULL;
}
@@ -1510,18 +1495,13 @@ static int alloc_oa_buffer(struct i915_perf_stream *stream)
if (WARN_ON(stream->oa_buffer.vma))
return -ENODEV;
- ret = i915_mutex_lock_interruptible(&dev_priv->drm);
- if (ret)
- return ret;
-
BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE);
BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M);
bo = i915_gem_object_create_shmem(dev_priv, OA_BUFFER_SIZE);
if (IS_ERR(bo)) {
DRM_ERROR("Failed to allocate OA buffer\n");
- ret = PTR_ERR(bo);
- goto unlock;
+ return PTR_ERR(bo);
}
i915_gem_object_set_cache_coherency(bo, I915_CACHE_LLC);
@@ -1545,7 +1525,7 @@ static int alloc_oa_buffer(struct i915_perf_stream *stream)
i915_ggtt_offset(stream->oa_buffer.vma),
stream->oa_buffer.vaddr);
- goto unlock;
+ return 0;
err_unpin:
__i915_vma_unpin(vma);
@@ -1556,8 +1536,6 @@ static int alloc_oa_buffer(struct i915_perf_stream *stream)
stream->oa_buffer.vaddr = NULL;
stream->oa_buffer.vma = NULL;
-unlock:
- mutex_unlock(&dev_priv->drm.struct_mutex);
return ret;
}
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 6c4aa9471af3..f7b20fd945a9 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -32,6 +32,7 @@
#include "i915_drv.h"
#include "i915_globals.h"
+#include "i915_sw_fence_work.h"
#include "i915_trace.h"
#include "i915_vma.h"
@@ -110,7 +111,7 @@ vma_create(struct drm_i915_gem_object *obj,
if (vma == NULL)
return ERR_PTR(-ENOMEM);
- vma->vm = vm;
+ vma->vm = i915_vm_get(vm);
vma->ops = &vm->vma_ops;
vma->obj = obj;
vma->resv = obj->base.resv;
@@ -261,8 +262,6 @@ vma_lookup(struct drm_i915_gem_object *obj,
* Once created, the VMA is kept until either the object is freed, or the
* address space is closed.
*
- * Must be called with struct_mutex held.
- *
* Returns the vma, or an error pointer.
*/
struct i915_vma *
@@ -273,7 +272,7 @@ i915_vma_instance(struct drm_i915_gem_object *obj,
struct i915_vma *vma;
GEM_BUG_ON(view && !i915_is_ggtt(vm));
- GEM_BUG_ON(vm->closed);
+ GEM_BUG_ON(!atomic_read(&vm->open));
spin_lock(&obj->vma.lock);
vma = vma_lookup(obj, vm, view);
@@ -287,18 +286,63 @@ i915_vma_instance(struct drm_i915_gem_object *obj,
return vma;
}
+struct i915_vma_work {
+ struct dma_fence_work base;
+ struct i915_vma *vma;
+ enum i915_cache_level cache_level;
+ unsigned int flags;
+};
+
+static int __vma_bind(struct dma_fence_work *work)
+{
+ struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
+ struct i915_vma *vma = vw->vma;
+ int err;
+
+ err = vma->ops->bind_vma(vma, vw->cache_level, vw->flags);
+ if (err)
+ atomic_or(I915_VMA_ERROR, &vma->flags);
+
+ if (vma->obj)
+ atomic_dec(&vma->obj->mm.pages_pin_count);
+
+ return err;
+}
+
+static const struct dma_fence_work_ops bind_ops = {
+ .name = "bind",
+ .work = __vma_bind,
+};
+
+struct i915_vma_work *i915_vma_work(void)
+{
+ struct i915_vma_work *vw;
+
+ vw = kzalloc(sizeof(*vw), GFP_KERNEL);
+ if (!vw)
+ return NULL;
+
+ dma_fence_work_init(&vw->base, &bind_ops);
+ vw->base.dma.error = -EAGAIN; /* disable the worker by default */
+
+ return vw;
+}
+
/**
* i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
* @vma: VMA to map
* @cache_level: mapping cache level
* @flags: flags like global or local mapping
+ * @work: preallocated worker for allocating and binding the PTE
*
* DMA addresses are taken from the scatter-gather table of this object (or of
* this VMA in case of non-default GGTT views) and PTE entries set up.
* Note that DMA addresses are also the only part of the SG table we care about.
*/
-int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
- u32 flags)
+int i915_vma_bind(struct i915_vma *vma,
+ enum i915_cache_level cache_level,
+ u32 flags,
+ struct i915_vma_work *work)
{
u32 bind_flags;
u32 vma_flags;
@@ -315,11 +359,8 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
if (GEM_DEBUG_WARN_ON(!flags))
return -EINVAL;
- bind_flags = 0;
- if (flags & PIN_GLOBAL)
- bind_flags |= I915_VMA_GLOBAL_BIND;
- if (flags & PIN_USER)
- bind_flags |= I915_VMA_LOCAL_BIND;
+ bind_flags = flags;
+ bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
vma_flags = atomic_read(&vma->flags);
vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
@@ -333,9 +374,29 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
GEM_BUG_ON(!vma->pages);
trace_i915_vma_bind(vma, bind_flags);
- ret = vma->ops->bind_vma(vma, cache_level, bind_flags);
- if (ret)
- return ret;
+ if (work && (bind_flags & ~vma_flags) & vma->vm->bind_alloc) {
+ work->vma = vma;
+ work->cache_level = cache_level;
+ work->flags = bind_flags | I915_VMA_ALLOC;
+
+ if (!i915_vma_trylock(vma))
+ return -EAGAIN;
+
+ dma_fence_work_chain(&work->base, dma_resv_get_excl(vma->resv));
+ dma_resv_add_excl_fence(vma->resv, &work->base.dma);
+ i915_vma_unlock(vma);
+
+ i915_active_set_exclusive(&vma->active, &work->base.dma);
+ work->base.dma.error = 0; /* enable the queue_work() */
+
+ if (vma->obj)
+ atomic_inc(&vma->obj->mm.pages_pin_count);
+ } else {
+ GEM_BUG_ON((bind_flags & ~vma_flags) & vma->vm->bind_alloc);
+ ret = vma->ops->bind_vma(vma, cache_level, bind_flags);
+ if (ret)
+ return ret;
+ }
atomic_or(bind_flags, &vma->flags);
return 0;
@@ -348,9 +409,7 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
/* Access through the GTT requires the device to be awake. */
assert_rpm_wakelock_held(&vma->vm->i915->runtime_pm);
-
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
- if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
+ if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
err = -ENODEV;
goto err;
}
@@ -368,7 +427,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
goto err;
}
- vma->iomap = ptr;
+ if (unlikely(cmpxchg(&vma->iomap, NULL, ptr)))
+ io_mapping_unmap(ptr);
}
__i915_vma_pin(vma);
@@ -388,18 +448,12 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
void i915_vma_flush_writes(struct i915_vma *vma)
{
- if (!i915_vma_has_ggtt_write(vma))
- return;
-
- intel_gt_flush_ggtt_writes(vma->vm->gt);
-
- i915_vma_unset_ggtt_write(vma);
+ if (i915_vma_unset_ggtt_write(vma))
+ intel_gt_flush_ggtt_writes(vma->vm->gt);
}
void i915_vma_unpin_iomap(struct i915_vma *vma)
{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
-
GEM_BUG_ON(vma->iomap == NULL);
i915_vma_flush_writes(vma);
@@ -538,7 +592,6 @@ static void assert_bind_count(const struct drm_i915_gem_object *obj)
static int
i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
- struct drm_i915_private *dev_priv = vma->vm->i915;
unsigned int cache_level;
u64 start, end;
int ret;
@@ -564,7 +617,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
end = vma->vm->total;
if (flags & PIN_MAPPABLE)
- end = min_t(u64, end, dev_priv->ggtt.mappable_end);
+ end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end);
if (flags & PIN_ZONE_4G)
end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
@@ -580,35 +633,21 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
return -ENOSPC;
}
- if (vma->obj) {
- ret = i915_gem_object_pin_pages(vma->obj);
- if (ret)
- return ret;
-
+ cache_level = 0;
+ if (vma->obj)
cache_level = vma->obj->cache_level;
- } else {
- cache_level = 0;
- }
-
- GEM_BUG_ON(vma->pages);
-
- ret = vma->ops->set_pages(vma);
- if (ret)
- goto err_unpin;
if (flags & PIN_OFFSET_FIXED) {
u64 offset = flags & PIN_OFFSET_MASK;
if (!IS_ALIGNED(offset, alignment) ||
- range_overflows(offset, size, end)) {
- ret = -EINVAL;
- goto err_clear;
- }
+ range_overflows(offset, size, end))
+ return -EINVAL;
ret = i915_gem_gtt_reserve(vma->vm, &vma->node,
size, offset, cache_level,
flags);
if (ret)
- goto err_clear;
+ return ret;
} else {
/*
* We only support huge gtt pages through the 48b PPGTT,
@@ -647,7 +686,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
size, alignment, cache_level,
start, end, flags);
if (ret)
- goto err_clear;
+ return ret;
GEM_BUG_ON(vma->node.start < start);
GEM_BUG_ON(vma->node.start + vma->node.size > end);
@@ -655,23 +694,15 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level));
- mutex_lock(&vma->vm->mutex);
list_add_tail(&vma->vm_link, &vma->vm->bound_list);
- mutex_unlock(&vma->vm->mutex);
if (vma->obj) {
+ atomic_inc(&vma->obj->mm.pages_pin_count);
atomic_inc(&vma->obj->bind_count);
assert_bind_count(vma->obj);
}
return 0;
-
-err_clear:
- vma->ops->clear_pages(vma);
-err_unpin:
- if (vma->obj)
- i915_gem_object_unpin_pages(vma->obj);
- return ret;
}
static void
@@ -680,12 +711,8 @@ i915_vma_remove(struct i915_vma *vma)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
- vma->ops->clear_pages(vma);
-
- mutex_lock(&vma->vm->mutex);
drm_mm_remove_node(&vma->node);
list_del(&vma->vm_link);
- mutex_unlock(&vma->vm->mutex);
/*
* Since the unbound list is global, only move to that list if
@@ -706,49 +733,111 @@ i915_vma_remove(struct i915_vma *vma)
}
}
-int __i915_vma_do_pin(struct i915_vma *vma,
- u64 size, u64 alignment, u64 flags)
+int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
- const unsigned int bound = atomic_read(&vma->flags);
- int ret;
+ struct i915_vma_work *work = NULL;
+ bool insert = false;
+ unsigned int bound;
+ int err;
+
+ BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
+ BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
+ BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
+
+ GEM_BUG_ON(flags & I915_VMA_PIN_OVERFLOW);
+ GEM_BUG_ON(flags & I915_VMA_ERROR);
+
+ bound = atomic_inc_return(&vma->flags);
+ if (likely(((bound ^ flags) & I915_VMA_BIND_MASK) == 0)) {
+ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+ GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
+ return 0;
+ }
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
- GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
- GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
+ if (bound & I915_VMA_ERROR) {
+ err = -ENOMEM;
+ goto err_unpin;
+ }
if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
- ret = -EBUSY;
+ err = -ENOMEM;
goto err_unpin;
}
- if ((bound & I915_VMA_BIND_MASK) == 0) {
- ret = i915_vma_insert(vma, size, alignment, flags);
- if (ret)
- goto err_unpin;
+ if (vma->obj) {
+ err = i915_gem_object_pin_pages(vma->obj);
+ if (err)
+ return err;
}
- GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
- ret = i915_vma_bind(vma, vma->obj ? vma->obj->cache_level : 0, flags);
- if (ret)
- goto err_remove;
+ if (!vma->pages) {
+ GEM_BUG_ON(vma->pages);
+ err = vma->ops->set_pages(vma);
+ if (err)
+ goto err_pages;
+
+ GEM_BUG_ON(!vma->pages);
+ insert = true;
+ }
+
+ if (flags & PIN_USER) {
+ work = i915_vma_work();
+ if (!work) {
+ err = -ENOMEM;
+ goto err_clear;
+ }
+ }
- GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_BIND_MASK));
+ err = i915_active_acquire(&vma->active);
+ if (err)
+ goto err_fence;
- if ((bound ^ atomic_read(&vma->flags)) & I915_VMA_GLOBAL_BIND)
- __i915_vma_set_map_and_fenceable(vma);
+ err = mutex_lock_interruptible(&vma->vm->mutex);
+ if (err)
+ goto err_active;
- GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
- return 0;
+ if (insert) {
+ err = i915_vma_insert(vma, size, alignment, flags);
+ if (err)
+ goto err_unlock;
+ }
+
+ err = i915_vma_bind(vma,
+ vma->obj ? vma->obj->cache_level : 0,
+ flags, work);
+ if (err == 0) {
+ GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_BIND_MASK));
-err_remove:
- if ((bound & I915_VMA_BIND_MASK) == 0) {
- i915_vma_remove(vma);
+ if ((bound ^ atomic_read(&vma->flags)) & I915_VMA_GLOBAL_BIND)
+ __i915_vma_set_map_and_fenceable(vma);
+
+ GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
+
+ __i915_vma_pin(vma);
+ } else {
+ if (insert)
+ i915_vma_remove(vma);
+ }
+
+err_unlock:
+ mutex_unlock(&vma->vm->mutex);
+err_active:
+ i915_active_release(&vma->active);
+err_fence:
+ if (work)
+ dma_fence_work_commit(&work->base);
+err_clear:
+ if (err && insert) {
+ vma->ops->clear_pages(vma);
GEM_BUG_ON(vma->pages);
GEM_BUG_ON(atomic_read(&vma->flags) & I915_VMA_BIND_MASK);
}
+err_pages:
+ if (vma->obj)
+ i915_gem_object_unpin_pages(vma->obj);
err_unpin:
__i915_vma_unpin(vma);
- return ret;
+ return err;
}
void i915_vma_close(struct i915_vma *vma)
@@ -792,10 +881,18 @@ void i915_vma_reopen(struct i915_vma *vma)
__i915_vma_remove_closed(vma);
}
-static void __i915_vma_destroy(struct i915_vma *vma)
+void i915_vma_destroy(struct i915_vma *vma)
{
- GEM_BUG_ON(vma->node.allocated);
- GEM_BUG_ON(vma->fence);
+ __i915_vma_remove_closed(vma);
+
+ if (drm_mm_node_allocated(&vma->node)) {
+ mutex_lock(&vma->vm->mutex);
+ atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
+ WARN_ON(i915_vma_unbind(vma));
+ mutex_unlock(&vma->vm->mutex);
+ GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
+ }
+ GEM_BUG_ON(i915_vma_is_active(vma));
if (vma->obj) {
struct drm_i915_gem_object *obj = vma->obj;
@@ -806,36 +903,36 @@ static void __i915_vma_destroy(struct i915_vma *vma)
spin_unlock(&obj->vma.lock);
}
- i915_active_fini(&vma->active);
+ i915_vm_put(vma->vm);
+ i915_active_fini(&vma->active);
i915_vma_free(vma);
}
-void i915_vma_destroy(struct i915_vma *vma)
-{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
-
- GEM_BUG_ON(i915_vma_is_pinned(vma));
-
- __i915_vma_remove_closed(vma);
-
- WARN_ON(i915_vma_unbind(vma));
- GEM_BUG_ON(i915_vma_is_active(vma));
-
- __i915_vma_destroy(vma);
-}
-
void i915_vma_parked(struct drm_i915_private *i915)
{
struct i915_vma *vma, *next;
spin_lock_irq(&i915->gt.closed_lock);
list_for_each_entry_safe(vma, next, &i915->gt.closed_vma, closed_link) {
+ struct drm_i915_gem_object *obj = vma->obj;
+ struct i915_address_space *vm = vma->vm;
+
list_del_init(&vma->closed_link);
+ if (!i915_vm_tryopen(vm))
+ continue;
+
+ if (!kref_get_unless_zero(&obj->base.refcount))
+ obj = NULL;
+
spin_unlock_irq(&i915->gt.closed_lock);
- i915_vma_destroy(vma);
+ if (obj) {
+ i915_vma_destroy(vma);
+ i915_gem_object_put(obj);
+ }
+ i915_vm_close(vm);
spin_lock_irq(&i915->gt.closed_lock);
}
spin_unlock_irq(&i915->gt.closed_lock);
@@ -927,34 +1024,15 @@ int i915_vma_unbind(struct i915_vma *vma)
{
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
/*
* First wait upon any activity as retiring the request may
* have side-effects such as unpinning or even unbinding this vma.
*/
- might_sleep();
- if (i915_vma_is_active(vma)) {
- /*
- * When a closed VMA is retired, it is unbound - eek.
- * In order to prevent it from being recursively closed,
- * take a pin on the vma so that the second unbind is
- * aborted.
- *
- * Even more scary is that the retire callback may free
- * the object (last active vma). To prevent the explosion
- * we defer the actual object free to a worker that can
- * only proceed once it acquires the struct_mutex (which
- * we currently hold, therefore it cannot free this object
- * before we are finished).
- */
- __i915_vma_pin(vma);
- ret = i915_active_wait(&vma->active);
- __i915_vma_unpin(vma);
- if (ret)
- return ret;
- }
- GEM_BUG_ON(i915_vma_is_active(vma));
+ ret = i915_active_wait(&vma->active);
+ if (ret)
+ return ret;
if (i915_vma_is_pinned(vma)) {
vma_print_allocator(vma, "is pinned");
@@ -975,16 +1053,12 @@ int i915_vma_unbind(struct i915_vma *vma)
GEM_BUG_ON(i915_vma_has_ggtt_write(vma));
/* release the fence reg _after_ flushing */
- mutex_lock(&vma->vm->mutex);
ret = i915_vma_revoke_fence(vma);
- mutex_unlock(&vma->vm->mutex);
if (ret)
return ret;
/* Force a pagefault for domain tracking on next user access */
- mutex_lock(&vma->vm->mutex);
i915_vma_revoke_mmap(vma);
- mutex_unlock(&vma->vm->mutex);
__i915_vma_iounmap(vma);
clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
@@ -992,13 +1066,14 @@ int i915_vma_unbind(struct i915_vma *vma)
GEM_BUG_ON(vma->fence);
GEM_BUG_ON(i915_vma_has_userfault(vma));
- if (likely(!vma->vm->closed)) {
+ if (likely(atomic_read(&vma->vm->open))) {
trace_i915_vma_unbind(vma);
vma->ops->unbind_vma(vma);
}
atomic_and(~I915_VMA_BIND_MASK, &vma->flags);
i915_vma_remove(vma);
+ vma->ops->clear_pages(vma);
return 0;
}
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 38c6f3427e8c..c8ccf8536a82 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -73,6 +73,7 @@ struct i915_vma {
*/
atomic_t open_count;
atomic_t flags;
+
/**
* How many users have pinned this object in GTT space.
*
@@ -103,18 +104,24 @@ struct i915_vma {
/** Flags and address space this VMA is bound to */
#define I915_VMA_GLOBAL_BIND_BIT 9
#define I915_VMA_LOCAL_BIND_BIT 10
+#define I915_VMA_ERROR_BIT 11
#define I915_VMA_GLOBAL_BIND ((int)BIT(I915_VMA_GLOBAL_BIND_BIT))
#define I915_VMA_LOCAL_BIND ((int)BIT(I915_VMA_LOCAL_BIND_BIT))
+#define I915_VMA_ERROR ((int)BIT(I915_VMA_ERROR_BIT))
#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | \
I915_VMA_LOCAL_BIND | \
+ I915_VMA_ERROR | \
I915_VMA_PIN_OVERFLOW)
-#define I915_VMA_GGTT_BIT 11
-#define I915_VMA_CAN_FENCE_BIT 12
-#define I915_VMA_USERFAULT_BIT 13
-#define I915_VMA_GGTT_WRITE_BIT 14
+#define I915_VMA_ALLOC_BIT 12
+#define I915_VMA_ALLOC ((int)BIT(I915_VMA_ALLOC_BIT))
+
+#define I915_VMA_GGTT_BIT 13
+#define I915_VMA_CAN_FENCE_BIT 14
+#define I915_VMA_USERFAULT_BIT 15
+#define I915_VMA_GGTT_WRITE_BIT 16
#define I915_VMA_GGTT ((int)BIT(I915_VMA_GGTT_BIT))
#define I915_VMA_CAN_FENCE ((int)BIT(I915_VMA_CAN_FENCE_BIT))
@@ -308,8 +315,12 @@ i915_vma_compare(struct i915_vma *vma,
return memcmp(&vma->ggtt_view.partial, &view->partial, view->type);
}
-int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
- u32 flags);
+struct i915_vma_work *i915_vma_work(void);
+int i915_vma_bind(struct i915_vma *vma,
+ enum i915_cache_level cache_level,
+ u32 flags,
+ struct i915_vma_work *work);
+
bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level);
bool i915_vma_misplaced(const struct i915_vma *vma,
u64 size, u64 alignment, u64 flags);
@@ -328,32 +339,19 @@ static inline void i915_vma_lock(struct i915_vma *vma)
dma_resv_lock(vma->resv, NULL);
}
-static inline void i915_vma_unlock(struct i915_vma *vma)
+static inline bool i915_vma_trylock(struct i915_vma *vma)
{
- dma_resv_unlock(vma->resv);
+ return dma_resv_trylock(vma->resv);
}
-int __i915_vma_do_pin(struct i915_vma *vma,
- u64 size, u64 alignment, u64 flags);
-static inline int __must_check
-i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+static inline void i915_vma_unlock(struct i915_vma *vma)
{
- BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
- BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
- BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
-
- /* Pin early to prevent the shrinker/eviction logic from destroying
- * our vma as we insert and bind.
- */
- if (likely(((atomic_inc_return(&vma->flags) ^ flags) & I915_VMA_BIND_MASK) == 0)) {
- GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
- GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
- return 0;
- }
-
- return __i915_vma_do_pin(vma, size, alignment, flags);
+ dma_resv_unlock(vma->resv);
}
+int __must_check
+i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
+
static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
return atomic_read(&vma->flags) & I915_VMA_PIN_MASK;
@@ -397,8 +395,6 @@ static inline bool i915_vma_is_bound(const struct i915_vma *vma,
* the caller must call i915_vma_unpin_iomap to relinquish the pinning
* after the iomapping is no longer required.
*
- * Callers must hold the struct_mutex.
- *
* Returns a valid iomapped pointer or ERR_PTR.
*/
void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
@@ -410,8 +406,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
*
* Unpins the previously iomapped VMA from i915_vma_pin_iomap().
*
- * Callers must hold the struct_mutex. This function is only valid to be
- * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
+ * This function is only valid to be called on a VMA previously
+ * iomapped by the caller with i915_vma_pin_iomap().
*/
void i915_vma_unpin_iomap(struct i915_vma *vma);
@@ -439,6 +435,8 @@ static inline struct page *i915_vma_first_page(struct i915_vma *vma)
int __must_check i915_vma_pin_fence(struct i915_vma *vma);
int __must_check i915_vma_revoke_fence(struct i915_vma *vma);
+int __i915_vma_pin_fence(struct i915_vma *vma);
+
static inline void __i915_vma_unpin_fence(struct i915_vma *vma)
{
GEM_BUG_ON(atomic_read(&vma->fence->pin_count) <= 0);
@@ -456,7 +454,6 @@ static inline void __i915_vma_unpin_fence(struct i915_vma *vma)
static inline void
i915_vma_unpin_fence(struct i915_vma *vma)
{
- /* lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); */
if (vma->fence)
__i915_vma_unpin_fence(vma);
}
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index cb30c669b1b7..ba6064147173 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -106,14 +106,11 @@ static int populate_ggtt(struct drm_i915_private *i915,
static void unpin_ggtt(struct drm_i915_private *i915)
{
- struct i915_ggtt *ggtt = &i915->ggtt;
struct i915_vma *vma;
- mutex_lock(&ggtt->vm.mutex);
list_for_each_entry(vma, &i915->ggtt.vm.bound_list, vm_link)
if (vma->obj->mm.quirked)
i915_vma_unpin(vma);
- mutex_unlock(&ggtt->vm.mutex);
}
static void cleanup_objects(struct drm_i915_private *i915,
@@ -127,11 +124,7 @@ static void cleanup_objects(struct drm_i915_private *i915,
i915_gem_object_put(obj);
}
- mutex_unlock(&i915->drm.struct_mutex);
-
i915_gem_drain_freed_objects(i915);
-
- mutex_lock(&i915->drm.struct_mutex);
}
static int igt_evict_something(void *arg)
@@ -148,10 +141,12 @@ static int igt_evict_something(void *arg)
goto cleanup;
/* Everything is pinned, nothing should happen */
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_something(&ggtt->vm,
I915_GTT_PAGE_SIZE, 0, 0,
0, U64_MAX,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err != -ENOSPC) {
pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
err);
@@ -161,10 +156,12 @@ static int igt_evict_something(void *arg)
unpin_ggtt(i915);
/* Everything is unpinned, we should be able to evict something */
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_something(&ggtt->vm,
I915_GTT_PAGE_SIZE, 0, 0,
0, U64_MAX,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
err);
@@ -230,7 +227,9 @@ static int igt_evict_for_vma(void *arg)
goto cleanup;
/* Everything is pinned, nothing should happen */
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err != -ENOSPC) {
pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n",
err);
@@ -240,7 +239,9 @@ static int igt_evict_for_vma(void *arg)
unpin_ggtt(i915);
/* Everything is unpinned, we should be able to evict the node */
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_evict_for_node returned err=%d\n",
err);
@@ -317,7 +318,9 @@ static int igt_evict_for_cache_color(void *arg)
i915_vma_unpin(vma);
/* Remove just the second vma */
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err);
goto cleanup;
@@ -328,7 +331,9 @@ static int igt_evict_for_cache_color(void *arg)
*/
target.color = I915_CACHE_L3_LLC;
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
+ mutex_unlock(&ggtt->vm.mutex);
if (!err) {
pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err);
err = -EINVAL;
@@ -358,7 +363,9 @@ static int igt_evict_vm(void *arg)
goto cleanup;
/* Everything is pinned, nothing should happen */
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_vm(&ggtt->vm);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
err);
@@ -367,7 +374,9 @@ static int igt_evict_vm(void *arg)
unpin_ggtt(i915);
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_evict_vm(&ggtt->vm);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
err);
@@ -408,11 +417,11 @@ static int igt_evict_contexts(void *arg)
if (!HAS_FULL_PPGTT(i915))
return 0;
- mutex_lock(&i915->drm.struct_mutex);
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
/* Reserve a block so that we know we have enough to fit a few rq */
memset(&hole, 0, sizeof(hole));
+ mutex_lock(&i915->ggtt.vm.mutex);
err = i915_gem_gtt_insert(&i915->ggtt.vm, &hole,
PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE,
0, i915->ggtt.vm.total,
@@ -425,7 +434,9 @@ static int igt_evict_contexts(void *arg)
do {
struct reserved *r;
+ mutex_unlock(&i915->ggtt.vm.mutex);
r = kcalloc(1, sizeof(*r), GFP_KERNEL);
+ mutex_lock(&i915->ggtt.vm.mutex);
if (!r) {
err = -ENOMEM;
goto out_locked;
@@ -445,7 +456,7 @@ static int igt_evict_contexts(void *arg)
count++;
} while (1);
drm_mm_remove_node(&hole);
- mutex_unlock(&i915->drm.struct_mutex);
+ mutex_unlock(&i915->ggtt.vm.mutex);
pr_info("Filled GGTT with %lu 1MiB nodes\n", count);
/* Overfill the GGTT with context objects and so try to evict one. */
@@ -508,7 +519,7 @@ static int igt_evict_contexts(void *arg)
break;
}
- mutex_lock(&i915->drm.struct_mutex);
+ mutex_lock(&i915->ggtt.vm.mutex);
out_locked:
if (igt_flush_test(i915, I915_WAIT_LOCKED))
err = -EIO;
@@ -522,8 +533,8 @@ static int igt_evict_contexts(void *arg)
}
if (drm_mm_node_allocated(&hole))
drm_mm_remove_node(&hole);
+ mutex_unlock(&i915->ggtt.vm.mutex);
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
- mutex_unlock(&i915->drm.struct_mutex);
return err;
}
@@ -545,12 +556,9 @@ int i915_gem_evict_mock_selftests(void)
if (!i915)
return -ENOMEM;
- mutex_lock(&i915->drm.struct_mutex);
with_intel_runtime_pm(&i915->runtime_pm, wakeref)
err = i915_subtests(tests, i915);
- mutex_unlock(&i915->drm.struct_mutex);
-
drm_dev_put(&i915->drm);
return err;
}
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index a90c9be95f8c..3373f20fb144 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -35,16 +35,7 @@
static void cleanup_freed_objects(struct drm_i915_private *i915)
{
- /*
- * As we may hold onto the struct_mutex for inordinate lengths of
- * time, the NMI khungtaskd detector may fire for the free objects
- * worker.
- */
- mutex_unlock(&i915->drm.struct_mutex);
-
i915_gem_drain_freed_objects(i915);
-
- mutex_lock(&i915->drm.struct_mutex);
}
static void fake_free_pages(struct drm_i915_gem_object *obj,
@@ -318,6 +309,17 @@ static int lowlevel_hole(struct drm_i915_private *i915,
return 0;
}
+static int unlocked_vma_unbind(struct i915_vma *vma)
+{
+ int ret;
+
+ mutex_lock(&vma->vm->mutex);
+ ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+
+ return ret;
+}
+
static void close_object_list(struct list_head *objects,
struct i915_address_space *vm)
{
@@ -329,7 +331,7 @@ static void close_object_list(struct list_head *objects,
vma = i915_vma_instance(obj, vm, NULL);
if (!IS_ERR(vma))
- ignored = i915_vma_unbind(vma);
+ ignored = unlocked_vma_unbind(vma);
/* Only ppgtt vma may be closed before the object is freed */
if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma))
i915_vma_close(vma);
@@ -444,7 +446,7 @@ static int fill_hole(struct drm_i915_private *i915,
goto err;
}
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
if (err) {
pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
__func__, p->name, vma->node.start, vma->node.size,
@@ -517,7 +519,7 @@ static int fill_hole(struct drm_i915_private *i915,
goto err;
}
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
if (err) {
pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
__func__, p->name, vma->node.start, vma->node.size,
@@ -604,7 +606,7 @@ static int walk_hole(struct drm_i915_private *i915,
goto err_close;
}
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
if (err) {
pr_err("%s unbind failed at %llx + %llx with err=%d\n",
__func__, addr, vma->size, err);
@@ -685,13 +687,13 @@ static int pot_hole(struct drm_i915_private *i915,
pr_err("%s incorrect at %llx + %llx\n",
__func__, addr, vma->size);
i915_vma_unpin(vma);
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
err = -EINVAL;
goto err;
}
i915_vma_unpin(vma);
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
GEM_BUG_ON(err);
}
@@ -789,13 +791,13 @@ static int drunk_hole(struct drm_i915_private *i915,
pr_err("%s incorrect at %llx + %llx\n",
__func__, addr, BIT_ULL(size));
i915_vma_unpin(vma);
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
err = -EINVAL;
goto err;
}
i915_vma_unpin(vma);
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
GEM_BUG_ON(err);
if (igt_timeout(end_time,
@@ -867,7 +869,7 @@ static int __shrink_hole(struct drm_i915_private *i915,
pr_err("%s incorrect at %llx + %llx\n",
__func__, addr, size);
i915_vma_unpin(vma);
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
err = -EINVAL;
break;
}
@@ -875,6 +877,15 @@ static int __shrink_hole(struct drm_i915_private *i915,
i915_vma_unpin(vma);
addr += size;
+ /*
+ * Since we are injecting allocation faults at random intervals,
+ * wait for this allocation to complete before we change the
+ * faultinjection.
+ */
+ err = i915_active_wait(&vma->active);
+ if (err)
+ break;
+
if (igt_timeout(end_time,
"%s timed out at ofset %llx [%llx - %llx]\n",
__func__, addr, hole_start, hole_end)) {
@@ -1008,21 +1019,19 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv,
if (IS_ERR(file))
return PTR_ERR(file);
- mutex_lock(&dev_priv->drm.struct_mutex);
ppgtt = i915_ppgtt_create(dev_priv);
if (IS_ERR(ppgtt)) {
err = PTR_ERR(ppgtt);
- goto out_unlock;
+ goto out_free;
}
GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
- GEM_BUG_ON(ppgtt->vm.closed);
+ GEM_BUG_ON(!atomic_read(&ppgtt->vm.open));
err = func(dev_priv, &ppgtt->vm, 0, ppgtt->vm.total, end_time);
i915_vm_put(&ppgtt->vm);
-out_unlock:
- mutex_unlock(&dev_priv->drm.struct_mutex);
+out_free:
mock_file_free(dev_priv, file);
return err;
}
@@ -1085,7 +1094,6 @@ static int exercise_ggtt(struct drm_i915_private *i915,
IGT_TIMEOUT(end_time);
int err = 0;
- mutex_lock(&i915->drm.struct_mutex);
restart:
list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
@@ -1106,7 +1114,6 @@ static int exercise_ggtt(struct drm_i915_private *i915,
last = hole_end;
goto restart;
}
- mutex_unlock(&i915->drm.struct_mutex);
return err;
}
@@ -1148,13 +1155,9 @@ static int igt_ggtt_page(void *arg)
unsigned int *order, n;
int err;
- mutex_lock(&i915->drm.struct_mutex);
-
obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
- if (IS_ERR(obj)) {
- err = PTR_ERR(obj);
- goto out_unlock;
- }
+ if (IS_ERR(obj))
+ return PTR_ERR(obj);
err = i915_gem_object_pin_pages(obj);
if (err)
@@ -1222,8 +1225,6 @@ static int igt_ggtt_page(void *arg)
i915_gem_object_unpin_pages(obj);
out_free:
i915_gem_object_put(obj);
-out_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
return err;
}
@@ -1330,11 +1331,13 @@ static int igt_gtt_reserve(void *arg)
goto out;
}
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
obj->base.size,
total,
obj->cache_level,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
total, ggtt->vm.total, err);
@@ -1380,11 +1383,13 @@ static int igt_gtt_reserve(void *arg)
goto out;
}
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
obj->base.size,
total,
obj->cache_level,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
total, ggtt->vm.total, err);
@@ -1414,7 +1419,7 @@ static int igt_gtt_reserve(void *arg)
goto out;
}
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
if (err) {
pr_err("i915_vma_unbind failed with err=%d!\n", err);
goto out;
@@ -1424,11 +1429,13 @@ static int igt_gtt_reserve(void *arg)
2*I915_GTT_PAGE_SIZE,
I915_GTT_MIN_ALIGNMENT);
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
obj->base.size,
offset,
obj->cache_level,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
total, ggtt->vm.total, err);
@@ -1497,11 +1504,13 @@ static int igt_gtt_insert(void *arg)
/* Check a couple of obviously invalid requests */
for (ii = invalid_insert; ii->size; ii++) {
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_gtt_insert(&ggtt->vm, &tmp,
ii->size, ii->alignment,
I915_COLOR_UNEVICTABLE,
ii->start, ii->end,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err != -ENOSPC) {
pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
ii->size, ii->alignment, ii->start, ii->end,
@@ -1537,10 +1546,12 @@ static int igt_gtt_insert(void *arg)
goto out;
}
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
obj->base.size, 0, obj->cache_level,
0, ggtt->vm.total,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err == -ENOSPC) {
/* maxed out the GGTT space */
i915_gem_object_put(obj);
@@ -1589,16 +1600,18 @@ static int igt_gtt_insert(void *arg)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
offset = vma->node.start;
- err = i915_vma_unbind(vma);
+ err = unlocked_vma_unbind(vma);
if (err) {
pr_err("i915_vma_unbind failed with err=%d!\n", err);
goto out;
}
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
obj->base.size, 0, obj->cache_level,
0, ggtt->vm.total,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
total, ggtt->vm.total, err);
@@ -1642,10 +1655,12 @@ static int igt_gtt_insert(void *arg)
goto out;
}
+ mutex_lock(&ggtt->vm.mutex);
err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
obj->base.size, 0, obj->cache_level,
0, ggtt->vm.total,
0);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
total, ggtt->vm.total, err);
@@ -1689,8 +1704,9 @@ int i915_gem_gtt_mock_selftests(void)
}
mock_init_ggtt(i915, ggtt);
- mutex_lock(&i915->drm.struct_mutex);
err = i915_subtests(tests, ggtt);
+
+ mutex_lock(&i915->drm.struct_mutex);
mock_device_flush(i915);
mutex_unlock(&i915->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index a5bec0a4cdcc..53aafb9a3d64 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -337,7 +337,9 @@ static int igt_vma_pin1(void *arg)
if (!err) {
i915_vma_unpin(vma);
+ mutex_lock(&ggtt->vm.mutex);
err = i915_vma_unbind(vma);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("Failed to unbind single page from GGTT, err=%d\n", err);
goto out;
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 11/15] drm/i915: Push the i915_active.retire into a worker
2019-08-23 13:26 vm->mutex Chris Wilson
` (9 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 10/15] drm/i915: Pull i915_vma_pin under the vm->mutex Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 12/15] drm/i915: Coordinate i915_active with its own mutex Chris Wilson
` (6 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx; +Cc: Matthew Auld
As we need to use a mutex to serialise i915_active activation
(because we want to allow the callback to sleep), we need to push the
i915_active.retire into a worker callback in case we get need to retire
from an atomic context.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Matthew Auld <matthew.auld@intel.com>
---
.../gpu/drm/i915/display/intel_frontbuffer.c | 4 ++-
drivers/gpu/drm/i915/gem/i915_gem_context.c | 1 +
drivers/gpu/drm/i915/gt/intel_context.c | 2 ++
drivers/gpu/drm/i915/gt/intel_engine_pool.c | 1 +
drivers/gpu/drm/i915/gt/intel_timeline.c | 1 +
drivers/gpu/drm/i915/i915_active.c | 34 ++++++++++++++++---
drivers/gpu/drm/i915/i915_active_types.h | 13 ++++++-
drivers/gpu/drm/i915/i915_vma.c | 2 ++
drivers/gpu/drm/i915/selftests/i915_active.c | 6 ++--
9 files changed, 55 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
index 719379774fa5..d62810c1256a 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
@@ -206,6 +206,7 @@ static int frontbuffer_active(struct i915_active *ref)
return 0;
}
+__i915_active_call
static void frontbuffer_retire(struct i915_active *ref)
{
struct intel_frontbuffer *front =
@@ -250,7 +251,8 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj)
kref_init(&front->ref);
atomic_set(&front->bits, 0);
i915_active_init(i915, &front->write,
- frontbuffer_active, frontbuffer_retire);
+ frontbuffer_active,
+ i915_active_may_sleep(frontbuffer_retire));
spin_lock(&i915->fb_tracking.lock);
if (obj->frontbuffer) {
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 23c4a888041f..4dc30c640380 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -848,6 +848,7 @@ struct context_barrier_task {
void *data;
};
+__i915_active_call
static void cb_retire(struct i915_active *base)
{
struct context_barrier_task *cb = container_of(base, typeof(*cb), base);
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index c0495811f493..ae7c2689ef30 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -138,6 +138,7 @@ static void __context_unpin_state(struct i915_vma *vma)
__i915_vma_unpin(vma);
}
+__i915_active_call
static void __intel_context_retire(struct i915_active *active)
{
struct intel_context *ce = container_of(active, typeof(*ce), active);
@@ -150,6 +151,7 @@ static void __intel_context_retire(struct i915_active *active)
intel_timeline_unpin(ce->timeline);
intel_ring_unpin(ce->ring);
+
intel_context_put(ce);
}
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pool.c b/drivers/gpu/drm/i915/gt/intel_engine_pool.c
index 4cd54c569911..36fa8b7e524d 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pool.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pool.c
@@ -61,6 +61,7 @@ static int pool_active(struct i915_active *ref)
return 0;
}
+__i915_active_call
static void pool_retire(struct i915_active *ref)
{
struct intel_engine_pool_node *node =
diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c
index 9cb01d9828f1..d824bca43d55 100644
--- a/drivers/gpu/drm/i915/gt/intel_timeline.c
+++ b/drivers/gpu/drm/i915/gt/intel_timeline.c
@@ -136,6 +136,7 @@ static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
kfree(cl);
}
+__i915_active_call
static void __cacheline_retire(struct i915_active *active)
{
struct intel_timeline_cacheline *cl =
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index 5e9f826ff132..fba77d284fba 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -128,6 +128,7 @@ __active_retire(struct i915_active *ref)
bool retire = false;
lockdep_assert_held(&ref->mutex);
+ GEM_BUG_ON(i915_active_is_idle(ref));
/* return the unused nodes to our slabcache -- flushing the allocator */
if (atomic_dec_and_test(&ref->count)) {
@@ -153,6 +154,19 @@ __active_retire(struct i915_active *ref)
ref->retire(ref);
}
+static void
+active_work(struct work_struct *wrk)
+{
+ struct i915_active *ref = container_of(wrk, typeof(*ref), work);
+
+ GEM_BUG_ON(!atomic_read(&ref->count));
+ if (atomic_add_unless(&ref->count, -1, 1))
+ return;
+
+ mutex_lock(&ref->mutex);
+ __active_retire(ref);
+}
+
static void
active_retire(struct i915_active *ref)
{
@@ -160,8 +174,13 @@ active_retire(struct i915_active *ref)
if (atomic_add_unless(&ref->count, -1, 1))
return;
- /* One active may be flushed from inside the acquire of another */
- mutex_lock_nested(&ref->mutex, SINGLE_DEPTH_NESTING);
+ /* If we are inside interrupt context (fence signaling), defer */
+ if (ref->flags & I915_ACTIVE_RETIRE_SLEEPS ||
+ !mutex_trylock(&ref->mutex)) {
+ queue_work(system_unbound_wq, &ref->work);
+ return;
+ }
+
__active_retire(ref);
}
@@ -236,12 +255,16 @@ void __i915_active_init(struct drm_i915_private *i915,
void (*retire)(struct i915_active *ref),
struct lock_class_key *key)
{
+ unsigned long bits;
+
debug_active_init(ref);
ref->i915 = i915;
ref->flags = 0;
ref->active = active;
- ref->retire = retire;
+ ref->retire = ptr_unpack_bits(retire, &bits, 2);
+ if (bits & I915_ACTIVE_MAY_SLEEP)
+ ref->flags |= I915_ACTIVE_RETIRE_SLEEPS;
ref->excl = NULL;
ref->tree = RB_ROOT;
@@ -249,6 +272,7 @@ void __i915_active_init(struct drm_i915_private *i915,
init_llist_head(&ref->preallocated_barriers);
atomic_set(&ref->count, 0);
__mutex_init(&ref->mutex, "i915_active", key);
+ INIT_WORK(&ref->work, active_work);
}
static bool ____active_del_barrier(struct i915_active *ref,
@@ -499,6 +523,7 @@ int i915_active_wait(struct i915_active *ref)
if (wait_on_bit(&ref->flags, I915_ACTIVE_GRAB_BIT, TASK_KILLABLE))
return -EINTR;
+ flush_work(&ref->work);
if (!i915_active_is_idle(ref))
return -EBUSY;
@@ -543,8 +568,9 @@ int i915_request_await_active(struct i915_request *rq, struct i915_active *ref)
void i915_active_fini(struct i915_active *ref)
{
debug_active_fini(ref);
- GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree));
GEM_BUG_ON(atomic_read(&ref->count));
+ GEM_BUG_ON(work_pending(&ref->work));
+ GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree));
mutex_destroy(&ref->mutex);
}
#endif
diff --git a/drivers/gpu/drm/i915/i915_active_types.h b/drivers/gpu/drm/i915/i915_active_types.h
index 86e7a232ea3c..021167f0004d 100644
--- a/drivers/gpu/drm/i915/i915_active_types.h
+++ b/drivers/gpu/drm/i915/i915_active_types.h
@@ -13,6 +13,9 @@
#include <linux/mutex.h>
#include <linux/rbtree.h>
#include <linux/rcupdate.h>
+#include <linux/workqueue.h>
+
+#include "i915_utils.h"
struct drm_i915_private;
struct i915_active_request;
@@ -44,6 +47,11 @@ struct i915_active_request {
struct active_node;
+#define I915_ACTIVE_MAY_SLEEP BIT(0)
+
+#define __i915_active_call __aligned(4)
+#define i915_active_may_sleep(fn) ptr_pack_bits(&(fn), I915_ACTIVE_MAY_SLEEP, 2)
+
struct i915_active {
struct drm_i915_private *i915;
@@ -57,11 +65,14 @@ struct i915_active {
struct dma_fence_cb excl_cb;
unsigned long flags;
-#define I915_ACTIVE_GRAB_BIT 0
+#define I915_ACTIVE_RETIRE_SLEEPS BIT(0)
+#define I915_ACTIVE_GRAB_BIT 1
int (*active)(struct i915_active *ref);
void (*retire)(struct i915_active *ref);
+ struct work_struct work;
+
struct llist_head preallocated_barriers;
};
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index f7b20fd945a9..8f3654216b25 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -91,6 +91,7 @@ static int __i915_vma_active(struct i915_active *ref)
return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT;
}
+__i915_active_call
static void __i915_vma_retire(struct i915_active *ref)
{
i915_vma_put(active_to_vma(ref));
@@ -1039,6 +1040,7 @@ int i915_vma_unbind(struct i915_vma *vma)
return -EBUSY;
}
+ GEM_BUG_ON(i915_vma_is_active(vma));
if (!drm_mm_node_allocated(&vma->node))
return 0;
diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c
index 77d844ac8b71..d5ac9944d093 100644
--- a/drivers/gpu/drm/i915/selftests/i915_active.c
+++ b/drivers/gpu/drm/i915/selftests/i915_active.c
@@ -121,7 +121,7 @@ __live_active_setup(struct drm_i915_private *i915)
}
i915_active_release(&active->base);
- if (active->retired && count) {
+ if (READ_ONCE(active->retired) && count) {
pr_err("i915_active retired before submission!\n");
err = -EINVAL;
}
@@ -161,7 +161,7 @@ static int live_active_wait(void *arg)
}
i915_active_wait(&active->base);
- if (!active->retired) {
+ if (!READ_ONCE(active->retired)) {
pr_err("i915_active not retired after waiting!\n");
err = -EINVAL;
}
@@ -200,7 +200,7 @@ static int live_active_retire(void *arg)
if (igt_flush_test(i915, I915_WAIT_LOCKED))
err = -EIO;
- if (!active->retired) {
+ if (!READ_ONCE(active->retired)) {
pr_err("i915_active not retired after flushing!\n");
err = -EINVAL;
}
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 12/15] drm/i915: Coordinate i915_active with its own mutex
2019-08-23 13:26 vm->mutex Chris Wilson
` (10 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 11/15] drm/i915: Push the i915_active.retire into a worker Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 13/15] drm/i915: Move idle barrier cleanup into engine-pm Chris Wilson
` (5 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Forgo the struct_mutex serialisation for i915_active, and interpose its
own mutex handling for active/retire.
This is a multi-layered sleight-of-hand. First, we had to ensure that no
active/retire callbacks accidentally inverted the mutex ordering rules,
nor assumed that they were themselves serialised by struct_mutex. More
challenging though, is the rule over updating elements of the active
rbtree. Instead of the whole i915_active now being serialised by
struct_mutex, allocations/rotations of the tree are serialised by the
i915_active.mutex and individual nodes are serialised by the caller
using the i915_timeline.mutex (we need to use nested spinlocks to
interact with the dma_fence callback lists).
The pain point here is that instead of a single mutex around execbuf, we
now have to take a mutex for active tracker (one for each vma, context,
etc) and a couple of spinlocks for each fence update. The improvement in
fine grained locking allowing for multiple concurrent clients
(eventually!) should be worth it in typical loads.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
.../gpu/drm/i915/display/intel_frontbuffer.c | 2 +-
drivers/gpu/drm/i915/display/intel_overlay.c | 5 +-
.../gpu/drm/i915/gem/i915_gem_client_blt.c | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_context.c | 8 +-
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 2 +-
.../gpu/drm/i915/gem/i915_gem_object_types.h | 1 +
drivers/gpu/drm/i915/gem/i915_gem_pm.c | 9 +-
drivers/gpu/drm/i915/gt/intel_context.c | 6 +-
drivers/gpu/drm/i915/gt/intel_engine_pool.c | 2 +-
drivers/gpu/drm/i915/gt/intel_engine_pool.h | 2 +-
drivers/gpu/drm/i915/gt/intel_reset.c | 10 +-
drivers/gpu/drm/i915/gt/intel_timeline.c | 9 +-
.../gpu/drm/i915/gt/intel_timeline_types.h | 2 +-
drivers/gpu/drm/i915/gt/selftest_context.c | 16 +-
drivers/gpu/drm/i915/gt/selftest_lrc.c | 10 +-
.../gpu/drm/i915/gt/selftests/mock_timeline.c | 2 +-
drivers/gpu/drm/i915/gvt/scheduler.c | 3 -
drivers/gpu/drm/i915/i915_active.c | 256 ++++++--------
drivers/gpu/drm/i915/i915_active.h | 315 ++++--------------
drivers/gpu/drm/i915/i915_active_types.h | 20 +-
drivers/gpu/drm/i915/i915_gem.c | 42 ++-
drivers/gpu/drm/i915/i915_gem_gtt.c | 3 +-
drivers/gpu/drm/i915/i915_gpu_error.c | 4 +-
drivers/gpu/drm/i915/i915_request.c | 39 +--
drivers/gpu/drm/i915/i915_request.h | 1 -
drivers/gpu/drm/i915/i915_vma.c | 8 +-
drivers/gpu/drm/i915/selftests/i915_active.c | 36 +-
27 files changed, 256 insertions(+), 559 deletions(-)
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
index d62810c1256a..eeb5964797a7 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
@@ -250,7 +250,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj)
front->obj = obj;
kref_init(&front->ref);
atomic_set(&front->bits, 0);
- i915_active_init(i915, &front->write,
+ i915_active_init(&front->write,
frontbuffer_active,
i915_active_may_sleep(frontbuffer_retire));
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
index 4f36557b3f3b..544e953342ea 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.c
+++ b/drivers/gpu/drm/i915/display/intel_overlay.c
@@ -230,7 +230,7 @@ alloc_request(struct intel_overlay *overlay, void (*fn)(struct intel_overlay *))
if (IS_ERR(rq))
return rq;
- err = i915_active_ref(&overlay->last_flip, rq->timeline, rq);
+ err = i915_active_ref(&overlay->last_flip, rq->timeline, &rq->fence);
if (err) {
i915_request_add(rq);
return ERR_PTR(err);
@@ -1360,8 +1360,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv)
overlay->contrast = 75;
overlay->saturation = 146;
- i915_active_init(dev_priv,
- &overlay->last_flip,
+ i915_active_init(&overlay->last_flip,
NULL, intel_overlay_last_flip_retire);
ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(dev_priv));
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
index f99920652751..c345d3d62df1 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
@@ -211,7 +211,7 @@ static void clear_pages_worker(struct work_struct *work)
* keep track of the GPU activity within this vma/request, and
* propagate the signal from the request to w->dma.
*/
- err = i915_active_ref(&vma->active, rq->timeline, rq);
+ err = i915_active_ref(&vma->active, rq->timeline, &rq->fence);
if (err)
goto out_request;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 4dc30c640380..095ea647c862 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -868,20 +868,18 @@ static int context_barrier_task(struct i915_gem_context *ctx,
void (*task)(void *data),
void *data)
{
- struct drm_i915_private *i915 = ctx->i915;
struct context_barrier_task *cb;
struct i915_gem_engines_iter it;
struct intel_context *ce;
int err = 0;
- lockdep_assert_held(&i915->drm.struct_mutex);
GEM_BUG_ON(!task);
cb = kmalloc(sizeof(*cb), GFP_KERNEL);
if (!cb)
return -ENOMEM;
- i915_active_init(i915, &cb->base, NULL, cb_retire);
+ i915_active_init(&cb->base, NULL, cb_retire);
err = i915_active_acquire(&cb->base);
if (err) {
kfree(cb);
@@ -913,7 +911,9 @@ static int context_barrier_task(struct i915_gem_context *ctx,
if (emit)
err = emit(rq, data);
if (err == 0)
- err = i915_active_ref(&cb->base, rq->timeline, rq);
+ err = i915_active_ref(&cb->base,
+ rq->timeline,
+ &rq->fence);
i915_request_add(rq);
if (err)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index 05a3e24248a5..2ac26fb63003 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -1268,7 +1268,7 @@ relocate_entry(struct i915_vma *vma,
if (!eb->reloc_cache.vaddr &&
(DBG_FORCE_RELOC == FORCE_GPU_RELOC ||
- !dma_resv_test_signaled_rcu(vma->resv, true))) {
+ i915_vma_is_active(vma))) {
const unsigned int gen = eb->reloc_cache.gen;
unsigned int len;
u32 *batch;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index a09e74e27734..86509bec3a1a 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -8,6 +8,7 @@
#define __I915_GEM_OBJECT_TYPES_H__
#include <drm/drm_gem.h>
+#include <uapi/drm/i915_drm.h>
#include "i915_active.h"
#include "i915_selftest.h"
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
index 92e53c25424c..92558fa47108 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
@@ -16,14 +16,11 @@ static void call_idle_barriers(struct intel_engine_cs *engine)
struct llist_node *node, *next;
llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
- struct i915_active_request *active =
+ struct dma_fence_cb *cb =
container_of((struct list_head *)node,
- typeof(*active), link);
+ typeof(*cb), node);
- INIT_LIST_HEAD(&active->link);
- RCU_INIT_POINTER(active->request, NULL);
-
- active->retire(active, NULL);
+ cb->func(NULL, cb);
}
}
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index ae7c2689ef30..57e13c6f59c5 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -240,7 +240,7 @@ intel_context_init(struct intel_context *ce,
mutex_init(&ce->pin_mutex);
- i915_active_init(ctx->i915, &ce->active,
+ i915_active_init(&ce->active,
__intel_context_active, __intel_context_retire);
}
@@ -307,7 +307,7 @@ int intel_context_prepare_remote_request(struct intel_context *ce,
return err;
/* Queue this switch after current activity by this context. */
- err = i915_active_request_set(&tl->last_request, rq);
+ err = i915_active_fence_set(&tl->last_request, rq);
mutex_unlock(&tl->mutex);
if (err)
return err;
@@ -321,7 +321,7 @@ int intel_context_prepare_remote_request(struct intel_context *ce,
* words transfer the pinned ce object to tracked active request.
*/
GEM_BUG_ON(i915_active_is_idle(&ce->active));
- return i915_active_ref(&ce->active, rq->timeline, rq);
+ return i915_active_ref(&ce->active, rq->timeline, &rq->fence);
}
struct i915_request *intel_context_create_request(struct intel_context *ce)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pool.c b/drivers/gpu/drm/i915/gt/intel_engine_pool.c
index 36fa8b7e524d..25b05c3db637 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pool.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pool.c
@@ -95,7 +95,7 @@ node_create(struct intel_engine_pool *pool, size_t sz)
return ERR_PTR(-ENOMEM);
node->pool = pool;
- i915_active_init(engine->i915, &node->active, pool_active, pool_retire);
+ i915_active_init(&node->active, pool_active, pool_retire);
obj = i915_gem_object_create_internal(engine->i915, sz);
if (IS_ERR(obj)) {
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pool.h b/drivers/gpu/drm/i915/gt/intel_engine_pool.h
index 8d069efd9457..7e83459b8da6 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pool.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pool.h
@@ -18,7 +18,7 @@ static inline int
intel_engine_pool_mark_active(struct intel_engine_pool_node *node,
struct i915_request *rq)
{
- return i915_active_ref(&node->active, rq->timeline, rq);
+ return i915_active_ref(&node->active, rq->timeline, &rq->fence);
}
static inline void
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index b9d84d52e986..4825c82aefee 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -814,10 +814,10 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
*/
spin_lock_irqsave(&timelines->lock, flags);
list_for_each_entry(tl, &timelines->active_list, link) {
- struct i915_request *rq;
+ struct dma_fence *fence;
- rq = i915_active_request_get_unlocked(&tl->last_request);
- if (!rq)
+ fence = i915_active_fence_get(&tl->last_request);
+ if (!fence)
continue;
spin_unlock_irqrestore(&timelines->lock, flags);
@@ -829,8 +829,8 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
* (I915_FENCE_TIMEOUT) so this wait should not be unbounded
* in the worst case.
*/
- dma_fence_default_wait(&rq->fence, false, MAX_SCHEDULE_TIMEOUT);
- i915_request_put(rq);
+ dma_fence_default_wait(fence, false, MAX_SCHEDULE_TIMEOUT);
+ dma_fence_put(fence);
/* Restart iteration after droping lock */
spin_lock_irqsave(&timelines->lock, flags);
diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c
index d824bca43d55..75d896167cfb 100644
--- a/drivers/gpu/drm/i915/gt/intel_timeline.c
+++ b/drivers/gpu/drm/i915/gt/intel_timeline.c
@@ -178,8 +178,7 @@ cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
cl->hwsp = hwsp;
cl->vaddr = page_pack_bits(vaddr, cacheline);
- i915_active_init(hwsp->gt->i915, &cl->active,
- __cacheline_active, __cacheline_retire);
+ i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
return cl;
}
@@ -255,7 +254,7 @@ int intel_timeline_init(struct intel_timeline *timeline,
mutex_init(&timeline->mutex);
- INIT_ACTIVE_REQUEST(&timeline->last_request, &timeline->mutex);
+ INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
INIT_LIST_HEAD(&timeline->requests);
i915_syncmap_init(&timeline->sync);
@@ -443,7 +442,7 @@ __intel_timeline_get_seqno(struct intel_timeline *tl,
* free it after the current request is retired, which ensures that
* all writes into the cacheline from previous requests are complete.
*/
- err = i915_active_ref(&tl->hwsp_cacheline->active, tl, rq);
+ err = i915_active_ref(&tl->hwsp_cacheline->active, tl, &rq->fence);
if (err)
goto err_cacheline;
@@ -494,7 +493,7 @@ int intel_timeline_get_seqno(struct intel_timeline *tl,
static int cacheline_ref(struct intel_timeline_cacheline *cl,
struct i915_request *rq)
{
- return i915_active_ref(&cl->active, rq->timeline, rq);
+ return i915_active_ref(&cl->active, rq->timeline, &rq->fence);
}
int intel_timeline_read_hwsp(struct i915_request *from,
diff --git a/drivers/gpu/drm/i915/gt/intel_timeline_types.h b/drivers/gpu/drm/i915/gt/intel_timeline_types.h
index 2b1baf2fcc8e..6d7ac129ce8a 100644
--- a/drivers/gpu/drm/i915/gt/intel_timeline_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_timeline_types.h
@@ -63,7 +63,7 @@ struct intel_timeline {
* the request using i915_active_request_get_request_rcu(), or hold the
* struct_mutex.
*/
- struct i915_active_request last_request;
+ struct i915_active_fence last_request;
/**
* We track the most recent seqno that we wait on in every context so
diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c
index 9d1ea26c7a2d..1420533e8fd5 100644
--- a/drivers/gpu/drm/i915/gt/selftest_context.c
+++ b/drivers/gpu/drm/i915/gt/selftest_context.c
@@ -41,24 +41,20 @@ static int context_sync(struct intel_context *ce)
mutex_lock(&tl->mutex);
do {
- struct i915_request *rq;
+ struct dma_fence *fence;
long timeout;
- rcu_read_lock();
- rq = rcu_dereference(tl->last_request.request);
- if (rq)
- rq = i915_request_get_rcu(rq);
- rcu_read_unlock();
- if (!rq)
+ fence = i915_active_fence_get(&tl->last_request);
+ if (!fence)
break;
- timeout = i915_request_wait(rq, 0, HZ / 10);
+ timeout = dma_fence_wait_timeout(fence, false, HZ / 10);
if (timeout < 0)
err = timeout;
else
- i915_request_retire_upto(rq);
+ i915_request_retire_upto(to_request(fence));
- i915_request_put(rq);
+ dma_fence_put(fence);
} while (!err);
mutex_unlock(&tl->mutex);
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index d791158988d6..aca1b3a9c5de 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -984,9 +984,13 @@ static struct i915_request *dummy_request(struct intel_engine_cs *engine)
if (!rq)
return NULL;
- INIT_LIST_HEAD(&rq->active_list);
rq->engine = engine;
+ spin_lock_init(&rq->lock);
+ INIT_LIST_HEAD(&rq->fence.cb_list);
+ rq->fence.lock = &rq->lock;
+ rq->fence.ops = &i915_fence_ops;
+
i915_sched_node_init(&rq->sched);
/* mark this request as permanently incomplete */
@@ -1079,8 +1083,8 @@ static int live_suppress_wait_preempt(void *arg)
}
/* Disable NEWCLIENT promotion */
- __i915_active_request_set(&rq[i]->timeline->last_request,
- dummy);
+ __i915_active_fence_set(&rq[i]->timeline->last_request,
+ &dummy->fence);
i915_request_add(rq[i]);
}
diff --git a/drivers/gpu/drm/i915/gt/selftests/mock_timeline.c b/drivers/gpu/drm/i915/gt/selftests/mock_timeline.c
index 598170efcaf6..2a77c051f36a 100644
--- a/drivers/gpu/drm/i915/gt/selftests/mock_timeline.c
+++ b/drivers/gpu/drm/i915/gt/selftests/mock_timeline.c
@@ -15,7 +15,7 @@ void mock_timeline_init(struct intel_timeline *timeline, u64 context)
mutex_init(&timeline->mutex);
- INIT_ACTIVE_REQUEST(&timeline->last_request, &timeline->mutex);
+ INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
INIT_LIST_HEAD(&timeline->requests);
i915_syncmap_init(&timeline->sync);
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 8940fa8d391a..6beb753b1ea1 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -385,11 +385,8 @@ intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload)
{
struct intel_vgpu *vgpu = workload->vgpu;
struct intel_vgpu_submission *s = &vgpu->submission;
- struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
struct i915_request *rq;
- lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
if (workload->req)
return 0;
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index fba77d284fba..2ecb78cb677e 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -12,8 +12,6 @@
#include "i915_active.h"
#include "i915_globals.h"
-#define BKL(ref) (&(ref)->i915->drm.struct_mutex)
-
/*
* Active refs memory management
*
@@ -27,35 +25,35 @@ static struct i915_global_active {
} global;
struct active_node {
- struct i915_active_request base;
+ struct i915_active_fence base;
struct i915_active *ref;
struct rb_node node;
u64 timeline;
};
static inline struct active_node *
-node_from_active(struct i915_active_request *active)
+node_from_active(struct i915_active_fence *active)
{
return container_of(active, struct active_node, base);
}
#define take_preallocated_barriers(x) llist_del_all(&(x)->preallocated_barriers)
-static inline bool is_barrier(const struct i915_active_request *active)
+static inline bool is_barrier(const struct i915_active_fence *active)
{
- return IS_ERR(rcu_access_pointer(active->request));
+ return IS_ERR(rcu_access_pointer(active->fence));
}
static inline struct llist_node *barrier_to_ll(struct active_node *node)
{
GEM_BUG_ON(!is_barrier(&node->base));
- return (struct llist_node *)&node->base.link;
+ return (struct llist_node *)&node->base.cb.node;
}
static inline struct intel_engine_cs *
__barrier_to_engine(struct active_node *node)
{
- return (struct intel_engine_cs *)READ_ONCE(node->base.link.prev);
+ return (struct intel_engine_cs *)READ_ONCE(node->base.cb.node.prev);
}
static inline struct intel_engine_cs *
@@ -68,7 +66,7 @@ barrier_to_engine(struct active_node *node)
static inline struct active_node *barrier_from_ll(struct llist_node *x)
{
return container_of((struct list_head *)x,
- struct active_node, base.link);
+ struct active_node, base.cb.node);
}
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) && IS_ENABLED(CONFIG_DEBUG_OBJECTS)
@@ -145,13 +143,16 @@ __active_retire(struct i915_active *ref)
GEM_BUG_ON(rcu_access_pointer(ref->excl));
rbtree_postorder_for_each_entry_safe(it, n, &root, node) {
- GEM_BUG_ON(i915_active_request_isset(&it->base));
+ GEM_BUG_ON(i915_active_fence_isset(&it->base));
kmem_cache_free(global.slab_cache, it);
}
/* After the final retire, the entire struct may be freed */
if (ref->retire)
ref->retire(ref);
+
+ /* ... except if you wait on it, you must manage your own references! */
+ wake_up_var(ref);
}
static void
@@ -185,12 +186,13 @@ active_retire(struct i915_active *ref)
}
static void
-node_retire(struct i915_active_request *base, struct i915_request *rq)
+node_retire(struct dma_fence *fence, struct dma_fence_cb *cb)
{
- active_retire(node_from_active(base)->ref);
+ i915_active_fence_cb(fence, cb);
+ active_retire(container_of(cb, struct active_node, base.cb)->ref);
}
-static struct i915_active_request *
+static struct i915_active_fence *
active_instance(struct i915_active *ref, struct intel_timeline *tl)
{
struct active_node *node, *prealloc;
@@ -234,7 +236,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl)
}
node = prealloc;
- i915_active_request_init(&node->base, &tl->mutex, NULL, node_retire);
+ __i915_active_fence_init(&node->base, &tl->mutex, NULL, node_retire);
node->ref = ref;
node->timeline = idx;
@@ -249,8 +251,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl)
return &node->base;
}
-void __i915_active_init(struct drm_i915_private *i915,
- struct i915_active *ref,
+void __i915_active_init(struct i915_active *ref,
int (*active)(struct i915_active *ref),
void (*retire)(struct i915_active *ref),
struct lock_class_key *key)
@@ -259,7 +260,6 @@ void __i915_active_init(struct drm_i915_private *i915,
debug_active_init(ref);
- ref->i915 = i915;
ref->flags = 0;
ref->active = active;
ref->retire = ptr_unpack_bits(retire, &bits, 2);
@@ -325,9 +325,9 @@ __active_del_barrier(struct i915_active *ref, struct active_node *node)
int i915_active_ref(struct i915_active *ref,
struct intel_timeline *tl,
- struct i915_request *rq)
+ struct dma_fence *fence)
{
- struct i915_active_request *active;
+ struct i915_active_fence *active;
int err;
lockdep_assert_held(&tl->mutex);
@@ -350,14 +350,11 @@ int i915_active_ref(struct i915_active *ref,
* request that we want to emit on the kernel_context.
*/
__active_del_barrier(ref, node_from_active(active));
- RCU_INIT_POINTER(active->request, NULL);
- INIT_LIST_HEAD(&active->link);
- } else {
- if (!i915_active_request_isset(active))
- atomic_inc(&ref->count);
+ RCU_INIT_POINTER(active->fence, NULL);
+ atomic_dec(&ref->count);
}
- GEM_BUG_ON(!atomic_read(&ref->count));
- __i915_active_request_set(active, rq);
+ if (!__i915_active_fence_set(active, fence))
+ atomic_inc(&ref->count);
out:
i915_active_release(ref);
@@ -427,143 +424,53 @@ int i915_active_acquire(struct i915_active *ref)
return err;
}
-void i915_active_release(struct i915_active *ref)
+bool i915_active_acquire_if_busy(struct i915_active *ref)
{
debug_active_assert(ref);
- active_retire(ref);
-}
-static void __active_ungrab(struct i915_active *ref)
-{
- clear_and_wake_up_bit(I915_ACTIVE_GRAB_BIT, &ref->flags);
+ return atomic_add_unless(&ref->count, 1, 0);
}
-bool i915_active_trygrab(struct i915_active *ref)
+void i915_active_release(struct i915_active *ref)
{
debug_active_assert(ref);
-
- if (test_and_set_bit(I915_ACTIVE_GRAB_BIT, &ref->flags))
- return false;
-
- if (!atomic_add_unless(&ref->count, 1, 0)) {
- __active_ungrab(ref);
- return false;
- }
-
- return true;
-}
-
-void i915_active_ungrab(struct i915_active *ref)
-{
- GEM_BUG_ON(!test_bit(I915_ACTIVE_GRAB_BIT, &ref->flags));
-
active_retire(ref);
- __active_ungrab(ref);
-}
-
-static int excl_wait(struct i915_active *ref)
-{
- struct dma_fence *old;
- int err = 0;
-
- if (!rcu_access_pointer(ref->excl))
- return 0;
-
- rcu_read_lock();
- old = dma_fence_get_rcu_safe(&ref->excl);
- rcu_read_unlock();
- if (old) {
- err = dma_fence_wait(old, true);
- dma_fence_put(old);
- }
-
- return err;
}
int i915_active_wait(struct i915_active *ref)
{
struct active_node *it, *n;
- int err;
+ int err = 0;
might_sleep();
- might_lock(&ref->mutex);
- if (i915_active_is_idle(ref))
+ if (!i915_active_acquire_if_busy(ref))
return 0;
- err = mutex_lock_interruptible(&ref->mutex);
- if (err)
- return err;
-
- if (!atomic_add_unless(&ref->count, 1, 0)) {
- mutex_unlock(&ref->mutex);
- return 0;
- }
+ /* Flush lazy signals */
+ rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
+ struct dma_fence *fence;
- err = excl_wait(ref);
- if (err)
- goto out;
+ if (is_barrier(&it->base)) /* unconnected idle barrier */
+ continue;
- rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
- if (is_barrier(&it->base)) { /* unconnected idle-barrier */
- err = -EBUSY;
- break;
+ fence = i915_active_fence_get(&it->base);
+ if (fence) {
+ dma_fence_enable_sw_signaling(fence);
+ dma_fence_put(fence);
}
-
- err = i915_active_request_retire(&it->base, BKL(ref));
- if (err)
- break;
}
-
-out:
- __active_retire(ref);
+ /* Any fence added after the wait begins will not be auto-signaled */
+ i915_active_release(ref);
if (err)
return err;
- if (wait_on_bit(&ref->flags, I915_ACTIVE_GRAB_BIT, TASK_KILLABLE))
+ if (wait_var_event_interruptible(ref, i915_active_is_idle(ref)))
return -EINTR;
- flush_work(&ref->work);
- if (!i915_active_is_idle(ref))
- return -EBUSY;
-
return 0;
}
-int i915_request_await_active_request(struct i915_request *rq,
- struct i915_active_request *active)
-{
- struct i915_request *barrier =
- i915_active_request_raw(active, &rq->i915->drm.struct_mutex);
-
- return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0;
-}
-
-int i915_request_await_active(struct i915_request *rq, struct i915_active *ref)
-{
- struct active_node *it, *n;
- int err;
-
- if (RB_EMPTY_ROOT(&ref->tree))
- return 0;
-
- /* await allocates and so we need to avoid hitting the shrinker */
- err = i915_active_acquire(ref);
- if (err)
- return err;
-
- mutex_lock(&ref->mutex);
- rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
- err = i915_request_await_active_request(rq, &it->base);
- if (err)
- break;
- }
- mutex_unlock(&ref->mutex);
-
- i915_active_release(ref);
- return err;
-}
-
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
void i915_active_fini(struct i915_active *ref)
{
@@ -577,7 +484,7 @@ void i915_active_fini(struct i915_active *ref)
static inline bool is_idle_barrier(struct active_node *node, u64 idx)
{
- return node->timeline == idx && !i915_active_request_isset(&node->base);
+ return node->timeline == idx && !i915_active_fence_isset(&node->base);
}
static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx)
@@ -697,13 +604,13 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
node->base.lock =
&engine->kernel_context->timeline->mutex;
#endif
- RCU_INIT_POINTER(node->base.request, NULL);
- node->base.retire = node_retire;
+ RCU_INIT_POINTER(node->base.fence, NULL);
+ node->base.cb.func = node_retire;
node->timeline = idx;
node->ref = ref;
}
- if (!i915_active_request_isset(&node->base)) {
+ if (!i915_active_fence_isset(&node->base)) {
/*
* Mark this as being *our* unconnected proto-node.
*
@@ -713,8 +620,8 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
* and then we can use the rb_node and list pointers
* for our tracking of the pending barrier.
*/
- RCU_INIT_POINTER(node->base.request, ERR_PTR(-EAGAIN));
- node->base.link.prev = (void *)engine;
+ RCU_INIT_POINTER(node->base.fence, ERR_PTR(-EAGAIN));
+ node->base.cb.node.prev = (void *)engine;
atomic_inc(&ref->count);
}
@@ -781,25 +688,65 @@ void i915_request_add_active_barriers(struct i915_request *rq)
{
struct intel_engine_cs *engine = rq->engine;
struct llist_node *node, *next;
+ unsigned long flags;
GEM_BUG_ON(intel_engine_is_virtual(engine));
GEM_BUG_ON(rq->timeline != engine->kernel_context->timeline);
+ node = llist_del_all(&engine->barrier_tasks);
+ if (!node)
+ return;
/*
* Attach the list of proto-fences to the in-flight request such
* that the parent i915_active will be released when this request
* is retired.
*/
- llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
- RCU_INIT_POINTER(barrier_from_ll(node)->base.request, rq);
+ spin_lock_irqsave(&rq->lock, flags);
+ llist_for_each_safe(node, next, node) {
+ RCU_INIT_POINTER(barrier_from_ll(node)->base.fence, &rq->fence);
smp_wmb(); /* serialise with reuse_idle_barrier */
- list_add_tail((struct list_head *)node, &rq->active_list);
+ list_add_tail((struct list_head *)node, &rq->fence.cb_list);
}
+ spin_unlock_irqrestore(&rq->lock, flags);
}
-int i915_active_request_set(struct i915_active_request *active,
- struct i915_request *rq)
+#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
+#define active_is_held(active) lockdep_is_held((active)->lock)
+#else
+#define active_is_held(active) true
+#endif
+
+struct dma_fence *
+__i915_active_fence_set(struct i915_active_fence *active,
+ struct dma_fence *fence)
{
+ struct dma_fence *prev;
+ unsigned long flags;
+
+ /* NB: updates must be serialised by an outer timeline mutex */
+ spin_lock_irqsave(fence->lock, flags);
+ GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
+
+ prev = rcu_dereference_protected(active->fence, active_is_held(active));
+ if (prev) {
+ spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING);
+ __list_del_entry(&active->cb.node);
+ spin_unlock(prev->lock); /* serialise with prev->cb_list */
+ prev = rcu_access_pointer(active->fence);
+ }
+
+ rcu_assign_pointer(active->fence, fence);
+ list_add_tail(&active->cb.node, &fence->cb_list);
+
+ spin_unlock_irqrestore(fence->lock, flags);
+
+ return prev;
+}
+
+int i915_active_fence_set(struct i915_active_fence *active,
+ struct i915_request *rq)
+{
+ struct dma_fence *fence;
int err;
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
@@ -807,18 +754,25 @@ int i915_active_request_set(struct i915_active_request *active,
#endif
/* Must maintain ordering wrt previous active requests */
- err = i915_request_await_active_request(rq, active);
- if (err)
- return err;
+ rcu_read_lock();
+ fence = __i915_active_fence_set(active, &rq->fence);
+ if (fence)
+ fence = dma_fence_get_rcu(fence);
+ rcu_read_unlock();
+
+ if (fence) {
+ err = i915_request_await_dma_fence(rq, fence);
+ dma_fence_put(fence);
+ if (err)
+ return err;
+ }
- __i915_active_request_set(active, rq);
return 0;
}
-void i915_active_retire_noop(struct i915_active_request *active,
- struct i915_request *request)
+void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb)
{
- /* Space left intentionally blank */
+ i915_active_fence_cb(fence, cb);
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h
index bc1c9d5d4f63..05e94876c73d 100644
--- a/drivers/gpu/drm/i915/i915_active.h
+++ b/drivers/gpu/drm/i915/i915_active.h
@@ -10,7 +10,10 @@
#include <linux/lockdep.h>
#include "i915_active_types.h"
-#include "i915_request.h"
+
+struct i915_request;
+struct intel_engine_cs;
+struct intel_timeline;
/*
* We treat requests as fences. This is not be to confused with our
@@ -28,308 +31,108 @@
* write access so that we can perform concurrent read operations between
* the CPU and GPU engines, as well as waiting for all rendering to
* complete, or waiting for the last GPU user of a "fence register". The
- * object then embeds a #i915_active_request to track the most recent (in
+ * object then embeds a #i915_active_fence to track the most recent (in
* retirement order) request relevant for the desired mode of access.
- * The #i915_active_request is updated with i915_active_request_set() to
+ * The #i915_active_fence is updated with i915_active_fence_set() to
* track the most recent fence request, typically this is done as part of
* i915_vma_move_to_active().
*
- * When the #i915_active_request completes (is retired), it will
+ * When the #i915_active_fence completes (is retired), it will
* signal its completion to the owner through a callback as well as mark
- * itself as idle (i915_active_request.request == NULL). The owner
+ * itself as idle (i915_active_fence.request == NULL). The owner
* can then perform any action, such as delayed freeing of an active
* resource including itself.
*/
-void i915_active_retire_noop(struct i915_active_request *active,
- struct i915_request *request);
+void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb);
/**
- * i915_active_request_init - prepares the activity tracker for use
+ * __i915_active_fence_init - prepares the activity tracker for use
* @active - the active tracker
- * @rq - initial request to track, can be NULL
+ * @fence - initial fence to track, can be NULL
* @func - a callback when then the tracker is retired (becomes idle),
* can be NULL
*
- * i915_active_request_init() prepares the embedded @active struct for use as
- * an activity tracker, that is for tracking the last known active request
- * associated with it. When the last request becomes idle, when it is retired
+ * i915_active_fence_init() prepares the embedded @active struct for use as
+ * an activity tracker, that is for tracking the last known active fence
+ * associated with it. When the last fence becomes idle, when it is retired
* after completion, the optional callback @func is invoked.
*/
static inline void
-i915_active_request_init(struct i915_active_request *active,
+__i915_active_fence_init(struct i915_active_fence *active,
struct mutex *lock,
- struct i915_request *rq,
- i915_active_retire_fn retire)
+ void *fence,
+ dma_fence_func_t fn)
{
- RCU_INIT_POINTER(active->request, rq);
- INIT_LIST_HEAD(&active->link);
- active->retire = retire ?: i915_active_retire_noop;
+ RCU_INIT_POINTER(active->fence, fence);
+ active->cb.func = fn ?: i915_active_noop;
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
active->lock = lock;
#endif
}
-#define INIT_ACTIVE_REQUEST(name, lock) \
- i915_active_request_init((name), (lock), NULL, NULL)
-
-/**
- * i915_active_request_set - updates the tracker to watch the current request
- * @active - the active tracker
- * @request - the request to watch
- *
- * __i915_active_request_set() watches the given @request for completion. Whilst
- * that @request is busy, the @active reports busy. When that @request is
- * retired, the @active tracker is updated to report idle.
- */
-static inline void
-__i915_active_request_set(struct i915_active_request *active,
- struct i915_request *request)
-{
-#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
- lockdep_assert_held(active->lock);
-#endif
- list_move(&active->link, &request->active_list);
- rcu_assign_pointer(active->request, request);
-}
+#define INIT_ACTIVE_FENCE(A, LOCK) \
+ __i915_active_fence_init((A), (LOCK), NULL, NULL)
-int __must_check
-i915_active_request_set(struct i915_active_request *active,
- struct i915_request *rq);
+struct dma_fence *
+__i915_active_fence_set(struct i915_active_fence *active,
+ struct dma_fence *fence);
/**
- * i915_active_request_raw - return the active request
+ * i915_active_fence_set - updates the tracker to watch the current fence
* @active - the active tracker
+ * @rq - the request to watch
*
- * i915_active_request_raw() returns the current request being tracked, or NULL.
- * It does not obtain a reference on the request for the caller, so the caller
- * must hold struct_mutex.
+ * i915_active_fence_set() watches the given @rq for completion. While
+ * that @rq is busy, the @active reports busy. When that @rq is signaled
+ * (or else retired) the @active tracker is updated to report idle.
*/
-static inline struct i915_request *
-i915_active_request_raw(const struct i915_active_request *active,
- struct mutex *mutex)
-{
- return rcu_dereference_protected(active->request,
- lockdep_is_held(mutex));
-}
-
-/**
- * i915_active_request_peek - report the active request being monitored
- * @active - the active tracker
- *
- * i915_active_request_peek() returns the current request being tracked if
- * still active, or NULL. It does not obtain a reference on the request
- * for the caller, so the caller must hold struct_mutex.
- */
-static inline struct i915_request *
-i915_active_request_peek(const struct i915_active_request *active,
- struct mutex *mutex)
-{
- struct i915_request *request;
-
- request = i915_active_request_raw(active, mutex);
- if (!request || i915_request_completed(request))
- return NULL;
-
- return request;
-}
-
-/**
- * i915_active_request_get - return a reference to the active request
- * @active - the active tracker
- *
- * i915_active_request_get() returns a reference to the active request, or NULL
- * if the active tracker is idle. The caller must hold struct_mutex.
- */
-static inline struct i915_request *
-i915_active_request_get(const struct i915_active_request *active,
- struct mutex *mutex)
-{
- return i915_request_get(i915_active_request_peek(active, mutex));
-}
-
-/**
- * __i915_active_request_get_rcu - return a reference to the active request
- * @active - the active tracker
- *
- * __i915_active_request_get() returns a reference to the active request,
- * or NULL if the active tracker is idle. The caller must hold the RCU read
- * lock, but the returned pointer is safe to use outside of RCU.
- */
-static inline struct i915_request *
-__i915_active_request_get_rcu(const struct i915_active_request *active)
-{
- /*
- * Performing a lockless retrieval of the active request is super
- * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing
- * slab of request objects will not be freed whilst we hold the
- * RCU read lock. It does not guarantee that the request itself
- * will not be freed and then *reused*. Viz,
- *
- * Thread A Thread B
- *
- * rq = active.request
- * retire(rq) -> free(rq);
- * (rq is now first on the slab freelist)
- * active.request = NULL
- *
- * rq = new submission on a new object
- * ref(rq)
- *
- * To prevent the request from being reused whilst the caller
- * uses it, we take a reference like normal. Whilst acquiring
- * the reference we check that it is not in a destroyed state
- * (refcnt == 0). That prevents the request being reallocated
- * whilst the caller holds on to it. To check that the request
- * was not reallocated as we acquired the reference we have to
- * check that our request remains the active request across
- * the lookup, in the same manner as a seqlock. The visibility
- * of the pointer versus the reference counting is controlled
- * by using RCU barriers (rcu_dereference and rcu_assign_pointer).
- *
- * In the middle of all that, we inspect whether the request is
- * complete. Retiring is lazy so the request may be completed long
- * before the active tracker is updated. Querying whether the
- * request is complete is far cheaper (as it involves no locked
- * instructions setting cachelines to exclusive) than acquiring
- * the reference, so we do it first. The RCU read lock ensures the
- * pointer dereference is valid, but does not ensure that the
- * seqno nor HWS is the right one! However, if the request was
- * reallocated, that means the active tracker's request was complete.
- * If the new request is also complete, then both are and we can
- * just report the active tracker is idle. If the new request is
- * incomplete, then we acquire a reference on it and check that
- * it remained the active request.
- *
- * It is then imperative that we do not zero the request on
- * reallocation, so that we can chase the dangling pointers!
- * See i915_request_alloc().
- */
- do {
- struct i915_request *request;
-
- request = rcu_dereference(active->request);
- if (!request || i915_request_completed(request))
- return NULL;
-
- /*
- * An especially silly compiler could decide to recompute the
- * result of i915_request_completed, more specifically
- * re-emit the load for request->fence.seqno. A race would catch
- * a later seqno value, which could flip the result from true to
- * false. Which means part of the instructions below might not
- * be executed, while later on instructions are executed. Due to
- * barriers within the refcounting the inconsistency can't reach
- * past the call to i915_request_get_rcu, but not executing
- * that while still executing i915_request_put() creates
- * havoc enough. Prevent this with a compiler barrier.
- */
- barrier();
-
- request = i915_request_get_rcu(request);
-
- /*
- * What stops the following rcu_access_pointer() from occurring
- * before the above i915_request_get_rcu()? If we were
- * to read the value before pausing to get the reference to
- * the request, we may not notice a change in the active
- * tracker.
- *
- * The rcu_access_pointer() is a mere compiler barrier, which
- * means both the CPU and compiler are free to perform the
- * memory read without constraint. The compiler only has to
- * ensure that any operations after the rcu_access_pointer()
- * occur afterwards in program order. This means the read may
- * be performed earlier by an out-of-order CPU, or adventurous
- * compiler.
- *
- * The atomic operation at the heart of
- * i915_request_get_rcu(), see dma_fence_get_rcu(), is
- * atomic_inc_not_zero() which is only a full memory barrier
- * when successful. That is, if i915_request_get_rcu()
- * returns the request (and so with the reference counted
- * incremented) then the following read for rcu_access_pointer()
- * must occur after the atomic operation and so confirm
- * that this request is the one currently being tracked.
- *
- * The corresponding write barrier is part of
- * rcu_assign_pointer().
- */
- if (!request || request == rcu_access_pointer(active->request))
- return rcu_pointer_handoff(request);
-
- i915_request_put(request);
- } while (1);
-}
-
+int __must_check
+i915_active_fence_set(struct i915_active_fence *active,
+ struct i915_request *rq);
/**
- * i915_active_request_get_unlocked - return a reference to the active request
+ * i915_active_fence_get - return a reference to the active fence
* @active - the active tracker
*
- * i915_active_request_get_unlocked() returns a reference to the active request,
+ * i915_active_fence_get() returns a reference to the active fence,
* or NULL if the active tracker is idle. The reference is obtained under RCU,
* so no locking is required by the caller.
*
- * The reference should be freed with i915_request_put().
+ * The reference should be freed with dma_fence_put().
*/
-static inline struct i915_request *
-i915_active_request_get_unlocked(const struct i915_active_request *active)
+static inline struct dma_fence *
+i915_active_fence_get(struct i915_active_fence *active)
{
- struct i915_request *request;
+ struct dma_fence *fence;
rcu_read_lock();
- request = __i915_active_request_get_rcu(active);
+ fence = dma_fence_get_rcu_safe(&active->fence);
rcu_read_unlock();
- return request;
+ return fence;
}
/**
- * i915_active_request_isset - report whether the active tracker is assigned
+ * i915_active_fence_isset - report whether the active tracker is assigned
* @active - the active tracker
*
- * i915_active_request_isset() returns true if the active tracker is currently
- * assigned to a request. Due to the lazy retiring, that request may be idle
+ * i915_active_fence_isset() returns true if the active tracker is currently
+ * assigned to a fence. Due to the lazy retiring, that fence may be idle
* and this may report stale information.
*/
static inline bool
-i915_active_request_isset(const struct i915_active_request *active)
+i915_active_fence_isset(const struct i915_active_fence *active)
{
- return rcu_access_pointer(active->request);
+ return rcu_access_pointer(active->fence);
}
-/**
- * i915_active_request_retire - waits until the request is retired
- * @active - the active request on which to wait
- *
- * i915_active_request_retire() waits until the request is completed,
- * and then ensures that at least the retirement handler for this
- * @active tracker is called before returning. If the @active
- * tracker is idle, the function returns immediately.
- */
-static inline int __must_check
-i915_active_request_retire(struct i915_active_request *active,
- struct mutex *mutex)
+static inline void
+i915_active_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
{
- struct i915_request *request;
- long ret;
-
- request = i915_active_request_raw(active, mutex);
- if (!request)
- return 0;
-
- ret = i915_request_wait(request,
- I915_WAIT_INTERRUPTIBLE,
- MAX_SCHEDULE_TIMEOUT);
- if (ret < 0)
- return ret;
+ struct i915_active_fence *active =
+ container_of(cb, typeof(*active), cb);
- list_del_init(&active->link);
- RCU_INIT_POINTER(active->request, NULL);
-
- active->retire(active, request);
-
- return 0;
+ RCU_INIT_POINTER(active->fence, NULL);
}
/*
@@ -358,20 +161,19 @@ i915_active_request_retire(struct i915_active_request *active,
* synchronisation.
*/
-void __i915_active_init(struct drm_i915_private *i915,
- struct i915_active *ref,
+void __i915_active_init(struct i915_active *ref,
int (*active)(struct i915_active *ref),
void (*retire)(struct i915_active *ref),
struct lock_class_key *key);
-#define i915_active_init(i915, ref, active, retire) do { \
+#define i915_active_init(ref, active, retire) do { \
static struct lock_class_key __key; \
\
- __i915_active_init(i915, ref, active, retire, &__key); \
+ __i915_active_init(ref, active, retire, &__key); \
} while (0)
int i915_active_ref(struct i915_active *ref,
struct intel_timeline *tl,
- struct i915_request *rq);
+ struct dma_fence *fence);
void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f);
@@ -379,15 +181,12 @@ int i915_active_wait(struct i915_active *ref);
int i915_request_await_active(struct i915_request *rq,
struct i915_active *ref);
-int i915_request_await_active_request(struct i915_request *rq,
- struct i915_active_request *active);
+int i915_request_await_active_fence(struct i915_request *rq,
+ struct i915_active_fence *active);
int i915_active_acquire(struct i915_active *ref);
+bool i915_active_acquire_if_busy(struct i915_active *ref);
void i915_active_release(struct i915_active *ref);
-void __i915_active_release_nested(struct i915_active *ref, int subclass);
-
-bool i915_active_trygrab(struct i915_active *ref);
-void i915_active_ungrab(struct i915_active *ref);
static inline bool
i915_active_is_idle(const struct i915_active *ref)
diff --git a/drivers/gpu/drm/i915/i915_active_types.h b/drivers/gpu/drm/i915/i915_active_types.h
index 021167f0004d..123f2e883261 100644
--- a/drivers/gpu/drm/i915/i915_active_types.h
+++ b/drivers/gpu/drm/i915/i915_active_types.h
@@ -17,17 +17,9 @@
#include "i915_utils.h"
-struct drm_i915_private;
-struct i915_active_request;
-struct i915_request;
-
-typedef void (*i915_active_retire_fn)(struct i915_active_request *,
- struct i915_request *);
-
-struct i915_active_request {
- struct i915_request __rcu *request;
- struct list_head link;
- i915_active_retire_fn retire;
+struct i915_active_fence {
+ struct dma_fence __rcu *fence;
+ struct dma_fence_cb cb;
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
/*
* Incorporeal!
@@ -53,12 +45,11 @@ struct active_node;
#define i915_active_may_sleep(fn) ptr_pack_bits(&(fn), I915_ACTIVE_MAY_SLEEP, 2)
struct i915_active {
- struct drm_i915_private *i915;
+ atomic_t count;
+ struct mutex mutex;
struct active_node *cache;
struct rb_root tree;
- struct mutex mutex;
- atomic_t count;
/* Preallocated "exclusive" node */
struct dma_fence __rcu *excl;
@@ -66,7 +57,6 @@ struct i915_active {
unsigned long flags;
#define I915_ACTIVE_RETIRE_SLEEPS BIT(0)
-#define I915_ACTIVE_GRAB_BIT 1
int (*active)(struct i915_active *ref);
void (*retire)(struct i915_active *ref);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 85323cd08022..8f4c8c29cab8 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -892,28 +892,38 @@ wait_for_timelines(struct drm_i915_private *i915,
spin_lock_irqsave(&timelines->lock, flags);
list_for_each_entry(tl, &timelines->active_list, link) {
- struct i915_request *rq;
+ struct dma_fence *fence;
- rq = i915_active_request_get_unlocked(&tl->last_request);
- if (!rq)
+ fence = i915_active_fence_get(&tl->last_request);
+ if (!fence)
continue;
spin_unlock_irqrestore(&timelines->lock, flags);
- /*
- * "Race-to-idle".
- *
- * Switching to the kernel context is often used a synchronous
- * step prior to idling, e.g. in suspend for flushing all
- * current operations to memory before sleeping. These we
- * want to complete as quickly as possible to avoid prolonged
- * stalls, so allow the gpu to boost to maximum clocks.
- */
- if (wait & I915_WAIT_FOR_IDLE_BOOST)
- gen6_rps_boost(rq);
+ if (!dma_fence_is_i915(fence)) {
+ timeout = dma_fence_wait_timeout(fence,
+ flags & I915_WAIT_INTERRUPTIBLE,
+ timeout);
+ } else {
+ struct i915_request *rq = to_request(fence);
+
+ /*
+ * "Race-to-idle".
+ *
+ * Switching to the kernel context is often used as
+ * a synchronous step prior to idling, e.g. in suspend
+ * for flushing all current operations to memory before
+ * sleeping. These we want to complete as quickly as
+ * possible to avoid prolonged stalls, so allow the gpu
+ * to boost to maximum clocks.
+ */
+ if (flags & I915_WAIT_FOR_IDLE_BOOST)
+ gen6_rps_boost(rq);
+
+ timeout = i915_request_wait(rq, flags, timeout);
+ }
- timeout = i915_request_wait(rq, wait, timeout);
- i915_request_put(rq);
+ dma_fence_put(fence);
if (timeout < 0)
return timeout;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 8f742f3eec60..2680b388dc33 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -1862,7 +1862,6 @@ static const struct i915_vma_ops pd_vma_ops = {
static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
{
- struct drm_i915_private *i915 = ppgtt->base.vm.i915;
struct i915_ggtt *ggtt = ppgtt->base.vm.gt->ggtt;
struct i915_vma *vma;
@@ -1873,7 +1872,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
if (!vma)
return ERR_PTR(-ENOMEM);
- i915_active_init(i915, &vma->active, NULL, NULL);
+ i915_active_init(&vma->active, NULL, NULL);
vma->vm = i915_vm_get(&ggtt->vm);
vma->ops = &pd_vma_ops;
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index e284bd76fa86..dced86e4a95e 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -1291,7 +1291,7 @@ capture_vma(struct capture_vma *next,
if (!c)
return next;
- if (!i915_active_trygrab(&vma->active)) {
+ if (!i915_active_acquire_if_busy(&vma->active)) {
kfree(c);
return next;
}
@@ -1431,7 +1431,7 @@ gem_record_rings(struct i915_gpu_state *error, struct compress *compress)
*this->slot =
i915_error_object_create(i915, vma, compress);
- i915_active_ungrab(&vma->active);
+ i915_active_release(&vma->active);
i915_vma_put(vma);
capture = this->next;
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index a53777dd371c..e9c0a068ceec 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -196,9 +196,8 @@ static void free_capture_list(struct i915_request *request)
static bool i915_request_retire(struct i915_request *rq)
{
- struct i915_active_request *active, *next;
-
lockdep_assert_held(&rq->timeline->mutex);
+
if (!i915_request_completed(rq))
return false;
@@ -222,35 +221,6 @@ static bool i915_request_retire(struct i915_request *rq)
GEM_BUG_ON(!list_is_first(&rq->link, &rq->timeline->requests));
rq->ring->head = rq->postfix;
- /*
- * Walk through the active list, calling retire on each. This allows
- * objects to track their GPU activity and mark themselves as idle
- * when their *last* active request is completed (updating state
- * tracking lists for eviction, active references for GEM, etc).
- *
- * As the ->retire() may free the node, we decouple it first and
- * pass along the auxiliary information (to avoid dereferencing
- * the node after the callback).
- */
- list_for_each_entry_safe(active, next, &rq->active_list, link) {
- /*
- * In microbenchmarks or focusing upon time inside the kernel,
- * we may spend an inordinate amount of time simply handling
- * the retirement of requests and processing their callbacks.
- * Of which, this loop itself is particularly hot due to the
- * cache misses when jumping around the list of
- * i915_active_request. So we try to keep this loop as
- * streamlined as possible and also prefetch the next
- * i915_active_request to try and hide the likely cache miss.
- */
- prefetchw(next);
-
- INIT_LIST_HEAD(&active->link);
- RCU_INIT_POINTER(active->request, NULL);
-
- active->retire(active, rq);
- }
-
local_irq_disable();
/*
@@ -663,7 +633,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
rq->flags = 0;
rq->execution_mask = ALL_ENGINES;
- INIT_LIST_HEAD(&rq->active_list);
INIT_LIST_HEAD(&rq->execute_cb);
/*
@@ -702,7 +671,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
ce->ring->emit = rq->head;
/* Make sure we didn't add ourselves to external state before freeing */
- GEM_BUG_ON(!list_empty(&rq->active_list));
GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
@@ -1095,8 +1063,8 @@ __i915_request_add_to_timeline(struct i915_request *rq)
* precludes optimising to use semaphores serialisation of a single
* timeline across engines.
*/
- prev = rcu_dereference_protected(timeline->last_request.request,
- lockdep_is_held(&timeline->mutex));
+ prev = to_request(__i915_active_fence_set(&timeline->last_request,
+ &rq->fence));
if (prev && !i915_request_completed(prev)) {
if (is_power_of_2(prev->engine->mask | rq->engine->mask))
i915_sw_fence_await_sw_fence(&rq->submit,
@@ -1121,7 +1089,6 @@ __i915_request_add_to_timeline(struct i915_request *rq)
* us, the timeline will hold its seqno which is later than ours.
*/
GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
- __i915_active_request_set(&timeline->last_request, rq);
return prev;
}
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 8ac6e1226a56..3251d2bdbeea 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -211,7 +211,6 @@ struct i915_request {
* on the active_list (of their final request).
*/
struct i915_capture_list *capture_list;
- struct list_head active_list;
/** Time at which this request was emitted, in jiffies. */
unsigned long emitted_jiffies;
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 8f3654216b25..ce57dd366bf2 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -119,8 +119,7 @@ vma_create(struct drm_i915_gem_object *obj,
vma->size = obj->base.size;
vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
- i915_active_init(vm->i915, &vma->active,
- __i915_vma_active, __i915_vma_retire);
+ i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire);
/* Declare ourselves safe for use inside shrinkers */
if (IS_ENABLED(CONFIG_LOCKDEP)) {
@@ -993,7 +992,7 @@ int i915_vma_move_to_active(struct i915_vma *vma,
* add the active reference first and queue for it to be dropped
* *last*.
*/
- err = i915_active_ref(&vma->active, rq->timeline, rq);
+ err = i915_active_ref(&vma->active, rq->timeline, &rq->fence);
if (unlikely(err))
return err;
@@ -1001,7 +1000,7 @@ int i915_vma_move_to_active(struct i915_vma *vma,
if (intel_frontbuffer_invalidate(obj->frontbuffer, ORIGIN_CS))
i915_active_ref(&obj->frontbuffer->write,
rq->timeline,
- rq);
+ &rq->fence);
dma_resv_add_excl_fence(vma->resv, &rq->fence);
obj->write_domain = I915_GEM_DOMAIN_RENDER;
@@ -1035,6 +1034,7 @@ int i915_vma_unbind(struct i915_vma *vma)
if (ret)
return ret;
+ GEM_BUG_ON(i915_vma_is_active(vma));
if (i915_vma_is_pinned(vma)) {
vma_print_allocator(vma, "is pinned");
return -EBUSY;
diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c
index d5ac9944d093..af5827aac7b2 100644
--- a/drivers/gpu/drm/i915/selftests/i915_active.c
+++ b/drivers/gpu/drm/i915/selftests/i915_active.c
@@ -68,7 +68,7 @@ static struct live_active *__live_alloc(struct drm_i915_private *i915)
return NULL;
kref_init(&active->ref);
- i915_active_init(i915, &active->base, __live_active, __live_retire);
+ i915_active_init(&active->base, __live_active, __live_retire);
return active;
}
@@ -110,7 +110,9 @@ __live_active_setup(struct drm_i915_private *i915)
submit,
GFP_KERNEL);
if (err >= 0)
- err = i915_active_ref(&active->base, rq->timeline, rq);
+ err = i915_active_ref(&active->base,
+ rq->timeline,
+ &rq->fence);
i915_request_add(rq);
if (err) {
pr_err("Failed to track active ref!\n");
@@ -146,19 +148,13 @@ static int live_active_wait(void *arg)
{
struct drm_i915_private *i915 = arg;
struct live_active *active;
- intel_wakeref_t wakeref;
int err = 0;
/* Check that we get a callback when requests retire upon waiting */
- mutex_lock(&i915->drm.struct_mutex);
- wakeref = intel_runtime_pm_get(&i915->runtime_pm);
-
active = __live_active_setup(i915);
- if (IS_ERR(active)) {
- err = PTR_ERR(active);
- goto err;
- }
+ if (IS_ERR(active))
+ return PTR_ERR(active);
i915_active_wait(&active->base);
if (!READ_ONCE(active->retired)) {
@@ -168,11 +164,9 @@ static int live_active_wait(void *arg)
__live_put(active);
+ mutex_lock(&i915->drm.struct_mutex);
if (igt_flush_test(i915, I915_WAIT_LOCKED))
err = -EIO;
-
-err:
- intel_runtime_pm_put(&i915->runtime_pm, wakeref);
mutex_unlock(&i915->drm.struct_mutex);
return err;
@@ -182,23 +176,19 @@ static int live_active_retire(void *arg)
{
struct drm_i915_private *i915 = arg;
struct live_active *active;
- intel_wakeref_t wakeref;
int err = 0;
/* Check that we get a callback when requests are indirectly retired */
- mutex_lock(&i915->drm.struct_mutex);
- wakeref = intel_runtime_pm_get(&i915->runtime_pm);
-
active = __live_active_setup(i915);
- if (IS_ERR(active)) {
- err = PTR_ERR(active);
- goto err;
- }
+ if (IS_ERR(active))
+ return PTR_ERR(active);
/* waits for & retires all requests */
+ mutex_lock(&i915->drm.struct_mutex);
if (igt_flush_test(i915, I915_WAIT_LOCKED))
err = -EIO;
+ mutex_unlock(&i915->drm.struct_mutex);
if (!READ_ONCE(active->retired)) {
pr_err("i915_active not retired after flushing!\n");
@@ -207,10 +197,6 @@ static int live_active_retire(void *arg)
__live_put(active);
-err:
- intel_runtime_pm_put(&i915->runtime_pm, wakeref);
- mutex_unlock(&i915->drm.struct_mutex);
-
return err;
}
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 13/15] drm/i915: Move idle barrier cleanup into engine-pm
2019-08-23 13:26 vm->mutex Chris Wilson
` (11 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 12/15] drm/i915: Coordinate i915_active with its own mutex Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:26 ` [PATCH 14/15] drm/i915: Drop struct_mutex from around i915_retire_requests() Chris Wilson
` (4 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
Now that we now longer need to guarantee that the active callback is
under the struct_mutex, we can lift it out of the i915_gem_park() and
into the engine parking itself.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
drivers/gpu/drm/i915/gem/i915_gem_pm.c | 19 -------------------
drivers/gpu/drm/i915/gt/intel_engine_pm.c | 15 +++++++++++++++
drivers/gpu/drm/i915/i915_active.c | 1 +
3 files changed, 16 insertions(+), 19 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
index 92558fa47108..6e4cc177cc7a 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
@@ -11,29 +11,10 @@
#include "i915_drv.h"
#include "i915_globals.h"
-static void call_idle_barriers(struct intel_engine_cs *engine)
-{
- struct llist_node *node, *next;
-
- llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
- struct dma_fence_cb *cb =
- container_of((struct list_head *)node,
- typeof(*cb), node);
-
- cb->func(NULL, cb);
- }
-}
-
static void i915_gem_park(struct drm_i915_private *i915)
{
- struct intel_engine_cs *engine;
- enum intel_engine_id id;
-
lockdep_assert_held(&i915->drm.struct_mutex);
- for_each_engine(engine, i915, id)
- call_idle_barriers(engine); /* cleanup after wedging */
-
i915_vma_parked(i915);
i915_globals_park();
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
index 65b5ca74b394..472b2259f629 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -123,6 +123,19 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine)
return result;
}
+static void call_idle_barriers(struct intel_engine_cs *engine)
+{
+ struct llist_node *node, *next;
+
+ llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
+ struct dma_fence_cb *cb =
+ container_of((struct list_head *)node,
+ typeof(*cb), node);
+
+ cb->func(NULL, cb);
+ }
+}
+
static int __engine_park(struct intel_wakeref *wf)
{
struct intel_engine_cs *engine =
@@ -142,6 +155,8 @@ static int __engine_park(struct intel_wakeref *wf)
GEM_TRACE("%s\n", engine->name);
+ call_idle_barriers(engine); /* cleanup after wedging */
+
intel_engine_disarm_breadcrumbs(engine);
intel_engine_pool_park(&engine->pool);
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index 2ecb78cb677e..aa644cc941d3 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -678,6 +678,7 @@ void i915_active_acquire_barrier(struct i915_active *ref)
rb_link_node(&node->node, parent, p);
rb_insert_color(&node->node, &ref->tree);
+ GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
llist_add(barrier_to_ll(node), &engine->barrier_tasks);
intel_engine_pm_put(engine);
}
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 14/15] drm/i915: Drop struct_mutex from around i915_retire_requests()
2019-08-23 13:26 vm->mutex Chris Wilson
` (12 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 13/15] drm/i915: Move idle barrier cleanup into engine-pm Chris Wilson
@ 2019-08-23 13:26 ` Chris Wilson
2019-08-23 13:27 ` [PATCH 15/15] drm/i915: Serialise the fill BLT with the vma pinning Chris Wilson
` (3 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:26 UTC (permalink / raw)
To: intel-gfx
We don't need to hold struct_mutex now for retiring requests, so drop it
from i915_retire_requests() and i915_gem_wait_for_idle(), finally
removing I915_WAIT_LOCKED for good.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
.../gpu/drm/i915/gem/i915_gem_client_blt.c | 7 +--
drivers/gpu/drm/i915/gem/i915_gem_context.c | 20 +--------
drivers/gpu/drm/i915/gem/i915_gem_pm.c | 45 +++++++------------
.../drm/i915/gem/selftests/i915_gem_context.c | 4 +-
.../drm/i915/gem/selftests/i915_gem_mman.c | 6 +--
.../i915/gem/selftests/i915_gem_object_blt.c | 2 -
drivers/gpu/drm/i915/gt/selftest_context.c | 4 +-
drivers/gpu/drm/i915/gt/selftest_hangcheck.c | 28 ++++--------
drivers/gpu/drm/i915/gt/selftest_lrc.c | 21 +++++----
drivers/gpu/drm/i915/gt/selftest_timeline.c | 6 +--
.../gpu/drm/i915/gt/selftest_workarounds.c | 6 +--
drivers/gpu/drm/i915/i915_debugfs.c | 42 +++++------------
drivers/gpu/drm/i915/i915_gem.c | 19 ++++----
drivers/gpu/drm/i915/i915_request.h | 7 ++-
drivers/gpu/drm/i915/selftests/i915_active.c | 8 +---
.../gpu/drm/i915/selftests/i915_gem_evict.c | 2 +-
.../gpu/drm/i915/selftests/i915_selftest.c | 8 +---
.../gpu/drm/i915/selftests/igt_flush_test.c | 30 +++++--------
.../gpu/drm/i915/selftests/igt_flush_test.h | 2 +-
.../gpu/drm/i915/selftests/igt_live_test.c | 9 +---
.../gpu/drm/i915/selftests/mock_gem_device.c | 4 --
21 files changed, 91 insertions(+), 189 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
index c345d3d62df1..3502071e1391 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
@@ -155,7 +155,6 @@ static void clear_pages_dma_fence_cb(struct dma_fence *fence,
static void clear_pages_worker(struct work_struct *work)
{
struct clear_pages_work *w = container_of(work, typeof(*w), work);
- struct drm_i915_private *i915 = w->ce->engine->i915;
struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
struct i915_vma *vma = w->sleeve->vma;
struct i915_request *rq;
@@ -173,11 +172,9 @@ static void clear_pages_worker(struct work_struct *work)
obj->read_domains = I915_GEM_GPU_DOMAINS;
obj->write_domain = 0;
- /* XXX: we need to kill this */
- mutex_lock(&i915->drm.struct_mutex);
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (unlikely(err))
- goto out_unlock;
+ goto out_signal;
batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
if (IS_ERR(batch)) {
@@ -229,8 +226,6 @@ static void clear_pages_worker(struct work_struct *work)
intel_emit_vma_release(w->ce, batch);
out_unpin:
i915_vma_unpin(vma);
-out_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
out_signal:
if (unlikely(err)) {
dma_fence_set_error(&w->dma, err);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 095ea647c862..dc2113767603 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -1164,8 +1164,7 @@ gen8_modify_rpcs(struct intel_context *ce, struct intel_sseu sseu)
}
static int
-__intel_context_reconfigure_sseu(struct intel_context *ce,
- struct intel_sseu sseu)
+intel_context_reconfigure_sseu(struct intel_context *ce, struct intel_sseu sseu)
{
int ret;
@@ -1188,23 +1187,6 @@ __intel_context_reconfigure_sseu(struct intel_context *ce,
return ret;
}
-static int
-intel_context_reconfigure_sseu(struct intel_context *ce, struct intel_sseu sseu)
-{
- struct drm_i915_private *i915 = ce->engine->i915;
- int ret;
-
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- return ret;
-
- ret = __intel_context_reconfigure_sseu(ce, sseu);
-
- mutex_unlock(&i915->drm.struct_mutex);
-
- return ret;
-}
-
static int
user_to_context_sseu(struct drm_i915_private *i915,
const struct drm_i915_gem_context_param_sseu *user,
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
index 6e4cc177cc7a..fec0b410d1d9 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
@@ -48,11 +48,7 @@ static void retire_work_handler(struct work_struct *work)
struct drm_i915_private *i915 =
container_of(work, typeof(*i915), gem.retire_work.work);
- /* Come back later if the device is busy... */
- if (mutex_trylock(&i915->drm.struct_mutex)) {
- i915_retire_requests(i915);
- mutex_unlock(&i915->drm.struct_mutex);
- }
+ i915_retire_requests(i915);
queue_delayed_work(i915->wq,
&i915->gem.retire_work,
@@ -86,26 +82,23 @@ static bool switch_to_kernel_context_sync(struct intel_gt *gt)
{
bool result = !intel_gt_is_wedged(gt);
- do {
- if (i915_gem_wait_for_idle(gt->i915,
- I915_WAIT_LOCKED |
- I915_WAIT_FOR_IDLE_BOOST,
- I915_GEM_IDLE_TIMEOUT) == -ETIME) {
- /* XXX hide warning from gem_eio */
- if (i915_modparams.reset) {
- dev_err(gt->i915->drm.dev,
- "Failed to idle engines, declaring wedged!\n");
- GEM_TRACE_DUMP();
- }
-
- /*
- * Forcibly cancel outstanding work and leave
- * the gpu quiet.
- */
- intel_gt_set_wedged(gt);
- result = false;
+ if (i915_gem_wait_for_idle(gt->i915,
+ I915_WAIT_FOR_IDLE_BOOST,
+ I915_GEM_IDLE_TIMEOUT) == -ETIME) {
+ /* XXX hide warning from gem_eio */
+ if (i915_modparams.reset) {
+ dev_err(gt->i915->drm.dev,
+ "Failed to idle engines, declaring wedged!\n");
+ GEM_TRACE_DUMP();
}
- } while (i915_retire_requests(gt->i915) && result);
+
+ /*
+ * Forcibly cancel outstanding work and leave
+ * the gpu quiet.
+ */
+ intel_gt_set_wedged(gt);
+ result = false;
+ }
if (intel_gt_pm_wait_for_idle(gt))
result = false;
@@ -125,8 +118,6 @@ void i915_gem_suspend(struct drm_i915_private *i915)
intel_wakeref_auto(&i915->ggtt.userfault_wakeref, 0);
flush_workqueue(i915->wq);
- mutex_lock(&i915->drm.struct_mutex);
-
/*
* We have to flush all the executing contexts to main memory so
* that they can saved in the hibernation image. To ensure the last
@@ -138,8 +129,6 @@ void i915_gem_suspend(struct drm_i915_private *i915)
*/
switch_to_kernel_context_sync(&i915->gt);
- mutex_unlock(&i915->drm.struct_mutex);
-
cancel_delayed_work_sync(&i915->gt.hangcheck.work);
i915_gem_drain_freed_objects(i915);
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
index 3adb60c2fd1f..1d0a2ec3307a 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
@@ -893,7 +893,7 @@ __sseu_test(const char *name,
if (ret)
return ret;
- ret = __intel_context_reconfigure_sseu(ce, sseu);
+ ret = intel_context_reconfigure_sseu(ce, sseu);
if (ret)
goto out_spin;
@@ -997,7 +997,7 @@ __igt_ctx_sseu(struct drm_i915_private *i915,
goto out_fail;
out_fail:
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
ret = -EIO;
intel_context_unpin(ce);
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
index 9c217dfe96a9..39c01bc4eb51 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
@@ -393,12 +393,8 @@ static void disable_retire_worker(struct drm_i915_private *i915)
static void restore_retire_worker(struct drm_i915_private *i915)
{
+ igt_flush_test(i915);
intel_gt_pm_put(&i915->gt);
-
- mutex_lock(&i915->drm.struct_mutex);
- igt_flush_test(i915, I915_WAIT_LOCKED);
- mutex_unlock(&i915->drm.struct_mutex);
-
i915_gem_driver_register__shrinker(i915);
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c
index c21d747e7d05..5a351998a98d 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c
@@ -166,9 +166,7 @@ static int igt_copy_blt(void *arg)
if (!(dst->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
dst->cache_dirty = true;
- mutex_lock(&i915->drm.struct_mutex);
err = i915_gem_object_copy_blt(src, dst, ce);
- mutex_unlock(&i915->drm.struct_mutex);
if (err)
goto err_unpin;
diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c
index 1420533e8fd5..883739354b07 100644
--- a/drivers/gpu/drm/i915/gt/selftest_context.c
+++ b/drivers/gpu/drm/i915/gt/selftest_context.c
@@ -312,7 +312,7 @@ static int live_active_context(void *arg)
if (err)
break;
- err = igt_flush_test(gt->i915, I915_WAIT_LOCKED);
+ err = igt_flush_test(gt->i915);
if (err)
break;
}
@@ -425,7 +425,7 @@ static int live_remote_context(void *arg)
if (err)
break;
- err = igt_flush_test(gt->i915, I915_WAIT_LOCKED);
+ err = igt_flush_test(gt->i915);
if (err)
break;
}
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index e53eea1050f8..35ab703dac34 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -285,7 +285,7 @@ static void hang_fini(struct hang *h)
kernel_context_close(h->ctx);
- igt_flush_test(h->gt->i915, I915_WAIT_LOCKED);
+ igt_flush_test(h->gt->i915);
}
static bool wait_until_running(struct hang *h, struct i915_request *rq)
@@ -429,16 +429,13 @@ static int igt_reset_nop(void *arg)
break;
}
- err = igt_flush_test(gt->i915, 0);
+ err = igt_flush_test(gt->i915);
if (err)
break;
} while (time_before(jiffies, end_time));
pr_info("%s: %d resets\n", __func__, count);
- mutex_lock(>->i915->drm.struct_mutex);
- err = igt_flush_test(gt->i915, I915_WAIT_LOCKED);
- mutex_unlock(>->i915->drm.struct_mutex);
-
+ err = igt_flush_test(gt->i915);
out:
mock_file_free(gt->i915, file);
if (intel_gt_is_wedged(gt))
@@ -533,15 +530,12 @@ static int igt_reset_nop_engine(void *arg)
if (err)
break;
- err = igt_flush_test(gt->i915, 0);
+ err = igt_flush_test(gt->i915);
if (err)
break;
}
- mutex_lock(>->i915->drm.struct_mutex);
- err = igt_flush_test(gt->i915, I915_WAIT_LOCKED);
- mutex_unlock(>->i915->drm.struct_mutex);
-
+ err = igt_flush_test(gt->i915);
out:
mock_file_free(gt->i915, file);
if (intel_gt_is_wedged(gt))
@@ -647,7 +641,7 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active)
if (err)
break;
- err = igt_flush_test(gt->i915, 0);
+ err = igt_flush_test(gt->i915);
if (err)
break;
}
@@ -977,9 +971,7 @@ static int __igt_reset_engines(struct intel_gt *gt,
if (err)
break;
- mutex_lock(>->i915->drm.struct_mutex);
- err = igt_flush_test(gt->i915, I915_WAIT_LOCKED);
- mutex_unlock(>->i915->drm.struct_mutex);
+ err = igt_flush_test(gt->i915);
if (err)
break;
}
@@ -1511,7 +1503,7 @@ static int igt_reset_queue(void *arg)
i915_request_put(prev);
- err = igt_flush_test(gt->i915, I915_WAIT_LOCKED);
+ err = igt_flush_test(gt->i915);
if (err)
break;
}
@@ -1751,10 +1743,6 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
err = intel_gt_live_subtests(tests, gt);
- mutex_lock(>->i915->drm.struct_mutex);
- igt_flush_test(gt->i915, I915_WAIT_LOCKED);
- mutex_unlock(>->i915->drm.struct_mutex);
-
i915_modparams.enable_hangcheck = saved_hangcheck;
intel_runtime_pm_put(>->i915->runtime_pm, wakeref);
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index aca1b3a9c5de..222a7375c787 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -61,7 +61,7 @@ static int live_sanitycheck(void *arg)
}
igt_spinner_end(&spin);
- if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+ if (igt_flush_test(i915)) {
err = -EIO;
goto err_ctx;
}
@@ -206,8 +206,7 @@ slice_semaphore_queue(struct intel_engine_cs *outer,
if (err)
goto out;
- if (i915_request_wait(head,
- I915_WAIT_LOCKED,
+ if (i915_request_wait(head, 0,
2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
count, n);
@@ -279,7 +278,7 @@ static int live_timeslice_preempt(void *arg)
if (err)
goto err_pin;
- if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+ if (igt_flush_test(i915)) {
err = -EIO;
goto err_pin;
}
@@ -832,7 +831,7 @@ static int live_nopreempt(void *arg)
goto err_wedged;
}
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
goto err_wedged;
}
@@ -948,7 +947,7 @@ static int live_suppress_self_preempt(void *arg)
goto err_client_b;
}
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
goto err_wedged;
}
@@ -1109,7 +1108,7 @@ static int live_suppress_wait_preempt(void *arg)
for (i = 0; i < ARRAY_SIZE(client); i++)
igt_spinner_end(&client[i].spin);
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
goto err_wedged;
if (engine->execlists.preempt_hang.count) {
@@ -1388,7 +1387,7 @@ static int live_preempt_hang(void *arg)
igt_spinner_end(&spin_hi);
igt_spinner_end(&spin_lo);
- if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+ if (igt_flush_test(i915)) {
err = -EIO;
goto err_ctx_lo;
}
@@ -1785,7 +1784,7 @@ static int nop_virtual_engine(struct drm_i915_private *i915,
prime, div64_u64(ktime_to_ns(times[1]), prime));
out:
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
for (nc = 0; nc < nctx; nc++) {
@@ -1930,7 +1929,7 @@ static int mask_virtual_engine(struct drm_i915_private *i915,
goto out;
out:
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
for (n = 0; n < nsibling; n++)
@@ -2108,7 +2107,7 @@ static int bond_virtual_engine(struct drm_i915_private *i915,
out:
for (n = 0; !IS_ERR(rq[n]); n++)
i915_request_put(rq[n]);
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
kernel_context_close(ctx);
diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c
index 321481403165..42d0bbbbc7ea 100644
--- a/drivers/gpu/drm/i915/gt/selftest_timeline.c
+++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c
@@ -545,7 +545,7 @@ static int live_hwsp_engine(void *arg)
}
out:
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
for (n = 0; n < count; n++) {
@@ -621,7 +621,7 @@ static int live_hwsp_alternate(void *arg)
}
out:
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
for (n = 0; n < count; n++) {
@@ -747,7 +747,7 @@ static int live_hwsp_wrap(void *arg)
}
out:
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
intel_timeline_unpin(tl);
diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c
index d06d68ac2a3b..173df5507525 100644
--- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c
@@ -668,7 +668,7 @@ static int check_dirty_whitelist(struct i915_gem_context *ctx,
break;
}
- if (igt_flush_test(ctx->i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(ctx->i915))
err = -EIO;
out_batch:
i915_vma_unpin_and_release(&batch, 0);
@@ -1074,7 +1074,7 @@ static int live_isolated_whitelist(void *arg)
kernel_context_close(client[i].ctx);
}
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
return err;
@@ -1232,7 +1232,7 @@ live_engine_reset_workarounds(void *arg)
igt_global_reset_unlock(&i915->gt);
kernel_context_close(ctx);
- igt_flush_test(i915, I915_WAIT_LOCKED);
+ igt_flush_test(i915);
return ret;
}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index c36538f1b36c..32168b9a3f4e 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -3607,6 +3607,7 @@ static int
i915_drop_caches_set(void *data, u64 val)
{
struct drm_i915_private *i915 = data;
+ int ret;
DRM_DEBUG("Dropping caches: 0x%08llx [0x%08llx]\n",
val, val & DROP_ALL);
@@ -3616,40 +3617,21 @@ i915_drop_caches_set(void *data, u64 val)
I915_IDLE_ENGINES_TIMEOUT))
intel_gt_set_wedged(&i915->gt);
- /* No need to check and wait for gpu resets, only libdrm auto-restarts
- * on ioctls on -EAGAIN. */
- if (val & (DROP_ACTIVE | DROP_IDLE | DROP_RETIRE | DROP_RESET_SEQNO)) {
- int ret;
+ if (val & DROP_RETIRE)
+ i915_retire_requests(i915);
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
+ if (val & (DROP_IDLE | DROP_ACTIVE)) {
+ ret = i915_gem_wait_for_idle(i915,
+ I915_WAIT_INTERRUPTIBLE,
+ MAX_SCHEDULE_TIMEOUT);
if (ret)
return ret;
+ }
- /*
- * To finish the flush of the idle_worker, we must complete
- * the switch-to-kernel-context, which requires a double
- * pass through wait_for_idle: first queues the switch,
- * second waits for the switch.
- */
- if (ret == 0 && val & (DROP_IDLE | DROP_ACTIVE))
- ret = i915_gem_wait_for_idle(i915,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED,
- MAX_SCHEDULE_TIMEOUT);
-
- if (ret == 0 && val & DROP_IDLE)
- ret = i915_gem_wait_for_idle(i915,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED,
- MAX_SCHEDULE_TIMEOUT);
-
- if (val & DROP_RETIRE)
- i915_retire_requests(i915);
-
- mutex_unlock(&i915->drm.struct_mutex);
-
- if (ret == 0 && val & DROP_IDLE)
- ret = intel_gt_pm_wait_for_idle(&i915->gt);
+ if (val & DROP_IDLE) {
+ ret = intel_gt_pm_wait_for_idle(&i915->gt);
+ if (ret)
+ return ret;
}
if (val & DROP_RESET_ACTIVE && intel_gt_terminally_wedged(&i915->gt))
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 8f4c8c29cab8..a14b2ad13cd5 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -943,19 +943,16 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915,
if (!intel_gt_pm_is_awake(&i915->gt))
return 0;
- GEM_TRACE("flags=%x (%s), timeout=%ld%s\n",
- flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked",
- timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "");
-
- timeout = wait_for_timelines(i915, flags, timeout);
- if (timeout < 0)
- return timeout;
+ do {
+ timeout = wait_for_timelines(i915, flags, timeout);
+ if (timeout < 0)
+ return timeout;
- if (flags & I915_WAIT_LOCKED) {
- lockdep_assert_held(&i915->drm.struct_mutex);
+ cond_resched();
+ if (signal_pending(current))
+ return -EINTR;
- i915_retire_requests(i915);
- }
+ } while (i915_retire_requests(i915));
return 0;
}
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 3251d2bdbeea..57a2193c64d1 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -308,10 +308,9 @@ long i915_request_wait(struct i915_request *rq,
long timeout)
__attribute__((nonnull(1)));
#define I915_WAIT_INTERRUPTIBLE BIT(0)
-#define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */
-#define I915_WAIT_PRIORITY BIT(2) /* small priority bump for the request */
-#define I915_WAIT_ALL BIT(3) /* used by i915_gem_object_wait() */
-#define I915_WAIT_FOR_IDLE_BOOST BIT(4)
+#define I915_WAIT_PRIORITY BIT(1) /* small priority bump for the request */
+#define I915_WAIT_ALL BIT(2) /* used by i915_gem_object_wait() */
+#define I915_WAIT_FOR_IDLE_BOOST BIT(3)
static inline bool i915_request_signaled(const struct i915_request *rq)
{
diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c
index af5827aac7b2..ff1337e34522 100644
--- a/drivers/gpu/drm/i915/selftests/i915_active.c
+++ b/drivers/gpu/drm/i915/selftests/i915_active.c
@@ -164,10 +164,8 @@ static int live_active_wait(void *arg)
__live_put(active);
- mutex_lock(&i915->drm.struct_mutex);
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
- mutex_unlock(&i915->drm.struct_mutex);
return err;
}
@@ -185,10 +183,8 @@ static int live_active_retire(void *arg)
return PTR_ERR(active);
/* waits for & retires all requests */
- mutex_lock(&i915->drm.struct_mutex);
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
- mutex_unlock(&i915->drm.struct_mutex);
if (!READ_ONCE(active->retired)) {
pr_err("i915_active not retired after flushing!\n");
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index ba6064147173..42139db0d69c 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -521,7 +521,7 @@ static int igt_evict_contexts(void *arg)
mutex_lock(&i915->ggtt.vm.mutex);
out_locked:
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
while (reserved) {
struct reserved *next = reserved->next;
diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c
index 438ea0eaa416..825a8286cbe8 100644
--- a/drivers/gpu/drm/i915/selftests/i915_selftest.c
+++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c
@@ -263,10 +263,8 @@ int __i915_live_teardown(int err, void *data)
{
struct drm_i915_private *i915 = data;
- mutex_lock(&i915->drm.struct_mutex);
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
err = -EIO;
- mutex_unlock(&i915->drm.struct_mutex);
i915_gem_drain_freed_objects(i915);
@@ -284,10 +282,8 @@ int __intel_gt_live_teardown(int err, void *data)
{
struct intel_gt *gt = data;
- mutex_lock(>->i915->drm.struct_mutex);
- if (igt_flush_test(gt->i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(gt->i915))
err = -EIO;
- mutex_unlock(>->i915->drm.struct_mutex);
i915_gem_drain_freed_objects(gt->i915);
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
index d3b5eb402d33..2a5fbe46ea9f 100644
--- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
@@ -12,31 +12,25 @@
#include "igt_flush_test.h"
-int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
+int igt_flush_test(struct drm_i915_private *i915)
{
int ret = intel_gt_is_wedged(&i915->gt) ? -EIO : 0;
- int repeat = !!(flags & I915_WAIT_LOCKED);
cond_resched();
- do {
- if (i915_gem_wait_for_idle(i915, flags, HZ / 5) == -ETIME) {
- pr_err("%pS timed out, cancelling all further testing.\n",
- __builtin_return_address(0));
+ i915_retire_requests(i915);
+ if (i915_gem_wait_for_idle(i915, 0, HZ / 5) == -ETIME) {
+ pr_err("%pS timed out, cancelling all further testing.\n",
+ __builtin_return_address(0));
- GEM_TRACE("%pS timed out.\n",
- __builtin_return_address(0));
- GEM_TRACE_DUMP();
+ GEM_TRACE("%pS timed out.\n",
+ __builtin_return_address(0));
+ GEM_TRACE_DUMP();
- intel_gt_set_wedged(&i915->gt);
- repeat = 0;
- ret = -EIO;
- }
-
- /* Ensure we also flush after wedging. */
- if (flags & I915_WAIT_LOCKED)
- i915_retire_requests(i915);
- } while (repeat--);
+ intel_gt_set_wedged(&i915->gt);
+ ret = -EIO;
+ }
+ i915_retire_requests(i915);
return ret;
}
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.h b/drivers/gpu/drm/i915/selftests/igt_flush_test.h
index 63e009927c43..7541fa74e641 100644
--- a/drivers/gpu/drm/i915/selftests/igt_flush_test.h
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.h
@@ -9,6 +9,6 @@
struct drm_i915_private;
-int igt_flush_test(struct drm_i915_private *i915, unsigned int flags);
+int igt_flush_test(struct drm_i915_private *i915);
#endif /* IGT_FLUSH_TEST_H */
diff --git a/drivers/gpu/drm/i915/selftests/igt_live_test.c b/drivers/gpu/drm/i915/selftests/igt_live_test.c
index 3e902761cd16..04a6f88fdf64 100644
--- a/drivers/gpu/drm/i915/selftests/igt_live_test.c
+++ b/drivers/gpu/drm/i915/selftests/igt_live_test.c
@@ -19,15 +19,12 @@ int igt_live_test_begin(struct igt_live_test *t,
enum intel_engine_id id;
int err;
- lockdep_assert_held(&i915->drm.struct_mutex);
-
t->i915 = i915;
t->func = func;
t->name = name;
err = i915_gem_wait_for_idle(i915,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED,
+ I915_WAIT_INTERRUPTIBLE,
MAX_SCHEDULE_TIMEOUT);
if (err) {
pr_err("%s(%s): failed to idle before, with err=%d!",
@@ -50,9 +47,7 @@ int igt_live_test_end(struct igt_live_test *t)
struct intel_engine_cs *engine;
enum intel_engine_id id;
- lockdep_assert_held(&i915->drm.struct_mutex);
-
- if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ if (igt_flush_test(i915))
return -EIO;
if (t->reset_global != i915_reset_count(&i915->gpu_error)) {
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 01a89c071bf5..1956006a0d5b 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -41,8 +41,6 @@ void mock_device_flush(struct drm_i915_private *i915)
struct intel_engine_cs *engine;
enum intel_engine_id id;
- lockdep_assert_held(&i915->drm.struct_mutex);
-
do {
for_each_engine(engine, i915, id)
mock_engine_flush(engine);
@@ -55,9 +53,7 @@ static void mock_device_release(struct drm_device *dev)
struct intel_engine_cs *engine;
enum intel_engine_id id;
- mutex_lock(&i915->drm.struct_mutex);
mock_device_flush(i915);
- mutex_unlock(&i915->drm.struct_mutex);
flush_work(&i915->gem.idle_work);
i915_gem_drain_workqueue(i915);
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH 15/15] drm/i915: Serialise the fill BLT with the vma pinning
2019-08-23 13:26 vm->mutex Chris Wilson
` (13 preceding siblings ...)
2019-08-23 13:26 ` [PATCH 14/15] drm/i915: Drop struct_mutex from around i915_retire_requests() Chris Wilson
@ 2019-08-23 13:27 ` Chris Wilson
2019-08-23 16:48 ` ✗ Fi.CI.CHECKPATCH: warning for series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev3) Patchwork
` (2 subsequent siblings)
17 siblings, 0 replies; 25+ messages in thread
From: Chris Wilson @ 2019-08-23 13:27 UTC (permalink / raw)
To: intel-gfx; +Cc: Matthew Auld
Make sure that we wait for the vma to be pinned prior to telling the GPU
to fill the pages through that vma.
However, since our async operations fight over obj->resv->excl_fence we
must manually order them. This makes it much more fragile, and gives an
outside observer the chance to see the intermediate fences. To be
discussed!
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Matthew Auld <matthew.auld@intel.com>
Reviewed-by: Matthew Auld <matthew.auld@intel.com>
---
.../gpu/drm/i915/gem/i915_gem_client_blt.c | 46 ++++++++++++++-----
1 file changed, 35 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
index 3502071e1391..bbbc10499099 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
@@ -71,10 +71,30 @@ static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
goto err_free;
}
+ /*
+ * XXX fix scheduling with get_pages & clear workers
+ *
+ * The complication is that we end up overwriting the same
+ * obj->resv->excl_fence for each stage of the operation. That fence
+ * should be set on scheduling the work, and only signaled upon
+ * completion of the entire workqueue.
+ *
+ * Within the workqueue, we use the fence to schedule each individual
+ * task. Each individual task knows to use obj->resv->fence.
+ *
+ * To an outsider, they must wait until the end and so the
+ * obj->resv->fence must be the composite.
+ *
+ * Ideas?
+ */
+ err = i915_vma_pin(vma, 0, 0, PIN_USER);
+ if (unlikely(err))
+ goto err_free;
+
vma->private = sleeve;
vma->ops = &proxy_vma_ops;
- sleeve->vma = vma;
+ sleeve->vma = i915_vma_get(vma);
sleeve->pages = pages;
sleeve->page_sizes = *page_sizes;
@@ -87,6 +107,13 @@ static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
static void destroy_sleeve(struct i915_sleeve *sleeve)
{
+ struct i915_vma *vma = sleeve->vma;
+
+ if (vma) {
+ i915_vma_unpin(vma);
+ i915_vma_put(vma);
+ }
+
kfree(sleeve);
}
@@ -155,8 +182,8 @@ static void clear_pages_dma_fence_cb(struct dma_fence *fence,
static void clear_pages_worker(struct work_struct *work)
{
struct clear_pages_work *w = container_of(work, typeof(*w), work);
- struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
- struct i915_vma *vma = w->sleeve->vma;
+ struct i915_vma *vma = fetch_and_zero(&w->sleeve->vma);
+ struct drm_i915_gem_object *obj = vma->obj;
struct i915_request *rq;
struct i915_vma *batch;
int err = w->dma.error;
@@ -166,20 +193,16 @@ static void clear_pages_worker(struct work_struct *work)
if (obj->cache_dirty) {
if (i915_gem_object_has_struct_page(obj))
- drm_clflush_sg(w->sleeve->pages);
+ drm_clflush_sg(vma->pages);
obj->cache_dirty = false;
}
obj->read_domains = I915_GEM_GPU_DOMAINS;
obj->write_domain = 0;
- err = i915_vma_pin(vma, 0, 0, PIN_USER);
- if (unlikely(err))
- goto out_signal;
-
batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
if (IS_ERR(batch)) {
err = PTR_ERR(batch);
- goto out_unpin;
+ goto out_signal;
}
rq = intel_context_create_request(w->ce);
@@ -224,14 +247,15 @@ static void clear_pages_worker(struct work_struct *work)
i915_request_add(rq);
out_batch:
intel_emit_vma_release(w->ce, batch);
-out_unpin:
- i915_vma_unpin(vma);
out_signal:
if (unlikely(err)) {
dma_fence_set_error(&w->dma, err);
dma_fence_signal(&w->dma);
dma_fence_put(&w->dma);
}
+
+ i915_vma_unpin(vma);
+ i915_vma_put(vma);
}
static int __i915_sw_fence_call
--
2.23.0
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 25+ messages in thread* ✗ Fi.CI.CHECKPATCH: warning for series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev3)
2019-08-23 13:26 vm->mutex Chris Wilson
` (14 preceding siblings ...)
2019-08-23 13:27 ` [PATCH 15/15] drm/i915: Serialise the fill BLT with the vma pinning Chris Wilson
@ 2019-08-23 16:48 ` Patchwork
2019-08-23 17:43 ` ✗ Fi.CI.BAT: failure " Patchwork
2019-08-23 20:06 ` ✗ Fi.CI.BAT: failure for series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev4) Patchwork
17 siblings, 0 replies; 25+ messages in thread
From: Patchwork @ 2019-08-23 16:48 UTC (permalink / raw)
To: Chris Wilson; +Cc: intel-gfx
== Series Details ==
Series: series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev3)
URL : https://patchwork.freedesktop.org/series/65705/
State : warning
== Summary ==
$ dim checkpatch origin/drm-tip
173c1bc89e03 drm/i915: Hold irq-off for the entire fake lock period
d896cc87daca drm/i915/gtt: Preallocate Braswell top-level page directory
-:45: WARNING:ONE_SEMICOLON: Statements terminations use 1 semicolon
#45: FILE: drivers/gpu/drm/i915/gem/i915_gem_context.c:1017:
+ *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;;
-:57: WARNING:MEMORY_BARRIER: memory barrier without comment
#57: FILE: drivers/gpu/drm/i915/i915_gem_gtt.c:165:
+ wmb();
-:65: WARNING:MEMORY_BARRIER: memory barrier without comment
#65: FILE: drivers/gpu/drm/i915/i915_gem_gtt.c:1424:
+ wmb();
total: 0 errors, 3 warnings, 0 checks, 47 lines checked
bb4855e5c4eb drm/i915/selftests: Teach igt_gpu_fill_dw() to take intel_context
0f977fd1da12 dma-fence: Serialise signal enabling (dma_fence_enable_sw_signaling)
-:14: ERROR:GIT_COMMIT_ID: Please use git commit description style 'commit <12+ chars of sha1> ("<title line>")' - ie: 'commit 0fc89b6802ba ("dma-fence: Simply wrap dma_fence_signal_locked with dma_fence_signal")'
#14:
See also 0fc89b6802ba ("dma-fence: Simply wrap dma_fence_signal_locked
total: 1 errors, 0 warnings, 0 checks, 24 lines checked
a0f807f2b948 drm/i915: Make shrink/unshrink be atomic
bb43a3b5113b drm/i915: Mark up obj->pin_global as being atomic
3ec9eadcdacb drm/i915: Only track bound elements of the GTT
332370ea8b41 drm/i915: Make i915_vma.flags atomic_t for mutex reduction
-:7: WARNING:COMMIT_MESSAGE: Missing commit description - Add an appropriate one
total: 0 errors, 1 warnings, 0 checks, 337 lines checked
3631ac00b48a drm/i915: Mark up address spaces that may need to allocate
edba6cc97869 drm/i915: Pull i915_vma_pin under the vm->mutex
5ac14777c21e drm/i915: Push the i915_active.retire into a worker
79f3b830651c drm/i915: Coordinate i915_active with its own mutex
-:1299: CHECK:UNCOMMENTED_DEFINITION: struct mutex definition without comment
#1299: FILE: drivers/gpu/drm/i915/i915_active_types.h:49:
+ struct mutex mutex;
total: 0 errors, 0 warnings, 1 checks, 1448 lines checked
e2518334238f drm/i915: Move idle barrier cleanup into engine-pm
b23c41cf1147 drm/i915: Drop struct_mutex from around i915_retire_requests()
061e95161125 drm/i915: Serialise the fill BLT with the vma pinning
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 25+ messages in thread* ✗ Fi.CI.BAT: failure for series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev3)
2019-08-23 13:26 vm->mutex Chris Wilson
` (15 preceding siblings ...)
2019-08-23 16:48 ` ✗ Fi.CI.CHECKPATCH: warning for series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev3) Patchwork
@ 2019-08-23 17:43 ` Patchwork
2019-08-23 20:06 ` ✗ Fi.CI.BAT: failure for series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev4) Patchwork
17 siblings, 0 replies; 25+ messages in thread
From: Patchwork @ 2019-08-23 17:43 UTC (permalink / raw)
To: Chris Wilson; +Cc: intel-gfx
== Series Details ==
Series: series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev3)
URL : https://patchwork.freedesktop.org/series/65705/
State : failure
== Summary ==
CI Bug Log - changes from CI_DRM_6777 -> Patchwork_14169
====================================================
Summary
-------
**FAILURE**
Serious unknown changes coming with Patchwork_14169 absolutely need to be
verified manually.
If you think the reported changes have nothing to do with the changes
introduced in Patchwork_14169, please notify your bug team to allow them
to document this new failure mode, which will reduce false positives in CI.
External URL: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/
Possible new issues
-------------------
Here are the unknown changes that may have been introduced in Patchwork_14169:
### IGT changes ###
#### Possible regressions ####
* igt@gem_ctx_create@basic-files:
- fi-bwr-2160: [PASS][1] -> [TIMEOUT][2]
[1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-bwr-2160/igt@gem_ctx_create@basic-files.html
[2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-bwr-2160/igt@gem_ctx_create@basic-files.html
* igt@i915_selftest@live_gem_contexts:
- fi-byt-j1900: [PASS][3] -> [DMESG-FAIL][4]
[3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-byt-j1900/igt@i915_selftest@live_gem_contexts.html
[4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-byt-j1900/igt@i915_selftest@live_gem_contexts.html
* igt@i915_selftest@live_hugepages:
- fi-hsw-4770: [PASS][5] -> [INCOMPLETE][6]
[5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-hsw-4770/igt@i915_selftest@live_hugepages.html
[6]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-hsw-4770/igt@i915_selftest@live_hugepages.html
- fi-snb-2520m: [PASS][7] -> [INCOMPLETE][8]
[7]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-snb-2520m/igt@i915_selftest@live_hugepages.html
[8]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-snb-2520m/igt@i915_selftest@live_hugepages.html
- fi-whl-u: [PASS][9] -> [INCOMPLETE][10]
[9]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-whl-u/igt@i915_selftest@live_hugepages.html
[10]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-whl-u/igt@i915_selftest@live_hugepages.html
- fi-kbl-7567u: [PASS][11] -> [INCOMPLETE][12]
[11]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-kbl-7567u/igt@i915_selftest@live_hugepages.html
[12]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-kbl-7567u/igt@i915_selftest@live_hugepages.html
- fi-bdw-gvtdvm: [PASS][13] -> [INCOMPLETE][14]
[13]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-bdw-gvtdvm/igt@i915_selftest@live_hugepages.html
[14]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-bdw-gvtdvm/igt@i915_selftest@live_hugepages.html
- fi-kbl-guc: [PASS][15] -> [INCOMPLETE][16]
[15]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-kbl-guc/igt@i915_selftest@live_hugepages.html
[16]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-kbl-guc/igt@i915_selftest@live_hugepages.html
- fi-cfl-8109u: [PASS][17] -> [INCOMPLETE][18]
[17]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-cfl-8109u/igt@i915_selftest@live_hugepages.html
[18]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-cfl-8109u/igt@i915_selftest@live_hugepages.html
- fi-kbl-r: [PASS][19] -> [INCOMPLETE][20]
[19]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-kbl-r/igt@i915_selftest@live_hugepages.html
[20]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-kbl-r/igt@i915_selftest@live_hugepages.html
- fi-skl-6260u: [PASS][21] -> [INCOMPLETE][22]
[21]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-6260u/igt@i915_selftest@live_hugepages.html
[22]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-6260u/igt@i915_selftest@live_hugepages.html
- fi-bsw-kefka: [PASS][23] -> [INCOMPLETE][24]
[23]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-bsw-kefka/igt@i915_selftest@live_hugepages.html
[24]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-bsw-kefka/igt@i915_selftest@live_hugepages.html
- fi-skl-6770hq: [PASS][25] -> [INCOMPLETE][26]
[25]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-6770hq/igt@i915_selftest@live_hugepages.html
[26]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-6770hq/igt@i915_selftest@live_hugepages.html
- fi-cfl-guc: [PASS][27] -> [INCOMPLETE][28]
[27]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-cfl-guc/igt@i915_selftest@live_hugepages.html
[28]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-cfl-guc/igt@i915_selftest@live_hugepages.html
- fi-skl-lmem: [PASS][29] -> [INCOMPLETE][30]
[29]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-lmem/igt@i915_selftest@live_hugepages.html
[30]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-lmem/igt@i915_selftest@live_hugepages.html
- fi-skl-6700k2: [PASS][31] -> [INCOMPLETE][32]
[31]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-6700k2/igt@i915_selftest@live_hugepages.html
[32]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-6700k2/igt@i915_selftest@live_hugepages.html
- fi-cfl-8700k: [PASS][33] -> [INCOMPLETE][34]
[33]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-cfl-8700k/igt@i915_selftest@live_hugepages.html
[34]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-cfl-8700k/igt@i915_selftest@live_hugepages.html
- fi-bdw-5557u: [PASS][35] -> [INCOMPLETE][36]
[35]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-bdw-5557u/igt@i915_selftest@live_hugepages.html
[36]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-bdw-5557u/igt@i915_selftest@live_hugepages.html
- fi-skl-6600u: [PASS][37] -> [INCOMPLETE][38]
[37]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-6600u/igt@i915_selftest@live_hugepages.html
[38]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-6600u/igt@i915_selftest@live_hugepages.html
- fi-skl-guc: [PASS][39] -> [INCOMPLETE][40]
[39]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-guc/igt@i915_selftest@live_hugepages.html
[40]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-guc/igt@i915_selftest@live_hugepages.html
- fi-hsw-4770r: NOTRUN -> [INCOMPLETE][41]
[41]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-hsw-4770r/igt@i915_selftest@live_hugepages.html
- fi-kbl-7500u: [PASS][42] -> [INCOMPLETE][43]
[42]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-kbl-7500u/igt@i915_selftest@live_hugepages.html
[43]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-kbl-7500u/igt@i915_selftest@live_hugepages.html
- fi-hsw-peppy: NOTRUN -> [INCOMPLETE][44]
[44]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-hsw-peppy/igt@i915_selftest@live_hugepages.html
- fi-kbl-8809g: [PASS][45] -> [INCOMPLETE][46]
[45]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-kbl-8809g/igt@i915_selftest@live_hugepages.html
[46]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-kbl-8809g/igt@i915_selftest@live_hugepages.html
- fi-skl-gvtdvm: [PASS][47] -> [INCOMPLETE][48]
[47]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-gvtdvm/igt@i915_selftest@live_hugepages.html
[48]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-gvtdvm/igt@i915_selftest@live_hugepages.html
- fi-kbl-x1275: [PASS][49] -> [INCOMPLETE][50]
[49]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-kbl-x1275/igt@i915_selftest@live_hugepages.html
[50]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-kbl-x1275/igt@i915_selftest@live_hugepages.html
- fi-bsw-n3050: [PASS][51] -> [INCOMPLETE][52]
[51]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-bsw-n3050/igt@i915_selftest@live_hugepages.html
[52]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-bsw-n3050/igt@i915_selftest@live_hugepages.html
- fi-skl-iommu: [PASS][53] -> [INCOMPLETE][54]
[53]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-iommu/igt@i915_selftest@live_hugepages.html
[54]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-iommu/igt@i915_selftest@live_hugepages.html
* igt@i915_selftest@live_workarounds:
- fi-skl-6600u: [PASS][55] -> [DMESG-FAIL][56]
[55]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-6600u/igt@i915_selftest@live_workarounds.html
[56]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-6600u/igt@i915_selftest@live_workarounds.html
- fi-skl-6260u: [PASS][57] -> [DMESG-FAIL][58]
[57]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-skl-6260u/igt@i915_selftest@live_workarounds.html
[58]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-skl-6260u/igt@i915_selftest@live_workarounds.html
- fi-icl-u2: NOTRUN -> [DMESG-FAIL][59]
[59]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-icl-u2/igt@i915_selftest@live_workarounds.html
#### Suppressed ####
The following results come from untrusted machines, tests, or statuses.
They do not affect the overall result.
* igt@i915_selftest@live_workarounds:
- {fi-icl-guc}: [PASS][60] -> [DMESG-FAIL][61]
[60]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-icl-guc/igt@i915_selftest@live_workarounds.html
[61]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-icl-guc/igt@i915_selftest@live_workarounds.html
Known issues
------------
Here are the changes found in Patchwork_14169 that come from known issues:
### IGT changes ###
#### Issues hit ####
* igt@i915_module_load@reload:
- fi-blb-e6850: [PASS][62] -> [INCOMPLETE][63] ([fdo#107718])
[62]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-blb-e6850/igt@i915_module_load@reload.html
[63]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-blb-e6850/igt@i915_module_load@reload.html
* igt@i915_selftest@live_hugepages:
- fi-icl-u3: [PASS][64] -> [INCOMPLETE][65] ([fdo#107713] / [fdo#109663])
[64]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-icl-u3/igt@i915_selftest@live_hugepages.html
[65]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-icl-u3/igt@i915_selftest@live_hugepages.html
- fi-snb-2600: [PASS][66] -> [INCOMPLETE][67] ([fdo#105411])
[66]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-snb-2600/igt@i915_selftest@live_hugepages.html
[67]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-snb-2600/igt@i915_selftest@live_hugepages.html
- fi-apl-guc: [PASS][68] -> [INCOMPLETE][69] ([fdo#103927])
[68]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-apl-guc/igt@i915_selftest@live_hugepages.html
[69]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-apl-guc/igt@i915_selftest@live_hugepages.html
- fi-glk-dsi: [PASS][70] -> [INCOMPLETE][71] ([fdo#103359] / [k.org#198133])
[70]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-glk-dsi/igt@i915_selftest@live_hugepages.html
[71]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-glk-dsi/igt@i915_selftest@live_hugepages.html
- fi-cml-u2: [PASS][72] -> [INCOMPLETE][73] ([fdo#110566])
[72]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-cml-u2/igt@i915_selftest@live_hugepages.html
[73]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-cml-u2/igt@i915_selftest@live_hugepages.html
* igt@i915_selftest@live_sanitycheck:
- fi-icl-u3: [PASS][74] -> [DMESG-WARN][75] ([fdo#107724])
[74]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-icl-u3/igt@i915_selftest@live_sanitycheck.html
[75]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-icl-u3/igt@i915_selftest@live_sanitycheck.html
* igt@kms_chamelium@dp-edid-read:
- fi-cml-u2: [PASS][76] -> [FAIL][77] ([fdo#109483])
[76]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-cml-u2/igt@kms_chamelium@dp-edid-read.html
[77]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-cml-u2/igt@kms_chamelium@dp-edid-read.html
#### Possible fixes ####
* igt@gem_ctx_switch@legacy-render:
- fi-bxt-dsi: [INCOMPLETE][78] ([fdo#103927] / [fdo#111381]) -> [PASS][79]
[78]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-bxt-dsi/igt@gem_ctx_switch@legacy-render.html
[79]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-bxt-dsi/igt@gem_ctx_switch@legacy-render.html
* igt@i915_module_load@reload-with-fault-injection:
- fi-hsw-4770r: [DMESG-WARN][80] ([fdo#107732]) -> [PASS][81]
[80]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-hsw-4770r/igt@i915_module_load@reload-with-fault-injection.html
[81]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-hsw-4770r/igt@i915_module_load@reload-with-fault-injection.html
#### Warnings ####
* igt@runner@aborted:
- fi-hsw-4770r: [FAIL][82] ([fdo#107732] / [fdo#111249]) -> [FAIL][83] ([fdo#110326 ] / [fdo#111249])
[82]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6777/fi-hsw-4770r/igt@runner@aborted.html
[83]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/fi-hsw-4770r/igt@runner@aborted.html
{name}: This element is suppressed. This means it is ignored when computing
the status of the difference (SUCCESS, WARNING, or FAILURE).
[fdo#102505]: https://bugs.freedesktop.org/show_bug.cgi?id=102505
[fdo#103167]: https://bugs.freedesktop.org/show_bug.cgi?id=103167
[fdo#103359]: https://bugs.freedesktop.org/show_bug.cgi?id=103359
[fdo#103927]: https://bugs.freedesktop.org/show_bug.cgi?id=103927
[fdo#105411]: https://bugs.freedesktop.org/show_bug.cgi?id=105411
[fdo#107713]: https://bugs.freedesktop.org/show_bug.cgi?id=107713
[fdo#107718]: https://bugs.freedesktop.org/show_bug.cgi?id=107718
[fdo#107724]: https://bugs.freedesktop.org/show_bug.cgi?id=107724
[fdo#107732]: https://bugs.freedesktop.org/show_bug.cgi?id=107732
[fdo#109483]: https://bugs.freedesktop.org/show_bug.cgi?id=109483
[fdo#109663]: https://bugs.freedesktop.org/show_bug.cgi?id=109663
[fdo#109673]: https://bugs.freedesktop.org/show_bug.cgi?id=109673
[fdo#110326 ]: https://bugs.freedesktop.org/show_bug.cgi?id=110326
[fdo#110566]: https://bugs.freedesktop.org/show_bug.cgi?id=110566
[fdo#111045]: https://bugs.freedesktop.org/show_bug.cgi?id=111045
[fdo#111096]: https://bugs.freedesktop.org/show_bug.cgi?id=111096
[fdo#111249]: https://bugs.freedesktop.org/show_bug.cgi?id=111249
[fdo#111381]: https://bugs.freedesktop.org/show_bug.cgi?id=111381
[k.org#198133]: https://bugzilla.kernel.org/show_bug.cgi?id=198133
Participating hosts (53 -> 47)
------------------------------
Additional (2): fi-icl-u2 fi-gdg-551
Missing (8): fi-kbl-soraka fi-ilk-m540 fi-hsw-4200u fi-byt-squawks fi-bsw-cyan fi-icl-y fi-byt-clapper fi-bdw-samus
Build changes
-------------
* CI: CI-20190529 -> None
* Linux: CI_DRM_6777 -> Patchwork_14169
CI-20190529: 20190529
CI_DRM_6777: f3035d74f2d44bab3dbc6673f6660b447cbefd54 @ git://anongit.freedesktop.org/gfx-ci/linux
IGT_5148: 50390dd7adaccae21cafa85b866c17606cec94c3 @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
Patchwork_14169: 061e951611256a30d69c8dff84afb1adab619e45 @ git://anongit.freedesktop.org/gfx-ci/linux
== Linux commits ==
061e95161125 drm/i915: Serialise the fill BLT with the vma pinning
b23c41cf1147 drm/i915: Drop struct_mutex from around i915_retire_requests()
e2518334238f drm/i915: Move idle barrier cleanup into engine-pm
79f3b830651c drm/i915: Coordinate i915_active with its own mutex
5ac14777c21e drm/i915: Push the i915_active.retire into a worker
edba6cc97869 drm/i915: Pull i915_vma_pin under the vm->mutex
3631ac00b48a drm/i915: Mark up address spaces that may need to allocate
332370ea8b41 drm/i915: Make i915_vma.flags atomic_t for mutex reduction
3ec9eadcdacb drm/i915: Only track bound elements of the GTT
bb43a3b5113b drm/i915: Mark up obj->pin_global as being atomic
a0f807f2b948 drm/i915: Make shrink/unshrink be atomic
0f977fd1da12 dma-fence: Serialise signal enabling (dma_fence_enable_sw_signaling)
bb4855e5c4eb drm/i915/selftests: Teach igt_gpu_fill_dw() to take intel_context
d896cc87daca drm/i915/gtt: Preallocate Braswell top-level page directory
173c1bc89e03 drm/i915: Hold irq-off for the entire fake lock period
== Logs ==
For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_14169/
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 25+ messages in thread* ✗ Fi.CI.BAT: failure for series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev4)
2019-08-23 13:26 vm->mutex Chris Wilson
` (16 preceding siblings ...)
2019-08-23 17:43 ` ✗ Fi.CI.BAT: failure " Patchwork
@ 2019-08-23 20:06 ` Patchwork
17 siblings, 0 replies; 25+ messages in thread
From: Patchwork @ 2019-08-23 20:06 UTC (permalink / raw)
To: Chris Wilson; +Cc: intel-gfx
== Series Details ==
Series: series starting with [01/15] drm/i915: Hold irq-off for the entire fake lock period (rev4)
URL : https://patchwork.freedesktop.org/series/65705/
State : failure
== Summary ==
Applying: drm/i915: Hold irq-off for the entire fake lock period
Using index info to reconstruct a base tree...
M drivers/gpu/drm/i915/gt/intel_engine_pm.c
M drivers/gpu/drm/i915/gt/intel_reset.c
M drivers/gpu/drm/i915/gt/intel_timeline.c
M drivers/gpu/drm/i915/i915_gem.c
M drivers/gpu/drm/i915/i915_request.c
Falling back to patching base and 3-way merge...
Auto-merging drivers/gpu/drm/i915/i915_gem.c
No changes -- Patch already applied.
Applying: drm/i915/gtt: Preallocate Braswell top-level page directory
Using index info to reconstruct a base tree...
M drivers/gpu/drm/i915/gem/i915_gem_context.c
M drivers/gpu/drm/i915/i915_gem_gtt.c
Falling back to patching base and 3-way merge...
Auto-merging drivers/gpu/drm/i915/i915_gem_gtt.c
Auto-merging drivers/gpu/drm/i915/gem/i915_gem_context.c
CONFLICT (content): Merge conflict in drivers/gpu/drm/i915/gem/i915_gem_context.c
error: Failed to merge in the changes.
hint: Use 'git am --show-current-patch' to see the failed patch
Patch failed at 0002 drm/i915/gtt: Preallocate Braswell top-level page directory
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 25+ messages in thread