From: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
To: intel-gfx@lists.freedesktop.org
Subject: [Intel-gfx] [PATCH 12/23] drm/i915: Rework intel_context pinning to do everything outside of pin_mutex
Date: Wed, 8 Apr 2020 13:10:20 +0200 [thread overview]
Message-ID: <20200408111031.2330026-12-maarten.lankhorst@linux.intel.com> (raw)
In-Reply-To: <20200408111031.2330026-1-maarten.lankhorst@linux.intel.com>
Instead of doing everything inside of pin_mutex, we move all pinning
outside. Because i915_active has its own reference counting and
pinning is also having the same issues vs mutexes, we make sure
everything is pinned first, so the pinning in i915_active only needs
to bump refcounts. This allows us to take pin refcounts correctly
all the time.
Signed-off-by: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
---
drivers/gpu/drm/i915/gt/intel_context.c | 232 +++++++++++-------
drivers/gpu/drm/i915/gt/intel_context_types.h | 4 +-
drivers/gpu/drm/i915/gt/intel_lrc.c | 34 ++-
.../gpu/drm/i915/gt/intel_ring_submission.c | 13 +-
drivers/gpu/drm/i915/gt/mock_engine.c | 13 +-
5 files changed, 190 insertions(+), 106 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index e4aece20bc80..c039e87a46c4 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -93,79 +93,6 @@ static void intel_context_active_release(struct intel_context *ce)
i915_active_release(&ce->active);
}
-int __intel_context_do_pin(struct intel_context *ce)
-{
- int err;
-
- if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
- err = intel_context_alloc_state(ce);
- if (err)
- return err;
- }
-
- err = i915_active_acquire(&ce->active);
- if (err)
- return err;
-
- if (mutex_lock_interruptible(&ce->pin_mutex)) {
- err = -EINTR;
- goto out_release;
- }
-
- if (unlikely(intel_context_is_closed(ce))) {
- err = -ENOENT;
- goto out_unlock;
- }
-
- if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
- err = intel_context_active_acquire(ce);
- if (unlikely(err))
- goto out_unlock;
-
- err = ce->ops->pin(ce);
- if (unlikely(err))
- goto err_active;
-
- CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
- i915_ggtt_offset(ce->ring->vma),
- ce->ring->head, ce->ring->tail);
-
- smp_mb__before_atomic(); /* flush pin before it is visible */
- atomic_inc(&ce->pin_count);
- }
-
- GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
- GEM_BUG_ON(i915_active_is_idle(&ce->active));
- goto out_unlock;
-
-err_active:
- intel_context_active_release(ce);
-out_unlock:
- mutex_unlock(&ce->pin_mutex);
-out_release:
- i915_active_release(&ce->active);
- return err;
-}
-
-void intel_context_unpin(struct intel_context *ce)
-{
- if (!atomic_dec_and_test(&ce->pin_count))
- return;
-
- CE_TRACE(ce, "unpin\n");
- ce->ops->unpin(ce);
-
- /*
- * Once released, we may asynchronously drop the active reference.
- * As that may be the only reference keeping the context alive,
- * take an extra now so that it is not freed before we finish
- * dereferencing it.
- */
- intel_context_get(ce);
- intel_context_active_release(ce);
- intel_context_put(ce);
-}
-
static int __context_pin_state(struct i915_vma *vma)
{
unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
@@ -225,6 +152,138 @@ static void __ring_retire(struct intel_ring *ring)
i915_active_release(&ring->vma->active);
}
+static int intel_context_pre_pin(struct intel_context *ce)
+{
+ int err;
+
+ CE_TRACE(ce, "active\n");
+
+ err = __ring_active(ce->ring);
+ if (err)
+ return err;
+
+ err = intel_timeline_pin(ce->timeline);
+ if (err)
+ goto err_ring;
+
+ if (!ce->state)
+ return 0;
+
+ err = __context_pin_state(ce->state);
+ if (err)
+ goto err_timeline;
+
+
+ return 0;
+
+err_timeline:
+ intel_timeline_unpin(ce->timeline);
+err_ring:
+ __ring_retire(ce->ring);
+ return err;
+}
+
+static void intel_context_post_unpin(struct intel_context *ce)
+{
+ if (ce->state)
+ __context_unpin_state(ce->state);
+
+ intel_timeline_unpin(ce->timeline);
+ __ring_retire(ce->ring);
+}
+
+int __intel_context_do_pin(struct intel_context *ce)
+{
+ bool handoff = false;
+ void *vaddr;
+ int err = 0;
+
+ if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
+ err = intel_context_alloc_state(ce);
+ if (err)
+ return err;
+ }
+
+ /*
+ * We always pin the context/ring/timeline here, to ensure a pin
+ * refcount for __intel_context_active(), which prevent a lock
+ * inversion of ce->pin_mutex vs dma_resv_lock().
+ */
+ err = intel_context_pre_pin(ce);
+ if (err)
+ return err;
+
+ err = i915_active_acquire(&ce->active);
+ if (err)
+ goto err_ctx_unpin;
+
+ err = ce->ops->pre_pin(ce, &vaddr);
+ if (err)
+ goto err_release;
+
+ err = mutex_lock_interruptible(&ce->pin_mutex);
+ if (err)
+ goto err_post_unpin;
+
+ if (unlikely(intel_context_is_closed(ce))) {
+ err = -ENOENT;
+ goto err_unlock;
+ }
+
+ if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
+ err = intel_context_active_acquire(ce);
+ if (unlikely(err))
+ goto err_unlock;
+
+ err = ce->ops->pin(ce, vaddr);
+ if (err) {
+ intel_context_active_release(ce);
+ goto err_unlock;
+ }
+
+ CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
+ i915_ggtt_offset(ce->ring->vma),
+ ce->ring->head, ce->ring->tail);
+
+ handoff = true;
+ smp_mb__before_atomic(); /* flush pin before it is visible */
+ atomic_inc(&ce->pin_count);
+ }
+
+ GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
+
+err_unlock:
+ mutex_unlock(&ce->pin_mutex);
+err_post_unpin:
+ if (!handoff)
+ ce->ops->post_unpin(ce);
+err_release:
+ i915_active_release(&ce->active);
+err_ctx_unpin:
+ intel_context_post_unpin(ce);
+ return err;
+}
+
+void intel_context_unpin(struct intel_context *ce)
+{
+ if (!atomic_dec_and_test(&ce->pin_count))
+ return;
+
+ CE_TRACE(ce, "unpin\n");
+ ce->ops->unpin(ce);
+ ce->ops->post_unpin(ce);
+
+ /*
+ * Once released, we may asynchronously drop the active reference.
+ * As that may be the only reference keeping the context alive,
+ * take an extra now so that it is not freed before we finish
+ * dereferencing it.
+ */
+ intel_context_get(ce);
+ intel_context_active_release(ce);
+ intel_context_put(ce);
+}
+
__i915_active_call
static void __intel_context_retire(struct i915_active *active)
{
@@ -235,12 +294,7 @@ static void __intel_context_retire(struct i915_active *active)
intel_context_get_avg_runtime_ns(ce));
set_bit(CONTEXT_VALID_BIT, &ce->flags);
- if (ce->state)
- __context_unpin_state(ce->state);
-
- intel_timeline_unpin(ce->timeline);
- __ring_retire(ce->ring);
-
+ intel_context_post_unpin(ce);
intel_context_put(ce);
}
@@ -249,29 +303,25 @@ static int __intel_context_active(struct i915_active *active)
struct intel_context *ce = container_of(active, typeof(*ce), active);
int err;
- CE_TRACE(ce, "active\n");
-
intel_context_get(ce);
+ /* everything should already be activated by intel_context_pre_pin() */
err = __ring_active(ce->ring);
- if (err)
+ if (GEM_WARN_ON(err))
goto err_put;
err = intel_timeline_pin(ce->timeline);
- if (err)
+ if (GEM_WARN_ON(err))
goto err_ring;
- if (!ce->state)
- return 0;
-
- err = __context_pin_state(ce->state);
- if (err)
- goto err_timeline;
+ if (ce->state) {
+ GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->state->active));
+ __i915_vma_pin(ce->state);
+ i915_vma_make_unshrinkable(ce->state);
+ }
return 0;
-err_timeline:
- intel_timeline_unpin(ce->timeline);
err_ring:
__ring_retire(ce->ring);
err_put:
diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h
index 07cb83a0d017..395af0476a4e 100644
--- a/drivers/gpu/drm/i915/gt/intel_context_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_context_types.h
@@ -30,8 +30,10 @@ struct intel_ring;
struct intel_context_ops {
int (*alloc)(struct intel_context *ce);
- int (*pin)(struct intel_context *ce);
+ int (*pre_pin)(struct intel_context *ce, void **vaddr);
+ int (*pin)(struct intel_context *ce, void *vaddr);
void (*unpin)(struct intel_context *ce);
+ void (*post_unpin)(struct intel_context *ce);
void (*enter)(struct intel_context *ce);
void (*exit)(struct intel_context *ce);
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 19ffc7763683..081f7dad43b4 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -3110,7 +3110,10 @@ static void execlists_context_unpin(struct intel_context *ce)
{
check_redzone((void *)ce->lrc_reg_state - LRC_STATE_PN * PAGE_SIZE,
ce->engine);
+}
+static void execlists_context_post_unpin(struct intel_context *ce)
+{
i915_gem_object_unpin_map(ce->state->obj);
}
@@ -3140,20 +3143,23 @@ __execlists_update_reg_state(const struct intel_context *ce,
}
static int
-__execlists_context_pin(struct intel_context *ce,
- struct intel_engine_cs *engine)
+execlists_context_pre_pin(struct intel_context *ce, void **vaddr)
{
- void *vaddr;
-
GEM_BUG_ON(!ce->state);
GEM_BUG_ON(!i915_vma_is_pinned(ce->state));
- vaddr = i915_gem_object_pin_map(ce->state->obj,
- i915_coherent_map_type(engine->i915) |
+ *vaddr = i915_gem_object_pin_map(ce->state->obj,
+ i915_coherent_map_type(ce->engine->i915) |
I915_MAP_OVERRIDE);
- if (IS_ERR(vaddr))
- return PTR_ERR(vaddr);
+ return PTR_ERR_OR_ZERO(*vaddr);
+}
+
+static int
+__execlists_context_pin(struct intel_context *ce,
+ struct intel_engine_cs *engine,
+ void *vaddr)
+{
ce->lrc_desc = lrc_descriptor(ce, engine) | CTX_DESC_FORCE_RESTORE;
ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
__execlists_update_reg_state(ce, engine, ce->ring->tail);
@@ -3161,9 +3167,9 @@ __execlists_context_pin(struct intel_context *ce,
return 0;
}
-static int execlists_context_pin(struct intel_context *ce)
+static int execlists_context_pin(struct intel_context *ce, void *vaddr)
{
- return __execlists_context_pin(ce, ce->engine);
+ return __execlists_context_pin(ce, ce->engine, vaddr);
}
static int execlists_context_alloc(struct intel_context *ce)
@@ -3189,8 +3195,10 @@ static void execlists_context_reset(struct intel_context *ce)
static const struct intel_context_ops execlists_context_ops = {
.alloc = execlists_context_alloc,
+ .pre_pin = execlists_context_pre_pin,
.pin = execlists_context_pin,
.unpin = execlists_context_unpin,
+ .post_unpin = execlists_context_post_unpin,
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
@@ -4971,13 +4979,13 @@ static int virtual_context_alloc(struct intel_context *ce)
return __execlists_context_alloc(ce, ve->siblings[0]);
}
-static int virtual_context_pin(struct intel_context *ce)
+static int virtual_context_pin(struct intel_context *ce, void *vaddr)
{
struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
int err;
/* Note: we must use a real engine class for setting up reg state */
- err = __execlists_context_pin(ce, ve->siblings[0]);
+ err = __execlists_context_pin(ce, ve->siblings[0], vaddr);
if (err)
return err;
@@ -5010,8 +5018,10 @@ static void virtual_context_exit(struct intel_context *ce)
static const struct intel_context_ops virtual_context_ops = {
.alloc = virtual_context_alloc,
+ .pre_pin = execlists_context_pre_pin,
.pin = virtual_context_pin,
.unpin = execlists_context_unpin,
+ .post_unpin = execlists_context_post_unpin,
.enter = virtual_context_enter,
.exit = virtual_context_exit,
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
index d015f7b8b28e..d89475b8cfd6 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
@@ -1206,6 +1206,10 @@ static void __context_unpin_ppgtt(struct intel_context *ce)
}
static void ring_context_unpin(struct intel_context *ce)
+{
+}
+
+static void ring_context_post_unpin(struct intel_context *ce)
{
__context_unpin_ppgtt(ce);
}
@@ -1303,11 +1307,16 @@ static int ring_context_alloc(struct intel_context *ce)
return 0;
}
-static int ring_context_pin(struct intel_context *ce)
+static int ring_context_pre_pin(struct intel_context *ce, void **unused)
{
return __context_pin_ppgtt(ce);
}
+static int ring_context_pin(struct intel_context *ce, void *unused)
+{
+ return 0;
+}
+
static void ring_context_reset(struct intel_context *ce)
{
intel_ring_reset(ce->ring, ce->ring->emit);
@@ -1316,8 +1325,10 @@ static void ring_context_reset(struct intel_context *ce)
static const struct intel_context_ops ring_context_ops = {
.alloc = ring_context_alloc,
+ .pre_pin = ring_context_pre_pin,
.pin = ring_context_pin,
.unpin = ring_context_unpin,
+ .post_unpin = ring_context_post_unpin,
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c
index 4a53ded7c2dd..9ec42769e46e 100644
--- a/drivers/gpu/drm/i915/gt/mock_engine.c
+++ b/drivers/gpu/drm/i915/gt/mock_engine.c
@@ -132,6 +132,10 @@ static void mock_context_unpin(struct intel_context *ce)
{
}
+static void mock_context_post_unpin(struct intel_context *ce)
+{
+}
+
static void mock_context_destroy(struct kref *ref)
{
struct intel_context *ce = container_of(ref, typeof(*ce), ref);
@@ -165,7 +169,12 @@ static int mock_context_alloc(struct intel_context *ce)
return 0;
}
-static int mock_context_pin(struct intel_context *ce)
+static int mock_context_pre_pin(struct intel_context *ce, void **unused)
+{
+ return 0;
+}
+
+static int mock_context_pin(struct intel_context *ce, void *unused)
{
return 0;
}
@@ -177,8 +186,10 @@ static void mock_context_reset(struct intel_context *ce)
static const struct intel_context_ops mock_context_ops = {
.alloc = mock_context_alloc,
+ .pre_pin = mock_context_pre_pin,
.pin = mock_context_pin,
.unpin = mock_context_unpin,
+ .post_unpin = mock_context_post_unpin,
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
--
2.25.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
next prev parent reply other threads:[~2020-04-08 11:10 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-04-08 11:10 [Intel-gfx] [PATCH 01/23] perf/core: Only copy-to-user after completely unlocking all locks, v3 Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 02/23] Revert "drm/i915/gem: Drop relocation slowpath" Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 03/23] drm/i915: Add an implementation for i915_gem_ww_ctx locking, v2 Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 04/23] drm/i915: Remove locking from i915_gem_object_prepare_read/write Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 05/23] drm/i915: Parse command buffer earlier in eb_relocate(slow) Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 06/23] Revert "drm/i915/gem: Split eb_vma into its own allocation" Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 07/23] drm/i915: Use per object locking in execbuf, v7 Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 08/23] drm/i915: Use ww locking in intel_renderstate Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 09/23] drm/i915: Add ww context handling to context_barrier_task Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 10/23] drm/i915: Nuke arguments to eb_pin_engine Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 11/23] drm/i915: Pin engine before pinning all objects, v3 Maarten Lankhorst
2020-04-08 11:10 ` Maarten Lankhorst [this message]
2020-04-08 11:10 ` [Intel-gfx] [PATCH 13/23] drm/i915: Make sure execbuffer always passes ww state to i915_vma_pin Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 14/23] drm/i915: Convert i915_gem_object/client_blt.c to use ww locking as well, v2 Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 15/23] drm/i915: Kill last user of intel_context_create_request outside of selftests Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 16/23] drm/i915: Convert i915_perf to ww locking as well Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 17/23] drm/i915: Dirty hack to fix selftests locking inversion Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 18/23] drm/i915/selftests: Fix locking inversion in lrc selftest Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 19/23] drm/i915: Use ww pinning for intel_context_create_request() Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 20/23] drm/i915: Move i915_vma_lock in the selftests to avoid lock inversion, v2 Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 21/23] drm/i915: Add ww locking to vm_fault_gtt Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 22/23] drm/i915: Add ww locking to pin_to_display_plane Maarten Lankhorst
2020-04-08 11:10 ` [Intel-gfx] [PATCH 23/23] drm/i915: Ensure we hold the pin mutex Maarten Lankhorst
2020-04-08 14:18 ` [Intel-gfx] ✗ Fi.CI.CHECKPATCH: warning for series starting with [01/23] perf/core: Only copy-to-user after completely unlocking all locks, v3 Patchwork
2020-04-08 14:44 ` [Intel-gfx] ✗ Fi.CI.BAT: failure " Patchwork
2020-04-10 3:26 ` [Intel-gfx] ✗ Fi.CI.IGT: " Patchwork
-- strict thread matches above, loose matches on Subject: below --
2020-05-20 13:00 [Intel-gfx] [PATCH 01/23] Revert "drm/i915/gem: Drop relocation slowpath" Maarten Lankhorst
2020-05-20 13:00 ` [Intel-gfx] [PATCH 12/23] drm/i915: Rework intel_context pinning to do everything outside of pin_mutex Maarten Lankhorst
2020-05-11 14:29 [Intel-gfx] [PATCH 01/23] Revert "drm/i915/gem: Drop relocation slowpath" Maarten Lankhorst
2020-05-11 14:29 ` [Intel-gfx] [PATCH 12/23] drm/i915: Rework intel_context pinning to do everything outside of pin_mutex Maarten Lankhorst
2020-04-02 14:30 [Intel-gfx] [PATCH 01/23] perf/core: Only copy-to-user after completely unlocking all locks, v2 Maarten Lankhorst
2020-04-02 14:30 ` [Intel-gfx] [PATCH 12/23] drm/i915: Rework intel_context pinning to do everything outside of pin_mutex Maarten Lankhorst
2020-03-31 10:41 [Intel-gfx] [PATCH 01/23] Revert "drm/i915/gem: Drop relocation slowpath" Maarten Lankhorst
2020-03-31 10:41 ` [Intel-gfx] [PATCH 12/23] drm/i915: Rework intel_context pinning to do everything outside of pin_mutex Maarten Lankhorst
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200408111031.2330026-12-maarten.lankhorst@linux.intel.com \
--to=maarten.lankhorst@linux.intel.com \
--cc=intel-gfx@lists.freedesktop.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).